diff --git a/build_library/release_util.sh b/build_library/release_util.sh index bab54f07f6..ba4c94daaa 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -85,40 +85,50 @@ upload_packages() { make_digests() { local dirname=$(dirname "$1") - local filename=$(basename "$1") + local basename=$(basename "$1") + cd "${dirname}" - info "Computing DIGESTS for ${filename}" - echo -n > "${filename}.DIGESTS" - for hash in md5 sha1 sha512; do - echo "# $hash HASH" | tr "a-z" "A-Z" >> "${filename}.DIGESTS" - ${hash}sum "${filename}" >> "${filename}.DIGESTS" + echo -n > "${basename}.DIGESTS" + for filename in "$@"; do + filename=$(basename "$filename") + info "Computing DIGESTS for ${filename}" + for hash in md5 sha1 sha512; do + echo "# $hash HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS" + ${hash}sum "${filename}" >> "${basename}.DIGESTS" + done done cd - } +# Upload a image along with optional supporting files +# The image file must be the first argument upload_image() { [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" - local BUILT_IMAGE="$1" + local uploads=() + local filename + for filename in "$@"; do + if [[ ! -f "${filename}" ]]; then + die "File '${filename}' does not exist!" + fi - if [[ ! -f "${BUILT_IMAGE}" ]]; then - die "Image '${BUILT_IMAGE}' does not exist!" - fi - - # Compress raw images - if [[ "${BUILT_IMAGE}" =~ \.(img|bin)$ ]]; then - info "Compressing ${BUILT_IMAGE##*/}" - $IMAGE_ZIPPER "${BUILT_IMAGE}" - BUILT_IMAGE="${BUILT_IMAGE}${IMAGE_ZIPEXT}" - fi + # Compress disk images + if [[ "${filename}" =~ \.(img|bin|vdi|vmdk)$ ]]; then + info "Compressing ${filename##*/}" + $IMAGE_ZIPPER -f "${filename}" + uploads+=( "${filename}${IMAGE_ZIPEXT}" ) + else + uploads+=( "${filename}" ) + fi + done # For consistency generate a .DIGESTS file similar to the one catalyst # produces for the SDK tarballs and up upload it too. - make_digests "${BUILT_IMAGE}" + make_digests "${uploads[@]}" + uploads+=( "${uploads[0]}.DIGESTS" ) - local log_msg="${BUILT_IMAGE##*/}" + local log_msg="${1##*/}" local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}" - upload_files "${log_msg}" "${def_upload_path}" "" \ - "${BUILT_IMAGE}" "${BUILT_IMAGE}.DIGESTS" + upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}" } diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 3dcf608c5d..cf0dd573f1 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -26,6 +26,9 @@ VM_README= VM_NAME= VM_UUID= +# Contains a list of all generated files +VM_GENERATED_FILES=() + ## DEFAULT # If set to 1 use a hybrid GPT/MBR format instead of plain GPT IMG_DEFAULT_HYBRID_MBR=0 @@ -228,8 +231,9 @@ write_vm_disk() { fi local disk_format=$(_get_vm_opt DISK_FORMAT) - info "Writing $disk_format image to $(relpath "${VM_DST_IMG}")" + info "Writing $disk_format image $(basename "${VM_DST_IMG}")" _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}" + VM_GENERATED_FILES+=( "${VM_DST_IMG}" ) } _write_hybrid_mbr() { @@ -319,6 +323,8 @@ qemu-system-x86_64 -curses -m ${vm_mem} -readconfig "${conf_path##*/}" SSH into that host with: ssh 127.0.0.1 -p 2222 EOF + + VM_GENERATED_FILES+=( "${conf_path}" "${VM_README}" ) } # Generate the vmware config file @@ -346,6 +352,7 @@ guestOS = "otherlinux" ethernet0.addressType = "generated" floppy0.present = "FALSE"" EOF + VM_GENERATED_FILES+=( "${vmx_path}" ) } # Generate a new-style (xl) Xen config file for both pvgrub and pygrub @@ -393,6 +400,7 @@ xl console ${VM_NAME} Kill the vm with: xl destroy ${VM_NAME} EOF + VM_GENERATED_FILES+=( "${pygrub}" "${pvgrub}" "${VM_README}" ) } vm_cleanup() { @@ -401,6 +409,12 @@ vm_cleanup() { } print_readme() { + local filename + info "Files written to $(relpath "$(dirname "${VM_DST_IMG}")")" + for filename in "${VM_GENERATED_FILES[@]}"; do + info " - $(basename "${filename}")" + done + if [[ -f "${VM_README}" ]]; then cat "${VM_README}" fi diff --git a/image_to_vm.sh b/image_to_vm.sh index 097cc338c4..a91f20c9f3 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -52,6 +52,9 @@ DEFINE_boolean prod_image "${FLAGS_FALSE}" \ DEFINE_string to "" \ "Destination folder for VM output file(s)" +# include upload options +. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 + # Parse command line FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" @@ -116,6 +119,9 @@ write_vm_conf "${FLAGS_mem}" vm_cleanup trap - EXIT +# Optionally upload all of our hard work +upload_image "${VM_GENERATED_FILES[@]}" + # Ready to set sail! okboat command_completed diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh index c060836355..ee4867f148 100755 --- a/oem/ami/build_ebs_on_ec2.sh +++ b/oem/ami/build_ebs_on_ec2.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash # # This expects to run on an EC2 instance. # @@ -9,6 +9,9 @@ # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html # we need pv-grub-hd00 x86_64 +# Set pipefail along with -e in hopes that we catch more errors +set -e -o pipefail + declare -A AKI AKI["us-east-1"]=aki-b4aa75dd AKI["us-west-1"]=aki-eb7e26ae @@ -21,43 +24,90 @@ AKI["sa-east-1"]=aki-c88f51d5 # AKI["gov-west-1"]=aki-75a4c056 readonly COREOS_EPOCH=1372636800 -TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) -EXTRA_VERSION=$(date +%H-%M) +VERSION="master" +IMAGE="coreos_production_ami_image.bin.bz2" +URL_FMT="http://storage.core-os.net/coreos/amd64-generic/%s/$IMAGE" +IMG_URL="" +IMG_PATH="" -if [ -z "$1" ]; then - echo "usage: $0 [http|path/to/bin.bz2]" - exit 1 +USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2] +Options: + -V VERSION Set the version of this AMI, default is 'master' + -p PATH Path to compressed disk image, overrides -u + -u URL URL to compressed disk image, derived from -V if unset. + -K KEY Path to Amazon API private key. + -C CERT Path to Amazon API key certificate. + -h this ;-) + -v Verbose, see all the things! + +This script must be run from an ec2 host with the ec2 tools installed. +" + +while getopts "V:p:u:K:C:hv" OPTION +do + case $OPTION in + V) VERSION="$OPTARG";; + p) IMG_PATH="$OPTARG";; + u) IMG_URL="$OPTARG";; + K) export EC2_PRIVATE_KEY="$OPTARG";; + C) export EC2_CERT="$OPTARG";; + h) echo "$USAGE"; exit;; + v) set -x;; + *) exit 1;; + esac +done + +if [[ $(id -u) -ne 0 ]]; then + echo "$0: This command must be run as root!" >&2 + exit 1 +fi + +# Quick sanity check that the image exists +if [[ -n "$IMG_PATH" ]]; then + if [[ ! -f "$IMG_PATH" ]]; then + echo "$0: Image path does not exist: $IMG_PATH" >&2 + exit 1 + fi + IMG_URL=$(basename "$IMG_PATH") +else + if [[ -z "$IMG_URL" ]]; then + IMG_URL=$(printf "$URL_FMT" "$VERSION") + fi + if ! curl -s --head "$IMG_URL" >/dev/null; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi +fi + +if [[ "$VERSION" == "master" ]]; then + # Come up with something more descriptive and timestamped + TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) + VERSION="master-${TODAYS_VERSION}-$(date +%H-%M)" fi -binurl=$1 # Size of AMI file system +# TODO: Perhaps define size and arch in a metadata file image_to_vm creates? size=8 # GB - arch=x86_64 arch2=amd64 ephemeraldev=/dev/sdb -version="master-$TODAYS_VERSION-$EXTRA_VERSION" - -#TBD -name="CoreOS-$version" -description="CoreOS master" - -export EC2_CERT=$(echo /tmp/*cert*.pem) -export EC2_PRIVATE_KEY=$(echo /tmp/*pk*.pem) +# The name has a limited set of allowed characterrs +name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VERSION") +description="CoreOS $VERSION" zoneurl=http://instance-data/latest/meta-data/placement/availability-zone zone=$(curl -s $zoneurl) region=$(echo $zone | sed 's/.$//') - -# this is defined in: build_library/ami_constants.sh akiid=${AKI[$region]} if [ -z "$akiid" ]; then - echo "$0: Can't identify AKI, using region: $region"; + echo "$0: Can't identify AKI, using region: $region" >&2 exit 1 fi +# TODO: Once we sort out a long-term scheme for generating AMIs this likely +# will go away. What if we want to generate AMIs from CoreOS?!?!? if [ -z "$(which ec2-attach-volume)" ]; then # Update and install Ubuntu packages export DEBIAN_FRONTEND=noninteractive @@ -69,12 +119,14 @@ if [ -z "$(which ec2-attach-volume)" ]; then ec2-ami-tools fi -export EC2_URL=http://ec2.$region.amazonaws.com +export EC2_URL="http://ec2.${region}.amazonaws.com" +echo "Building AMI in zone $zone, region id $akiid" # Create and mount temporary EBS volume with file system to hold new AMI image volumeid=$(ec2-create-volume --size $size --availability-zone $zone | cut -f2) -instanceid=$(wget -qO- http://instance-data/latest/meta-data/instance-id) +instanceid=$(curl -s http://instance-data/latest/meta-data/instance-id) +echo "Attaching new volume $volumeid locally (instance $instanceid)" ec2-attach-volume --device /dev/sdi --instance "$instanceid" "$volumeid" while [ ! -e /dev/sdi -a ! -e /dev/xvdi ] do sleep 3; done @@ -84,13 +136,21 @@ else dev=/dev/sdi fi +echo "Attached volume $volumeid as $dev" +echo "Writing image from $IMG_URL to $dev" + # if it is on the local fs, just use it, otherwise try to download it -if [ -e "$binurl" ]; then - bunzip2 -c $binurl | dd of=$dev bs=128M +if [[ -n "$IMG_PATH" ]]; then + if [[ "$IMG_PATH" =~ \.bz2$ ]]; then + bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M + else + dd if="$IMG_PATH" of=$dev bs=1M + fi else - curl -s $binurl | bunzip2 | dd of=$dev bs=128M + curl "$IMG_URL" | bunzip2 | dd of=$dev bs=1M fi +echo "Detaching $volumeid and creating snapshot" ec2-detach-volume "$volumeid" while ec2-describe-volumes "$volumeid" | grep -q ATTACHMENT do sleep 3; done @@ -98,7 +158,7 @@ snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2) while ec2-describe-snapshots "$snapshotid" | grep -q pending do sleep 30; done -# Register the snapshot as a new AMI +echo "Created snapshot $snapshotid, registering as a new AMI" amiid=$(ec2-register \ --name "$name" \ --description "$description" \ @@ -111,9 +171,9 @@ amiid=$(ec2-register \ ec2-delete-volume "$volumeid" cat <