From c5cd245603e05fc244a99353cfec20f181787841 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 26 Jul 2013 16:31:11 -0400 Subject: [PATCH 1/2] feat(image_to_vm): Add support for uploading vm images After this I can make production AMI images available for download! :-D --- build_library/release_util.sh | 52 ++++++++++++++++++++-------------- build_library/vm_image_util.sh | 16 ++++++++++- image_to_vm.sh | 6 ++++ 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index bab54f07f6..ba4c94daaa 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -85,40 +85,50 @@ upload_packages() { make_digests() { local dirname=$(dirname "$1") - local filename=$(basename "$1") + local basename=$(basename "$1") + cd "${dirname}" - info "Computing DIGESTS for ${filename}" - echo -n > "${filename}.DIGESTS" - for hash in md5 sha1 sha512; do - echo "# $hash HASH" | tr "a-z" "A-Z" >> "${filename}.DIGESTS" - ${hash}sum "${filename}" >> "${filename}.DIGESTS" + echo -n > "${basename}.DIGESTS" + for filename in "$@"; do + filename=$(basename "$filename") + info "Computing DIGESTS for ${filename}" + for hash in md5 sha1 sha512; do + echo "# $hash HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS" + ${hash}sum "${filename}" >> "${basename}.DIGESTS" + done done cd - } +# Upload a image along with optional supporting files +# The image file must be the first argument upload_image() { [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0 [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first" - local BUILT_IMAGE="$1" + local uploads=() + local filename + for filename in "$@"; do + if [[ ! -f "${filename}" ]]; then + die "File '${filename}' does not exist!" + fi - if [[ ! -f "${BUILT_IMAGE}" ]]; then - die "Image '${BUILT_IMAGE}' does not exist!" - fi - - # Compress raw images - if [[ "${BUILT_IMAGE}" =~ \.(img|bin)$ ]]; then - info "Compressing ${BUILT_IMAGE##*/}" - $IMAGE_ZIPPER "${BUILT_IMAGE}" - BUILT_IMAGE="${BUILT_IMAGE}${IMAGE_ZIPEXT}" - fi + # Compress disk images + if [[ "${filename}" =~ \.(img|bin|vdi|vmdk)$ ]]; then + info "Compressing ${filename##*/}" + $IMAGE_ZIPPER -f "${filename}" + uploads+=( "${filename}${IMAGE_ZIPEXT}" ) + else + uploads+=( "${filename}" ) + fi + done # For consistency generate a .DIGESTS file similar to the one catalyst # produces for the SDK tarballs and up upload it too. - make_digests "${BUILT_IMAGE}" + make_digests "${uploads[@]}" + uploads+=( "${uploads[0]}.DIGESTS" ) - local log_msg="${BUILT_IMAGE##*/}" + local log_msg="${1##*/}" local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}" - upload_files "${log_msg}" "${def_upload_path}" "" \ - "${BUILT_IMAGE}" "${BUILT_IMAGE}.DIGESTS" + upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}" } diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 3dcf608c5d..cf0dd573f1 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -26,6 +26,9 @@ VM_README= VM_NAME= VM_UUID= +# Contains a list of all generated files +VM_GENERATED_FILES=() + ## DEFAULT # If set to 1 use a hybrid GPT/MBR format instead of plain GPT IMG_DEFAULT_HYBRID_MBR=0 @@ -228,8 +231,9 @@ write_vm_disk() { fi local disk_format=$(_get_vm_opt DISK_FORMAT) - info "Writing $disk_format image to $(relpath "${VM_DST_IMG}")" + info "Writing $disk_format image $(basename "${VM_DST_IMG}")" _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}" + VM_GENERATED_FILES+=( "${VM_DST_IMG}" ) } _write_hybrid_mbr() { @@ -319,6 +323,8 @@ qemu-system-x86_64 -curses -m ${vm_mem} -readconfig "${conf_path##*/}" SSH into that host with: ssh 127.0.0.1 -p 2222 EOF + + VM_GENERATED_FILES+=( "${conf_path}" "${VM_README}" ) } # Generate the vmware config file @@ -346,6 +352,7 @@ guestOS = "otherlinux" ethernet0.addressType = "generated" floppy0.present = "FALSE"" EOF + VM_GENERATED_FILES+=( "${vmx_path}" ) } # Generate a new-style (xl) Xen config file for both pvgrub and pygrub @@ -393,6 +400,7 @@ xl console ${VM_NAME} Kill the vm with: xl destroy ${VM_NAME} EOF + VM_GENERATED_FILES+=( "${pygrub}" "${pvgrub}" "${VM_README}" ) } vm_cleanup() { @@ -401,6 +409,12 @@ vm_cleanup() { } print_readme() { + local filename + info "Files written to $(relpath "$(dirname "${VM_DST_IMG}")")" + for filename in "${VM_GENERATED_FILES[@]}"; do + info " - $(basename "${filename}")" + done + if [[ -f "${VM_README}" ]]; then cat "${VM_README}" fi diff --git a/image_to_vm.sh b/image_to_vm.sh index 097cc338c4..a91f20c9f3 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -52,6 +52,9 @@ DEFINE_boolean prod_image "${FLAGS_FALSE}" \ DEFINE_string to "" \ "Destination folder for VM output file(s)" +# include upload options +. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 + # Parse command line FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" @@ -116,6 +119,9 @@ write_vm_conf "${FLAGS_mem}" vm_cleanup trap - EXIT +# Optionally upload all of our hard work +upload_image "${VM_GENERATED_FILES[@]}" + # Ready to set sail! okboat command_completed From 37c56b64de023a4fb54d0562e79c56c5e4ce8751 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Fri, 26 Jul 2013 17:45:27 -0400 Subject: [PATCH 2/2] feat(build_ebs_on_ec2): Fetch prod images by version, misc cleanup. The build host will start generating production ami disk images so to simplify the next step this script can automatically fetch them from that location by version. The default sticks with the existing 'master' versioning scheme. Added logging and turned off -x by default to make the output log more readable. Removing the zip_and_ship script since it isn't useful with officially built disk images and only works with locally built images and a very particular ec2 host. A different long term automation scheme will have to be found. --- oem/ami/build_ebs_on_ec2.sh | 118 +++++++++++++++++++++++++++--------- oem/ami/zip_and_ship_ami.sh | 10 --- 2 files changed, 89 insertions(+), 39 deletions(-) delete mode 100755 oem/ami/zip_and_ship_ami.sh diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh index c060836355..ee4867f148 100755 --- a/oem/ami/build_ebs_on_ec2.sh +++ b/oem/ami/build_ebs_on_ec2.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash # # This expects to run on an EC2 instance. # @@ -9,6 +9,9 @@ # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html # we need pv-grub-hd00 x86_64 +# Set pipefail along with -e in hopes that we catch more errors +set -e -o pipefail + declare -A AKI AKI["us-east-1"]=aki-b4aa75dd AKI["us-west-1"]=aki-eb7e26ae @@ -21,43 +24,90 @@ AKI["sa-east-1"]=aki-c88f51d5 # AKI["gov-west-1"]=aki-75a4c056 readonly COREOS_EPOCH=1372636800 -TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) -EXTRA_VERSION=$(date +%H-%M) +VERSION="master" +IMAGE="coreos_production_ami_image.bin.bz2" +URL_FMT="http://storage.core-os.net/coreos/amd64-generic/%s/$IMAGE" +IMG_URL="" +IMG_PATH="" -if [ -z "$1" ]; then - echo "usage: $0 [http|path/to/bin.bz2]" - exit 1 +USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2] +Options: + -V VERSION Set the version of this AMI, default is 'master' + -p PATH Path to compressed disk image, overrides -u + -u URL URL to compressed disk image, derived from -V if unset. + -K KEY Path to Amazon API private key. + -C CERT Path to Amazon API key certificate. + -h this ;-) + -v Verbose, see all the things! + +This script must be run from an ec2 host with the ec2 tools installed. +" + +while getopts "V:p:u:K:C:hv" OPTION +do + case $OPTION in + V) VERSION="$OPTARG";; + p) IMG_PATH="$OPTARG";; + u) IMG_URL="$OPTARG";; + K) export EC2_PRIVATE_KEY="$OPTARG";; + C) export EC2_CERT="$OPTARG";; + h) echo "$USAGE"; exit;; + v) set -x;; + *) exit 1;; + esac +done + +if [[ $(id -u) -ne 0 ]]; then + echo "$0: This command must be run as root!" >&2 + exit 1 +fi + +# Quick sanity check that the image exists +if [[ -n "$IMG_PATH" ]]; then + if [[ ! -f "$IMG_PATH" ]]; then + echo "$0: Image path does not exist: $IMG_PATH" >&2 + exit 1 + fi + IMG_URL=$(basename "$IMG_PATH") +else + if [[ -z "$IMG_URL" ]]; then + IMG_URL=$(printf "$URL_FMT" "$VERSION") + fi + if ! curl -s --head "$IMG_URL" >/dev/null; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi +fi + +if [[ "$VERSION" == "master" ]]; then + # Come up with something more descriptive and timestamped + TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) + VERSION="master-${TODAYS_VERSION}-$(date +%H-%M)" fi -binurl=$1 # Size of AMI file system +# TODO: Perhaps define size and arch in a metadata file image_to_vm creates? size=8 # GB - arch=x86_64 arch2=amd64 ephemeraldev=/dev/sdb -version="master-$TODAYS_VERSION-$EXTRA_VERSION" - -#TBD -name="CoreOS-$version" -description="CoreOS master" - -export EC2_CERT=$(echo /tmp/*cert*.pem) -export EC2_PRIVATE_KEY=$(echo /tmp/*pk*.pem) +# The name has a limited set of allowed characterrs +name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VERSION") +description="CoreOS $VERSION" zoneurl=http://instance-data/latest/meta-data/placement/availability-zone zone=$(curl -s $zoneurl) region=$(echo $zone | sed 's/.$//') - -# this is defined in: build_library/ami_constants.sh akiid=${AKI[$region]} if [ -z "$akiid" ]; then - echo "$0: Can't identify AKI, using region: $region"; + echo "$0: Can't identify AKI, using region: $region" >&2 exit 1 fi +# TODO: Once we sort out a long-term scheme for generating AMIs this likely +# will go away. What if we want to generate AMIs from CoreOS?!?!? if [ -z "$(which ec2-attach-volume)" ]; then # Update and install Ubuntu packages export DEBIAN_FRONTEND=noninteractive @@ -69,12 +119,14 @@ if [ -z "$(which ec2-attach-volume)" ]; then ec2-ami-tools fi -export EC2_URL=http://ec2.$region.amazonaws.com +export EC2_URL="http://ec2.${region}.amazonaws.com" +echo "Building AMI in zone $zone, region id $akiid" # Create and mount temporary EBS volume with file system to hold new AMI image volumeid=$(ec2-create-volume --size $size --availability-zone $zone | cut -f2) -instanceid=$(wget -qO- http://instance-data/latest/meta-data/instance-id) +instanceid=$(curl -s http://instance-data/latest/meta-data/instance-id) +echo "Attaching new volume $volumeid locally (instance $instanceid)" ec2-attach-volume --device /dev/sdi --instance "$instanceid" "$volumeid" while [ ! -e /dev/sdi -a ! -e /dev/xvdi ] do sleep 3; done @@ -84,13 +136,21 @@ else dev=/dev/sdi fi +echo "Attached volume $volumeid as $dev" +echo "Writing image from $IMG_URL to $dev" + # if it is on the local fs, just use it, otherwise try to download it -if [ -e "$binurl" ]; then - bunzip2 -c $binurl | dd of=$dev bs=128M +if [[ -n "$IMG_PATH" ]]; then + if [[ "$IMG_PATH" =~ \.bz2$ ]]; then + bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M + else + dd if="$IMG_PATH" of=$dev bs=1M + fi else - curl -s $binurl | bunzip2 | dd of=$dev bs=128M + curl "$IMG_URL" | bunzip2 | dd of=$dev bs=1M fi +echo "Detaching $volumeid and creating snapshot" ec2-detach-volume "$volumeid" while ec2-describe-volumes "$volumeid" | grep -q ATTACHMENT do sleep 3; done @@ -98,7 +158,7 @@ snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2) while ec2-describe-snapshots "$snapshotid" | grep -q pending do sleep 30; done -# Register the snapshot as a new AMI +echo "Created snapshot $snapshotid, registering as a new AMI" amiid=$(ec2-register \ --name "$name" \ --description "$description" \ @@ -111,9 +171,9 @@ amiid=$(ec2-register \ ec2-delete-volume "$volumeid" cat <