Merge pull request #61 from marineam/vm_upload

Vm upload

build_library/release_util.sh

@@ -85,40 +85,50 @@ upload_packages() {
 
 make_digests() {
     local dirname=$(dirname "$1")
-    local filename=$(basename "$1")
+    local basename=$(basename "$1")
+
     cd "${dirname}"
-    info "Computing DIGESTS for ${filename}"
+    echo -n > "${basename}.DIGESTS"
-    echo -n > "${filename}.DIGESTS"
+    for filename in "$@"; do
-    for hash in md5 sha1 sha512; do
+        filename=$(basename "$filename")
-        echo "# $hash HASH" | tr "a-z" "A-Z" >> "${filename}.DIGESTS"
+        info "Computing DIGESTS for ${filename}"
-        ${hash}sum "${filename}" >> "${filename}.DIGESTS"
+        for hash in md5 sha1 sha512; do
+            echo "# $hash HASH" | tr "a-z" "A-Z" >> "${basename}.DIGESTS"
+            ${hash}sum "${filename}" >> "${basename}.DIGESTS"
+        done
     done
     cd -
 }
 
+# Upload a image along with optional supporting files
+# The image file must be the first argument
 upload_image() {
     [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]] || return 0
     [[ -n "${BOARD}" ]] || die "board_options.sh must be sourced first"
 
-    local BUILT_IMAGE="$1"
+    local uploads=()
+    local filename
+    for filename in "$@"; do
+        if [[ ! -f "${filename}" ]]; then
+            die "File '${filename}' does not exist!"
+        fi
 
-    if [[ ! -f "${BUILT_IMAGE}" ]]; then
+        # Compress disk images
-        die "Image '${BUILT_IMAGE}' does not exist!"
+        if [[ "${filename}" =~ \.(img|bin|vdi|vmdk)$ ]]; then
-    fi
+            info "Compressing ${filename##*/}"
-
+            $IMAGE_ZIPPER -f "${filename}"
-    # Compress raw images
+            uploads+=( "${filename}${IMAGE_ZIPEXT}" )
-    if [[ "${BUILT_IMAGE}" =~ \.(img|bin)$ ]]; then
+        else
-        info "Compressing ${BUILT_IMAGE##*/}"
+            uploads+=( "${filename}" )
-        $IMAGE_ZIPPER "${BUILT_IMAGE}"
+        fi
-        BUILT_IMAGE="${BUILT_IMAGE}${IMAGE_ZIPEXT}"
+    done
-    fi
 
     # For consistency generate a .DIGESTS file similar to the one catalyst
     # produces for the SDK tarballs and up upload it too.
-    make_digests "${BUILT_IMAGE}"
+    make_digests "${uploads[@]}"
+    uploads+=( "${uploads[0]}.DIGESTS" )
 
-    local log_msg="${BUILT_IMAGE##*/}"
+    local log_msg="${1##*/}"
     local def_upload_path="${UPLOAD_ROOT}/${BOARD}/${COREOS_VERSION_STRING}"
-    upload_files "${log_msg}" "${def_upload_path}" "" \
+    upload_files "${log_msg}" "${def_upload_path}" "" "${uploads[@]}"
-        "${BUILT_IMAGE}" "${BUILT_IMAGE}.DIGESTS"
 }

build_library/vm_image_util.sh

@@ -26,6 +26,9 @@ VM_README=
 VM_NAME=
 VM_UUID=
 
+# Contains a list of all generated files
+VM_GENERATED_FILES=()
+
 ## DEFAULT
 # If set to 1 use a hybrid GPT/MBR format instead of plain GPT
 IMG_DEFAULT_HYBRID_MBR=0
@@ -228,8 +231,9 @@ write_vm_disk() {
     fi
 
     local disk_format=$(_get_vm_opt DISK_FORMAT)
-    info "Writing $disk_format image to $(relpath "${VM_DST_IMG}")"
+    info "Writing $disk_format image $(basename "${VM_DST_IMG}")"
     _write_${disk_format}_disk "${VM_TMP_IMG}" "${VM_DST_IMG}"
+    VM_GENERATED_FILES+=( "${VM_DST_IMG}" )
 }
 
 _write_hybrid_mbr() {
@@ -319,6 +323,8 @@ qemu-system-x86_64 -curses -m ${vm_mem} -readconfig "${conf_path##*/}"
 SSH into that host with:
 ssh 127.0.0.1 -p 2222
 EOF
+
+    VM_GENERATED_FILES+=( "${conf_path}" "${VM_README}" )
 }
 
 # Generate the vmware config file
@@ -346,6 +352,7 @@ guestOS = "otherlinux"
 ethernet0.addressType = "generated"
 floppy0.present = "FALSE""
 EOF
+    VM_GENERATED_FILES+=( "${vmx_path}" )
 }
 
 # Generate a new-style (xl) Xen config file for both pvgrub and pygrub
@@ -393,6 +400,7 @@ xl console ${VM_NAME}
 Kill the vm with:
 xl destroy ${VM_NAME}
 EOF
+    VM_GENERATED_FILES+=( "${pygrub}" "${pvgrub}" "${VM_README}" )
 }
 
 vm_cleanup() {
@@ -401,6 +409,12 @@ vm_cleanup() {
 }
 
 print_readme() {
+    local filename
+    info "Files written to $(relpath "$(dirname "${VM_DST_IMG}")")"
+    for filename in "${VM_GENERATED_FILES[@]}"; do
+        info " - $(basename "${filename}")"
+    done
+
     if [[ -f "${VM_README}" ]]; then
         cat "${VM_README}"
     fi

image_to_vm.sh

@@ -52,6 +52,9 @@ DEFINE_boolean prod_image "${FLAGS_FALSE}" \
 DEFINE_string to "" \
   "Destination folder for VM output file(s)"
 
+# include upload options
+. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
+
 # Parse command line
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -116,6 +119,9 @@ write_vm_conf "${FLAGS_mem}"
 vm_cleanup
 trap - EXIT
 
+# Optionally upload all of our hard work
+upload_image "${VM_GENERATED_FILES[@]}"
+
 # Ready to set sail!
 okboat
 command_completed

oem/ami/build_ebs_on_ec2.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash
 #
 # This expects to run on an EC2 instance.
 #
@@ -9,6 +9,9 @@
 #  http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html
 # we need pv-grub-hd00 x86_64
 
+# Set pipefail along with -e in hopes that we catch more errors
+set -e -o pipefail
+
 declare -A AKI
 AKI["us-east-1"]=aki-b4aa75dd 
 AKI["us-west-1"]=aki-eb7e26ae
@@ -21,43 +24,90 @@ AKI["sa-east-1"]=aki-c88f51d5
 # AKI["gov-west-1"]=aki-75a4c056
 
 readonly COREOS_EPOCH=1372636800
-TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
+VERSION="master"
-EXTRA_VERSION=$(date +%H-%M)
+IMAGE="coreos_production_ami_image.bin.bz2"
+URL_FMT="http://storage.core-os.net/coreos/amd64-generic/%s/$IMAGE"
+IMG_URL=""
+IMG_PATH=""
 
-if [ -z "$1" ]; then
+USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2]
-	echo "usage: $0 [http|path/to/bin.bz2]"
+Options:
-	exit 1
+    -V VERSION  Set the version of this AMI, default is 'master'
+    -p PATH     Path to compressed disk image, overrides -u
+    -u URL      URL to compressed disk image, derived from -V if unset.
+    -K KEY      Path to Amazon API private key.
+    -C CERT     Path to Amazon API key certificate.
+    -h          this ;-)
+    -v          Verbose, see all the things!
+
+This script must be run from an ec2 host with the ec2 tools installed.
+"
+
+while getopts "V:p:u:K:C:hv" OPTION
+do
+    case $OPTION in
+        V) VERSION="$OPTARG";;
+        p) IMG_PATH="$OPTARG";;
+        u) IMG_URL="$OPTARG";;
+        K) export EC2_PRIVATE_KEY="$OPTARG";;
+        C) export EC2_CERT="$OPTARG";;
+        h) echo "$USAGE"; exit;;
+        v) set -x;;
+        *) exit 1;;
+    esac
+done
+
+if [[ $(id -u) -ne 0 ]]; then
+    echo "$0: This command must be run as root!" >&2
+    exit 1
+fi
+
+# Quick sanity check that the image exists
+if [[ -n "$IMG_PATH" ]]; then
+    if [[ ! -f "$IMG_PATH" ]]; then
+        echo "$0: Image path does not exist: $IMG_PATH" >&2
+        exit 1
+    fi
+    IMG_URL=$(basename "$IMG_PATH")
+else
+    if [[ -z "$IMG_URL" ]]; then
+        IMG_URL=$(printf "$URL_FMT" "$VERSION")
+    fi
+    if ! curl -s --head "$IMG_URL" >/dev/null; then
+        echo "$0: Image URL unavailable: $IMG_URL" >&2
+        exit 1
+    fi
+fi
+
+if [[ "$VERSION" == "master" ]]; then
+    # Come up with something more descriptive and timestamped
+    TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
+    VERSION="master-${TODAYS_VERSION}-$(date +%H-%M)"
 fi
 
-binurl=$1
 
 # Size of AMI file system
+# TODO: Perhaps define size and arch in a metadata file image_to_vm creates?
 size=8 # GB
-
 arch=x86_64
 arch2=amd64
 ephemeraldev=/dev/sdb
-version="master-$TODAYS_VERSION-$EXTRA_VERSION"
+# The name has a limited set of allowed characterrs
-
+name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VERSION")
-#TBD
+description="CoreOS $VERSION"
-name="CoreOS-$version"
-description="CoreOS master"
-
-export EC2_CERT=$(echo /tmp/*cert*.pem)
-export EC2_PRIVATE_KEY=$(echo /tmp/*pk*.pem)
 
 zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
 zone=$(curl -s $zoneurl)
 region=$(echo $zone | sed 's/.$//')
-
-# this is defined in: build_library/ami_constants.sh
 akiid=${AKI[$region]}
 
 if [ -z "$akiid" ]; then
-   echo "$0: Can't identify AKI, using region: $region";
+   echo "$0: Can't identify AKI, using region: $region" >&2
    exit 1
 fi
 
+# TODO: Once we sort out a long-term scheme for generating AMIs this likely
+# will go away. What if we want to generate AMIs from CoreOS?!?!?
 if [ -z "$(which ec2-attach-volume)" ]; then
 	# Update and install Ubuntu packages
 	export DEBIAN_FRONTEND=noninteractive
@@ -69,12 +119,14 @@ if [ -z "$(which ec2-attach-volume)" ]; then
 	  ec2-ami-tools 
 fi
 
-export EC2_URL=http://ec2.$region.amazonaws.com
+export EC2_URL="http://ec2.${region}.amazonaws.com"
+echo "Building AMI in zone $zone, region id $akiid"
 
 # Create and mount temporary EBS volume with file system to hold new AMI image
 volumeid=$(ec2-create-volume --size $size --availability-zone $zone |
   cut -f2)
-instanceid=$(wget -qO- http://instance-data/latest/meta-data/instance-id)
+instanceid=$(curl -s http://instance-data/latest/meta-data/instance-id)
+echo "Attaching new volume $volumeid locally (instance $instanceid)"
 ec2-attach-volume --device /dev/sdi --instance "$instanceid" "$volumeid"
 while [ ! -e /dev/sdi -a ! -e /dev/xvdi ]
   do sleep 3; done
@@ -84,13 +136,21 @@ else
    dev=/dev/sdi 
 fi
 
+echo "Attached volume $volumeid as $dev"
+echo "Writing image from $IMG_URL to $dev"
+
 # if it is on the local fs, just use it, otherwise try to download it
-if [ -e "$binurl" ]; then
+if [[ -n "$IMG_PATH" ]]; then
-	bunzip2 -c $binurl | dd of=$dev bs=128M
+    if [[ "$IMG_PATH" =~ \.bz2$ ]]; then
+        bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M
+    else
+        dd if="$IMG_PATH" of=$dev bs=1M
+    fi
 else
-	curl -s $binurl | bunzip2 | dd of=$dev bs=128M
+    curl "$IMG_URL" | bunzip2 | dd of=$dev bs=1M
 fi
 
+echo "Detaching $volumeid and creating snapshot"
 ec2-detach-volume "$volumeid"
 while ec2-describe-volumes "$volumeid" | grep -q ATTACHMENT
   do sleep 3; done
@@ -98,7 +158,7 @@ snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2)
 while ec2-describe-snapshots "$snapshotid" | grep -q pending
   do sleep 30; done
 
-# Register the snapshot as a new AMI
+echo "Created snapshot $snapshotid, registering as a new AMI"
 amiid=$(ec2-register                                  \
   --name "$name"                                      \
   --description "$description"                        \
@@ -111,9 +171,9 @@ amiid=$(ec2-register                                  \
 ec2-delete-volume "$volumeid"
 
 cat <<EOF
-AMI: $amiid $codename $region $arch2
+AMI: $amiid $region $arch2
 
-CoreOS $version
+CoreOS $VERSION
 architecture: $arch ($arch2)
 region:       $region ($zone)
 aki id:       $akiid
@@ -122,11 +182,11 @@ description:  $description
 EBS volume:   $volumeid (deleted)
 EBS snapshot: $snapshotid
 AMI id:       $amiid
-bin url:      $binurl
+bin url:      $IMG_URL
 
 Test the new AMI using something like:
 
-  export EC2_URL=http://ec2.$region.amazonaws.com
+  export EC2_URL=http://ec2.${region}.amazonaws.com
   ec2-run-instances \\
     --key \$USER \\
     --instance-type t1.micro \\

oem/ami/zip_and_ship_ami.sh

@@ -1,10 +0,0 @@
-#!/bin/bash -xe
-#
-# this needs more refactoring, but is used to build the latest image
-
-IMG="../build/images/amd64-generic/latest/coreos_ami_image.bin"
-ssh -i ~/.ssh/coreos-images.pem ubuntu@23.22.1.1 "mkdir -p /mnt/tmp"
-
-bzip2 $IMG
-scp -i ~/.ssh/coreos-images.pem build_ebs_on_ec2.sh $IMG.bz2 ubuntu@23.22.1.1:/mnt/tmp
-ssh -i ~/.ssh/coreos-images.pem ubuntu@23.22.1.1 "sudo /mnt/tmp/build_ebs_on_ec2.sh /mnt/tmp/coreos_ami_image.bin.bz2"