feat(build_ebs_on_ec2): Fetch prod images by version, misc cleanup.

The build host will start generating production ami disk images so to
simplify the next step this script can automatically fetch them from
that location by version. The default sticks with the existing 'master'
versioning scheme. Added logging and turned off -x by default to make
the output log more readable.

Removing the zip_and_ship script since it isn't useful with officially
built disk images and only works with locally built images and a very
particular ec2 host. A different long term automation scheme will have
to be found.

oem/ami/build_ebs_on_ec2.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash
 #
 # This expects to run on an EC2 instance.
 #
@@ -9,6 +9,9 @@
 #  http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html
 # we need pv-grub-hd00 x86_64
 
+# Set pipefail along with -e in hopes that we catch more errors
+set -e -o pipefail
+
 declare -A AKI
 AKI["us-east-1"]=aki-b4aa75dd 
 AKI["us-west-1"]=aki-eb7e26ae
@@ -21,43 +24,90 @@ AKI["sa-east-1"]=aki-c88f51d5
 # AKI["gov-west-1"]=aki-75a4c056
 
 readonly COREOS_EPOCH=1372636800
-TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
+VERSION="master"
-EXTRA_VERSION=$(date +%H-%M)
+IMAGE="coreos_production_ami_image.bin.bz2"
+URL_FMT="http://storage.core-os.net/coreos/amd64-generic/%s/$IMAGE"
+IMG_URL=""
+IMG_PATH=""
 
-if [ -z "$1" ]; then
+USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2]
-	echo "usage: $0 [http|path/to/bin.bz2]"
+Options:
-	exit 1
+    -V VERSION  Set the version of this AMI, default is 'master'
+    -p PATH     Path to compressed disk image, overrides -u
+    -u URL      URL to compressed disk image, derived from -V if unset.
+    -K KEY      Path to Amazon API private key.
+    -C CERT     Path to Amazon API key certificate.
+    -h          this ;-)
+    -v          Verbose, see all the things!
+
+This script must be run from an ec2 host with the ec2 tools installed.
+"
+
+while getopts "V:p:u:K:C:hv" OPTION
+do
+    case $OPTION in
+        V) VERSION="$OPTARG";;
+        p) IMG_PATH="$OPTARG";;
+        u) IMG_URL="$OPTARG";;
+        K) export EC2_PRIVATE_KEY="$OPTARG";;
+        C) export EC2_CERT="$OPTARG";;
+        h) echo "$USAGE"; exit;;
+        v) set -x;;
+        *) exit 1;;
+    esac
+done
+
+if [[ $(id -u) -ne 0 ]]; then
+    echo "$0: This command must be run as root!" >&2
+    exit 1
+fi
+
+# Quick sanity check that the image exists
+if [[ -n "$IMG_PATH" ]]; then
+    if [[ ! -f "$IMG_PATH" ]]; then
+        echo "$0: Image path does not exist: $IMG_PATH" >&2
+        exit 1
+    fi
+    IMG_URL=$(basename "$IMG_PATH")
+else
+    if [[ -z "$IMG_URL" ]]; then
+        IMG_URL=$(printf "$URL_FMT" "$VERSION")
+    fi
+    if ! curl -s --head "$IMG_URL" >/dev/null; then
+        echo "$0: Image URL unavailable: $IMG_URL" >&2
+        exit 1
+    fi
+fi
+
+if [[ "$VERSION" == "master" ]]; then
+    # Come up with something more descriptive and timestamped
+    TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
+    VERSION="master-${TODAYS_VERSION}-$(date +%H-%M)"
 fi
 
-binurl=$1
 
 # Size of AMI file system
+# TODO: Perhaps define size and arch in a metadata file image_to_vm creates?
 size=8 # GB
-
 arch=x86_64
 arch2=amd64
 ephemeraldev=/dev/sdb
-version="master-$TODAYS_VERSION-$EXTRA_VERSION"
+# The name has a limited set of allowed characterrs
-
+name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VERSION")
-#TBD
+description="CoreOS $VERSION"
-name="CoreOS-$version"
-description="CoreOS master"
-
-export EC2_CERT=$(echo /tmp/*cert*.pem)
-export EC2_PRIVATE_KEY=$(echo /tmp/*pk*.pem)
 
 zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
 zone=$(curl -s $zoneurl)
 region=$(echo $zone | sed 's/.$//')
-
-# this is defined in: build_library/ami_constants.sh
 akiid=${AKI[$region]}
 
 if [ -z "$akiid" ]; then
-   echo "$0: Can't identify AKI, using region: $region";
+   echo "$0: Can't identify AKI, using region: $region" >&2
    exit 1
 fi
 
+# TODO: Once we sort out a long-term scheme for generating AMIs this likely
+# will go away. What if we want to generate AMIs from CoreOS?!?!?
 if [ -z "$(which ec2-attach-volume)" ]; then
 	# Update and install Ubuntu packages
 	export DEBIAN_FRONTEND=noninteractive
@@ -69,12 +119,14 @@ if [ -z "$(which ec2-attach-volume)" ]; then
 	  ec2-ami-tools 
 fi
 
-export EC2_URL=http://ec2.$region.amazonaws.com
+export EC2_URL="http://ec2.${region}.amazonaws.com"
+echo "Building AMI in zone $zone, region id $akiid"
 
 # Create and mount temporary EBS volume with file system to hold new AMI image
 volumeid=$(ec2-create-volume --size $size --availability-zone $zone |
   cut -f2)
-instanceid=$(wget -qO- http://instance-data/latest/meta-data/instance-id)
+instanceid=$(curl -s http://instance-data/latest/meta-data/instance-id)
+echo "Attaching new volume $volumeid locally (instance $instanceid)"
 ec2-attach-volume --device /dev/sdi --instance "$instanceid" "$volumeid"
 while [ ! -e /dev/sdi -a ! -e /dev/xvdi ]
   do sleep 3; done
@@ -84,13 +136,21 @@ else
    dev=/dev/sdi 
 fi
 
+echo "Attached volume $volumeid as $dev"
+echo "Writing image from $IMG_URL to $dev"
+
 # if it is on the local fs, just use it, otherwise try to download it
-if [ -e "$binurl" ]; then
+if [[ -n "$IMG_PATH" ]]; then
-	bunzip2 -c $binurl | dd of=$dev bs=128M
+    if [[ "$IMG_PATH" =~ \.bz2$ ]]; then
+        bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M
+    else
+        dd if="$IMG_PATH" of=$dev bs=1M
+    fi
 else
-	curl -s $binurl | bunzip2 | dd of=$dev bs=128M
+    curl "$IMG_URL" | bunzip2 | dd of=$dev bs=1M
 fi
 
+echo "Detaching $volumeid and creating snapshot"
 ec2-detach-volume "$volumeid"
 while ec2-describe-volumes "$volumeid" | grep -q ATTACHMENT
   do sleep 3; done
@@ -98,7 +158,7 @@ snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2)
 while ec2-describe-snapshots "$snapshotid" | grep -q pending
   do sleep 30; done
 
-# Register the snapshot as a new AMI
+echo "Created snapshot $snapshotid, registering as a new AMI"
 amiid=$(ec2-register                                  \
   --name "$name"                                      \
   --description "$description"                        \
@@ -111,9 +171,9 @@ amiid=$(ec2-register                                  \
 ec2-delete-volume "$volumeid"
 
 cat <<EOF
-AMI: $amiid $codename $region $arch2
+AMI: $amiid $region $arch2
 
-CoreOS $version
+CoreOS $VERSION
 architecture: $arch ($arch2)
 region:       $region ($zone)
 aki id:       $akiid
@@ -122,11 +182,11 @@ description:  $description
 EBS volume:   $volumeid (deleted)
 EBS snapshot: $snapshotid
 AMI id:       $amiid
-bin url:      $binurl
+bin url:      $IMG_URL
 
 Test the new AMI using something like:
 
-  export EC2_URL=http://ec2.$region.amazonaws.com
+  export EC2_URL=http://ec2.${region}.amazonaws.com
   ec2-run-instances \\
     --key \$USER \\
     --instance-type t1.micro \\

oem/ami/zip_and_ship_ami.sh

@@ -1,10 +0,0 @@
-#!/bin/bash -xe
-#
-# this needs more refactoring, but is used to build the latest image
-
-IMG="../build/images/amd64-generic/latest/coreos_ami_image.bin"
-ssh -i ~/.ssh/coreos-images.pem ubuntu@23.22.1.1 "mkdir -p /mnt/tmp"
-
-bzip2 $IMG
-scp -i ~/.ssh/coreos-images.pem build_ebs_on_ec2.sh $IMG.bz2 ubuntu@23.22.1.1:/mnt/tmp
-ssh -i ~/.ssh/coreos-images.pem ubuntu@23.22.1.1 "sudo /mnt/tmp/build_ebs_on_ec2.sh /mnt/tmp/coreos_ami_image.bin.bz2"