Merge pull request #121 from marineam/copy-updates

add(copy_ami): Move ami copying step to new script for easier testing

oem/ami/build_ebs_on_ec2.sh

@@ -176,23 +176,6 @@ ec2-modify-image-attribute "$amiid" --launch-permission -a all
 
 ec2-delete-volume "$volumeid"
 
-# hack job to copy AMIs
-export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0
-export JAVA_HOME=/usr
-for r in "${!AKI[@]}"
-do
-    [ "${r}" == "${region}" ] && continue
-    r_amiid=$($EC2_HOME/bin/ec2-copy-image \
-        --source-region "$region"            \
-        --source-ami-id "$amiid"             \
-        --name "$description"                \
-        --region "$r"                        |
-        cut -f2)
-    ec2-modify-image-attribute --region "$r" "$amiid" --launch-permission -a all
-    echo "$r $r_amiid"
-done
-
-
 cat <<EOF
 AMI: $amiid $region $arch2
 

oem/ami/copy_ami.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+#
+# This expects to run on an EC2 instance.
+#
+# mad props to Eric Hammond for the initial script
+#  https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami
+
+# AKI ids from:
+#  http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html
+# we need pv-grub-hd00 x86_64
+
+# Set pipefail along with -e in hopes that we catch more errors
+set -e -o pipefail
+
+declare -A AKI
+AKI["us-east-1"]=aki-b4aa75dd 
+AKI["us-west-1"]=aki-eb7e26ae
+AKI["us-west-2"]=aki-f837bac8 
+AKI["eu-west-1"]=aki-8b655dff
+AKI["ap-southeast-1"]=aki-fa1354a8
+AKI["ap-southeast-2"]=aki-3d990e07
+AKI["ap-northeast-1"]=aki-40992841
+AKI["sa-east-1"]=aki-c88f51d5
+# AKI["gov-west-1"]=aki-75a4c056
+
+USAGE="Usage: $0 -a ami-id
+    -a ami-id   ID of the AMI to be coppied.
+    -V VERSION  Find AMI by CoreOS version.
+    -K KEY      Path to Amazon API private key.
+    -C CERT     Path to Amazon API key certificate.
+    -h          this ;-)
+    -v          Verbose, see all the things!
+
+This script must be run from an ec2 host with the ec2 tools installed.
+"
+
+AMI=
+VER=
+
+while getopts "a:V:K:C:hv" OPTION
+do
+    case $OPTION in
+        a) AMI="$OPTARG";;
+        V) VER="$OPTARG";;
+        K) export EC2_PRIVATE_KEY="$OPTARG";;
+        C) export EC2_CERT="$OPTARG";;
+        h) echo "$USAGE"; exit;;
+        v) set -x;;
+        *) exit 1;;
+    esac
+done
+
+if [[ $(id -u) -eq 0 ]]; then
+    echo "$0: This command should not be ran run as root!" >&2
+    exit 1
+fi
+
+if [[ -z "$VER" ]]; then
+    echo "$0: Providing the verison via -V is required." >&2
+    exit 1
+fi
+
+if [[ -z "$AMI" ]]; then
+    AMI=$(ec2-describe-images -F name="CoreOS-$VER" | grep -m1 ^IMAGE \
+        | cut -f2) || true # Don't die silently, error messages are good
+    if [[ -z "$AMI" ]]; then
+        echo "$0: Cannot find an AMI for CoreOS $VER" >&2
+        exit 1
+    fi
+else
+    # check to make sure this is a valid image
+    if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then
+        echo "$0: Unknown image: $AMI" >&2
+        exit 1
+    fi
+fi
+
+# The name has a limited set of allowed characterrs
+name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VER")
+description="CoreOS $VER"
+
+zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
+zone=$(curl --fail -s $zoneurl)
+region=$(echo $zone | sed 's/.$//')
+
+# hack job to copy AMIs
+export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0
+export JAVA_HOME=/usr
+
+do_copy() {
+    local r="$1"
+    r_amiid=$($EC2_HOME/bin/ec2-copy-image \
+        --source-region "$region"            \
+        --source-ami-id "$AMI"               \
+        --name "$name"                       \
+        --description "$description"         \
+        --region "$r"                        |
+        cut -f2)
+    echo "AMI copy to $r as $r_amiid in progress"
+
+    while ec2-describe-images "$r_amiid" --region="$r" | grep -q pending; do
+        sleep 30
+    done
+    echo "AMI copy to $r as $r_amiid in complete"
+
+    # Not sure if this is needed, permissions seem to be copied
+    echo "Making $r_amiid in $r public"
+    ec2-modify-image-attribute --region "$r" "$r_amiid" --launch-permission -a all
+
+    # TODO: Add the awsmarket permissions to the snapshot backing the AMI which
+    # certainly isn't copied. Need to parse ec2-describe-images or something.
+}
+
+for r in "${!AKI[@]}"
+do
+    [ "${r}" == "${region}" ] && continue
+    echo "Starting copy of $AMI from $region to $r"
+    do_copy "$r" &
+    sleep 15
+done
+
+wait
+
+echo "Done"

oem/ami/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+VER="$1"
+DIR=/home/ubuntu/official
+
+if [ -z "$VER" ]; then
+  echo "Usage: $0 1.2.3" >&2
+  exit 1
+fi
+
+set -e
+sudo $DIR/build_ebs_on_ec2.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
+$DIR/test_ami2.sh -v -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
+$DIR/copy_ami.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
+$DIR/upload_ami_txt.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem

oem/ami/test_ami2.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+#
+# This expects to run on an EC2 instance.
+#
+# mad props to Eric Hammond for the initial script
+#  https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami
+
+# This script will launch three ec2 nodes with shared user-data, and then 
+# then test of the cluster is bootstrapped
+
+# Set pipefail along with -e in hopes that we catch more errors
+set -e -o pipefail
+
+USAGE="Usage: $0 -a ami-id
+    -a ami-id   ID of the AMI to be tests (required)
+    -K KEY      Path to Amazon API private key.
+    -C CERT     Path to Amazon API key certificate.
+    -h          this ;-)
+    -v          Verbose, see all the things!
+
+This script must be run from an ec2 host with the ec2 tools installed.
+"
+
+AMI=
+VER=
+
+while getopts "a:V:K:C:hv" OPTION
+do
+    case $OPTION in
+        a) AMI="$OPTARG";;
+        V) VER="$OPTARG";;
+        K) export EC2_PRIVATE_KEY="$OPTARG";;
+        C) export EC2_CERT="$OPTARG";;
+        h) echo "$USAGE"; exit;;
+        v) set -x;;
+        *) exit 1;;
+    esac
+done
+
+if [[ $(id -u) -eq 0 ]]; then
+    echo "$0: This command should not be ran run as root!" >&2
+    exit 1
+fi
+
+if [[ -z "$AMI" && -n "$VER" ]]; then
+    AMI=$(ec2-describe-images -F name="CoreOS-$VER" | grep -m1 ^IMAGE \
+        | cut -f2) || true # Don't die silently, error messages are good
+    if [[ -z "$AMI" ]]; then
+        echo "$0: Cannot find an AMI for CoreOS $VER" >&2
+        exit 1
+    fi
+elif [[ -n "$AMI" ]]; then
+    # check to make sure this is a valid image
+    if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then
+        echo "$0: Unknown image: $AMI" >&2
+        exit 1
+    fi
+else
+    echo "$0: AMI id or version required (-a or -V options)" >&2
+    echo "$USAGE" >&2
+    exit 1
+fi
+
+echo -n "Creating keys and security group... "
+key_name="autotest-`date +%s`"
+key_file="/tmp/$key_name"
+ec2-create-keypair $key_name | grep -v KEYPAIR > $key_file
+chmod 600 $key_file
+
+sg_name=$key_name
+sg=$(ec2-create-group $sg_name --description "$sg_name" | cut -f2)
+ec2-authorize "$sg_name" -P tcp -p 4001 > /dev/null
+ec2-authorize "$sg_name" -P tcp -p 7001 > /dev/null
+ec2-authorize "$sg_name" -P tcp -p 22 > /dev/null
+echo "OK ($key_name)"
+
+# might be needed later for multi-zone tests
+zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
+zone=$(curl --fail -s $zoneurl)
+region=$(echo $zone | sed 's/.$//')
+
+token=$(uuidgen)
+
+echo -n "Booting instances... "
+instances=$(ec2-run-instances \
+    --user-data "$token" \
+    --instance-type "t1.micro" \
+    --instance-count 3 \
+    --group "$sg_name" \
+    --key "$key_name" $AMI | \
+       grep INSTANCE | cut -f2)
+# little hack to create a describe instances command that only 
+# pulls data for these instances
+ec2_cmd=$(echo $instances | sed 's/ / --filter instance-id=/g')
+ec2_cmd="ec2-describe-instances --filter instance-id=$ec2_cmd"
+
+while $ec2_cmd | grep INSTANCE | grep -q pending
+  do sleep 10; done
+
+declare -a ips=($($ec2_cmd | grep INSTANCE | cut -f4))
+
+# sleep until all the sockets we need come up
+for host in ${ips[@]}; do
+    for port in 22 4001 7001; do
+        timeout 90 perl -MIO::Socket::INET -e "
+            until(new IO::Socket::INET('$host:$port')){sleep 1}"
+    done
+done
+echo "OK ($instances)"
+
+echo -n "Testing etcd... "
+test_key="v1/keys/test"
+# XXX: the sleep *should never* be required, this is a bug in etcd
+sleep 1
+curl --fail -s -L "${ips[0]}:4001/$test_key" -d value="$token" > /dev/null
+sleep 1
+for host in ${ips[@]}; do
+    if ! curl --fail -s -L "${host}:4001/$test_key" | grep -q $token; then
+        echo "etcd bootstrap appears to have failed for $host" >&2
+        exit 1
+    fi
+done
+echo "OK"
+
+echo -n "Cleaning up environment... "
+ec2-terminate-instances $instances > /dev/null
+while ! $ec2_cmd | grep INSTANCE | grep -q terminated
+  do sleep 10; done
+
+ec2-delete-group $sg_name > /dev/null
+ec2-delete-keypair $key_name > /dev/null
+rm $key_file
+echo "OK"

oem/ami/upload_ami_txt.sh

@@ -16,7 +16,7 @@ AKI["ap-northeast-1"]=aki-40992841
 AKI["sa-east-1"]=aki-c88f51d5
 # AKI["gov-west-1"]=aki-75a4c056
 
-USAGE="Usage: $0 -a ami-id
+USAGE="Usage: $0 -V 100.0.0
     -V VERSION  Find AMI by CoreOS version. (required)
     -K KEY      Path to Amazon API private key.
     -C CERT     Path to Amazon API key certificate.