fix(oem/ami): Remove key options, update wrapper for beta

The key/cert authentication method doesn't work any more, just rely on
sourcing a file with the right env vars exported.

Re-enable parallel copy.

Add group option to wrapper and custom google storage url options.

oem/ami/build_ebs_on_ec2.sh

@@ -28,7 +28,7 @@ VERSION="master"
 BOARD="amd64-usr"
 GROUP="alpha"
 IMAGE="coreos_production_ami_image.bin.bz2"
-URL_FMT="http://storage.core-os.net/coreos/%s/%s/$IMAGE"
+GS_URL="gs://storage.core-os.net/coreos"
 IMG_URL=""
 IMG_PATH=""
 
@@ -39,15 +39,14 @@ Options:
     -g GROUP    Set the update group, default is alpha or master
     -p PATH     Path to compressed disk image, overrides -u
     -u URL      URL to compressed disk image, derived from -V if unset.
-    -K KEY      Path to Amazon API private key.
+    -s STORAGE  GS URL for Google storage (used to generate URL)
-    -C CERT     Path to Amazon API key certificate.
     -h          this ;-)
     -v          Verbose, see all the things!
 
 This script must be run from an ec2 host with the ec2 tools installed.
 "
 
-while getopts "V:b:g:p:u:K:C:hv" OPTION
+while getopts "V:b:g:p:u:s:hv" OPTION
 do
     case $OPTION in
         V) VERSION="$OPTARG";;
@@ -55,8 +54,7 @@ do
         g) GROUP="$OPTARG";;
         p) IMG_PATH="$OPTARG";;
         u) IMG_URL="$OPTARG";;
-        K) export EC2_PRIVATE_KEY="$OPTARG";;
+        s) GS_URL="$OPTARG";;
-        C) export EC2_CERT="$OPTARG";;
         h) echo "$USAGE"; exit;;
         v) set -x;;
         *) exit 1;;
@@ -77,7 +75,7 @@ if [[ -n "$IMG_PATH" ]]; then
     IMG_URL=$(basename "$IMG_PATH")
 else
     if [[ -z "$IMG_URL" ]]; then
-        IMG_URL=$(printf "$URL_FMT" "$BOARD" "$VERSION")
+        IMG_URL="http://${GS_URL#gs://}/$BOARD/$VERSION/$IMAGE"
     fi
     if ! curl --fail -s --head "$IMG_URL" >/dev/null; then
         echo "$0: Image URL unavailable: $IMG_URL" >&2

oem/ami/copy_ami.sh

@@ -28,8 +28,6 @@ USAGE="Usage: $0 -a ami-id
     -V VERSION  Find AMI by CoreOS version.
     -b BOARD    Set to the board name, default is amd64-usr
     -g GROUP    Set the update group, default is alpha
-    -K KEY      Path to Amazon API private key.
-    -C CERT     Path to Amazon API key certificate.
     -h          this ;-)
     -v          Verbose, see all the things!
 
@@ -41,15 +39,13 @@ VER=
 BOARD="amd64-usr"
 GROUP="alpha"
 
-while getopts "a:V:b:g:K:C:hv" OPTION
+while getopts "a:V:b:g:hv" OPTION
 do
     case $OPTION in
         a) AMI="$OPTARG";;
         V) VER="$OPTARG";;
         b) BOARD="$OPTARG";;
         g) GROUP="$OPTARG";;
-        K) export EC2_PRIVATE_KEY="$OPTARG";;
-        C) export EC2_CERT="$OPTARG";;
         h) echo "$USAGE"; exit;;
         v) set -x;;
         *) exit 1;;
@@ -89,13 +85,9 @@ zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
 zone=$(curl --fail -s $zoneurl)
 region=$(echo $zone | sed 's/.$//')
 
-# hack job to copy AMIs
-export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0
-export JAVA_HOME=/usr
-
 do_copy() {
     local r="$1"
-    local r_amiid=$($EC2_HOME/bin/ec2-copy-image \
+    local r_amiid=$(ec2-copy-image \
         --source-region "$region"            \
         --source-ami-id "$AMI"               \
         --name "$name"                       \
@@ -112,7 +104,7 @@ do_copy() {
     echo "AMI copy to $r as $r_amiid in complete"
 
     local r_snapshotid=$(echo "$r_amidesc" | \
-        grep '^BLOCKDEVICEMAPPING./dev/sda' | cut -f4)
+        grep '^BLOCKDEVICEMAPPING.*/dev/sda' | cut -f5)
     echo "Sharing snapshot $r_snapshotid in $r with Amazon"
     ec2-modify-snapshot-attribute "$r_snapshotid" \
         -c --add 679593333241 --region "$r"
@@ -125,7 +117,9 @@ for r in "${!AKI[@]}"
 do
     [ "${r}" == "${region}" ] && continue
     echo "Starting copy of $AMI from $region to $r"
-    do_copy "$r"
+    do_copy "$r" &
 done
 
+wait
+
 echo "Done"

oem/ami/run.sh

@@ -1,17 +1,19 @@
 #!/bin/bash
 
-BOARD="$1"
+BOARD="amd64-usr"
+GROUP="$1"
 VER="$2"
 DIR=/home/ubuntu/official
 
-if [ -z "$BOARD" -o -z "$VER" ]; then
+if [ -z "$GROUP" -o -z "$VER" ]; then
-  echo "Usage: $0 amd64-usr 1.2.3" >&2
+  echo "Usage: $0 alpha 1.2.3" >&2
   exit 1
 fi
 
 set -e
-args=( -b $BOARD -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem )
+args="-b $BOARD -g $GROUP -V $VER"
-sudo $DIR/build_ebs_on_ec2.sh "${args[@]}"
+sudo bash -c ". $DIR/marineam-auth.sh && $DIR/build_ebs_on_ec2.sh ${args}"
-$DIR/test_ami.sh -v "${args[@]}"
+source $DIR/marineam-auth.sh
-$DIR/copy_ami.sh "${args[@]}"
+$DIR/test_ami.sh -v ${args}
-$DIR/upload_ami_txt.sh "${args[@]}"
+$DIR/copy_ami.sh ${args}
+$DIR/upload_ami_txt.sh ${args}

oem/ami/test_ami.sh

@@ -16,8 +16,6 @@ USAGE="Usage: $0 -a ami-id
     -V VERSION  Find AMI by CoreOS version.
     -b BOARD    Set to the board name, default is amd64-usr
     -g GROUP    Set the update group, default is alpha
-    -K KEY      Path to Amazon API private key.
-    -C CERT     Path to Amazon API key certificate.
     -h          this ;-)
     -v          Verbose, see all the things!
 
@@ -30,15 +28,13 @@ VER=
 BOARD="amd64-usr"
 GROUP="alpha"
 
-while getopts "a:V:b:g:K:C:hv" OPTION
+while getopts "a:V:b:g:hv" OPTION
 do
     case $OPTION in
         a) AMI="$OPTARG";;
         V) VER="$OPTARG";;
         b) BOARD="$OPTARG";;
         g) GROUP="$OPTARG";;
-        K) export EC2_PRIVATE_KEY="$OPTARG";;
-        C) export EC2_CERT="$OPTARG";;
         h) echo "$USAGE"; exit;;
         v) set -x;;
         *) exit 1;;

oem/ami/upload_ami_txt.sh

@@ -20,8 +20,7 @@ USAGE="Usage: $0 -V 100.0.0
     -V VERSION  Find AMI by CoreOS version. (required)
     -b BOARD    Set to the board name, default is amd64-usr
     -g GROUP    Set the update group, default is alpha
-    -K KEY      Path to Amazon API private key.
+    -s STORAGE  GS URL for Google storage to upload to.
-    -C CERT     Path to Amazon API key certificate.
     -h          this ;-)
     -v          Verbose, see all the things!
 
@@ -29,20 +28,19 @@ This script must be run from an ec2 host with the ec2 tools installed.
 "
 
 IMAGE="coreos_production_ami"
-URL_FMT="gs://storage.core-os.net/coreos/%s/%s/${IMAGE}_%s.txt"
+GS_URL="gs://storage.core-os.net/coreos"
 AMI=
 VER=
 BOARD="amd64-usr"
 GROUP="alpha"
 
-while getopts "V:b:g:K:C:hv" OPTION
+while getopts "V:b:g:s:hv" OPTION
 do
     case $OPTION in
         V) VER="$OPTARG";;
         b) BOARD="$OPTARG";;
         g) GROUP="$OPTARG";;
-        K) export EC2_PRIVATE_KEY="$OPTARG";;
+        s) GS_URL="$OPTARG";;
-        C) export EC2_CERT="$OPTARG";;
         h) echo "$USAGE"; exit;;
         v) set -x;;
         *) exit 1;;
@@ -74,7 +72,7 @@ done
 
 OUT=
 for r in "${!AMIS[@]}"; do
-    url=$(printf "$URL_FMT" "$BOARD" "$VER" "$r")
+    url="$GS_URL/$BOARD/$VER/${IMAGE}_${r}.txt"
     tmp=$(mktemp --suffix=.txt)
     trap "rm -f '$tmp'" EXIT
     echo "${AMIS[$r]}" > "$tmp"
@@ -86,7 +84,7 @@ for r in "${!AMIS[@]}"; do
         OUT="${OUT}|${r}=${AMIS[$r]}"
     fi
 done
-url=$(printf "$URL_FMT" "$BOARD" "$VER" "all")
+url="$GS_URL/$BOARD/$VER/${IMAGE}_all.txt"
 tmp=$(mktemp --suffix=.txt)
 trap "rm -f '$tmp'" EXIT
 echo "$OUT" > "$tmp"