From f2bafd79323c484dac3e344736a65f14ed53ac78 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Thu, 8 May 2014 15:52:38 -0700 Subject: [PATCH] fix(oem/ami): Remove key options, update wrapper for beta The key/cert authentication method doesn't work any more, just rely on sourcing a file with the right env vars exported. Re-enable parallel copy. Add group option to wrapper and custom google storage url options. --- oem/ami/build_ebs_on_ec2.sh | 12 +++++------- oem/ami/copy_ami.sh | 18 ++++++------------ oem/ami/run.sh | 18 ++++++++++-------- oem/ami/test_ami.sh | 6 +----- oem/ami/upload_ami_txt.sh | 14 ++++++-------- 5 files changed, 28 insertions(+), 40 deletions(-) diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh index 2d74c1b4ab..f919e97d19 100755 --- a/oem/ami/build_ebs_on_ec2.sh +++ b/oem/ami/build_ebs_on_ec2.sh @@ -28,7 +28,7 @@ VERSION="master" BOARD="amd64-usr" GROUP="alpha" IMAGE="coreos_production_ami_image.bin.bz2" -URL_FMT="http://storage.core-os.net/coreos/%s/%s/$IMAGE" +GS_URL="gs://storage.core-os.net/coreos" IMG_URL="" IMG_PATH="" @@ -39,15 +39,14 @@ Options: -g GROUP Set the update group, default is alpha or master -p PATH Path to compressed disk image, overrides -u -u URL URL to compressed disk image, derived from -V if unset. - -K KEY Path to Amazon API private key. - -C CERT Path to Amazon API key certificate. + -s STORAGE GS URL for Google storage (used to generate URL) -h this ;-) -v Verbose, see all the things! This script must be run from an ec2 host with the ec2 tools installed. " -while getopts "V:b:g:p:u:K:C:hv" OPTION +while getopts "V:b:g:p:u:s:hv" OPTION do case $OPTION in V) VERSION="$OPTARG";; @@ -55,8 +54,7 @@ do g) GROUP="$OPTARG";; p) IMG_PATH="$OPTARG";; u) IMG_URL="$OPTARG";; - K) export EC2_PRIVATE_KEY="$OPTARG";; - C) export EC2_CERT="$OPTARG";; + s) GS_URL="$OPTARG";; h) echo "$USAGE"; exit;; v) set -x;; *) exit 1;; @@ -77,7 +75,7 @@ if [[ -n "$IMG_PATH" ]]; then IMG_URL=$(basename "$IMG_PATH") else if [[ -z "$IMG_URL" ]]; then - IMG_URL=$(printf "$URL_FMT" "$BOARD" "$VERSION") + IMG_URL="http://${GS_URL#gs://}/$BOARD/$VERSION/$IMAGE" fi if ! curl --fail -s --head "$IMG_URL" >/dev/null; then echo "$0: Image URL unavailable: $IMG_URL" >&2 diff --git a/oem/ami/copy_ami.sh b/oem/ami/copy_ami.sh index 3b3dbe1fbc..daed96a095 100755 --- a/oem/ami/copy_ami.sh +++ b/oem/ami/copy_ami.sh @@ -28,8 +28,6 @@ USAGE="Usage: $0 -a ami-id -V VERSION Find AMI by CoreOS version. -b BOARD Set to the board name, default is amd64-usr -g GROUP Set the update group, default is alpha - -K KEY Path to Amazon API private key. - -C CERT Path to Amazon API key certificate. -h this ;-) -v Verbose, see all the things! @@ -41,15 +39,13 @@ VER= BOARD="amd64-usr" GROUP="alpha" -while getopts "a:V:b:g:K:C:hv" OPTION +while getopts "a:V:b:g:hv" OPTION do case $OPTION in a) AMI="$OPTARG";; V) VER="$OPTARG";; b) BOARD="$OPTARG";; g) GROUP="$OPTARG";; - K) export EC2_PRIVATE_KEY="$OPTARG";; - C) export EC2_CERT="$OPTARG";; h) echo "$USAGE"; exit;; v) set -x;; *) exit 1;; @@ -89,13 +85,9 @@ zoneurl=http://instance-data/latest/meta-data/placement/availability-zone zone=$(curl --fail -s $zoneurl) region=$(echo $zone | sed 's/.$//') -# hack job to copy AMIs -export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0 -export JAVA_HOME=/usr - do_copy() { local r="$1" - local r_amiid=$($EC2_HOME/bin/ec2-copy-image \ + local r_amiid=$(ec2-copy-image \ --source-region "$region" \ --source-ami-id "$AMI" \ --name "$name" \ @@ -112,7 +104,7 @@ do_copy() { echo "AMI copy to $r as $r_amiid in complete" local r_snapshotid=$(echo "$r_amidesc" | \ - grep '^BLOCKDEVICEMAPPING./dev/sda' | cut -f4) + grep '^BLOCKDEVICEMAPPING.*/dev/sda' | cut -f5) echo "Sharing snapshot $r_snapshotid in $r with Amazon" ec2-modify-snapshot-attribute "$r_snapshotid" \ -c --add 679593333241 --region "$r" @@ -125,7 +117,9 @@ for r in "${!AKI[@]}" do [ "${r}" == "${region}" ] && continue echo "Starting copy of $AMI from $region to $r" - do_copy "$r" + do_copy "$r" & done +wait + echo "Done" diff --git a/oem/ami/run.sh b/oem/ami/run.sh index 24c153c8cd..a95012770b 100755 --- a/oem/ami/run.sh +++ b/oem/ami/run.sh @@ -1,17 +1,19 @@ #!/bin/bash -BOARD="$1" +BOARD="amd64-usr" +GROUP="$1" VER="$2" DIR=/home/ubuntu/official -if [ -z "$BOARD" -o -z "$VER" ]; then - echo "Usage: $0 amd64-usr 1.2.3" >&2 +if [ -z "$GROUP" -o -z "$VER" ]; then + echo "Usage: $0 alpha 1.2.3" >&2 exit 1 fi set -e -args=( -b $BOARD -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem ) -sudo $DIR/build_ebs_on_ec2.sh "${args[@]}" -$DIR/test_ami.sh -v "${args[@]}" -$DIR/copy_ami.sh "${args[@]}" -$DIR/upload_ami_txt.sh "${args[@]}" +args="-b $BOARD -g $GROUP -V $VER" +sudo bash -c ". $DIR/marineam-auth.sh && $DIR/build_ebs_on_ec2.sh ${args}" +source $DIR/marineam-auth.sh +$DIR/test_ami.sh -v ${args} +$DIR/copy_ami.sh ${args} +$DIR/upload_ami_txt.sh ${args} diff --git a/oem/ami/test_ami.sh b/oem/ami/test_ami.sh index 3f31e7468f..9f3e7f53f0 100755 --- a/oem/ami/test_ami.sh +++ b/oem/ami/test_ami.sh @@ -16,8 +16,6 @@ USAGE="Usage: $0 -a ami-id -V VERSION Find AMI by CoreOS version. -b BOARD Set to the board name, default is amd64-usr -g GROUP Set the update group, default is alpha - -K KEY Path to Amazon API private key. - -C CERT Path to Amazon API key certificate. -h this ;-) -v Verbose, see all the things! @@ -30,15 +28,13 @@ VER= BOARD="amd64-usr" GROUP="alpha" -while getopts "a:V:b:g:K:C:hv" OPTION +while getopts "a:V:b:g:hv" OPTION do case $OPTION in a) AMI="$OPTARG";; V) VER="$OPTARG";; b) BOARD="$OPTARG";; g) GROUP="$OPTARG";; - K) export EC2_PRIVATE_KEY="$OPTARG";; - C) export EC2_CERT="$OPTARG";; h) echo "$USAGE"; exit;; v) set -x;; *) exit 1;; diff --git a/oem/ami/upload_ami_txt.sh b/oem/ami/upload_ami_txt.sh index de25e0239d..259ca462cc 100755 --- a/oem/ami/upload_ami_txt.sh +++ b/oem/ami/upload_ami_txt.sh @@ -20,8 +20,7 @@ USAGE="Usage: $0 -V 100.0.0 -V VERSION Find AMI by CoreOS version. (required) -b BOARD Set to the board name, default is amd64-usr -g GROUP Set the update group, default is alpha - -K KEY Path to Amazon API private key. - -C CERT Path to Amazon API key certificate. + -s STORAGE GS URL for Google storage to upload to. -h this ;-) -v Verbose, see all the things! @@ -29,20 +28,19 @@ This script must be run from an ec2 host with the ec2 tools installed. " IMAGE="coreos_production_ami" -URL_FMT="gs://storage.core-os.net/coreos/%s/%s/${IMAGE}_%s.txt" +GS_URL="gs://storage.core-os.net/coreos" AMI= VER= BOARD="amd64-usr" GROUP="alpha" -while getopts "V:b:g:K:C:hv" OPTION +while getopts "V:b:g:s:hv" OPTION do case $OPTION in V) VER="$OPTARG";; b) BOARD="$OPTARG";; g) GROUP="$OPTARG";; - K) export EC2_PRIVATE_KEY="$OPTARG";; - C) export EC2_CERT="$OPTARG";; + s) GS_URL="$OPTARG";; h) echo "$USAGE"; exit;; v) set -x;; *) exit 1;; @@ -74,7 +72,7 @@ done OUT= for r in "${!AMIS[@]}"; do - url=$(printf "$URL_FMT" "$BOARD" "$VER" "$r") + url="$GS_URL/$BOARD/$VER/${IMAGE}_${r}.txt" tmp=$(mktemp --suffix=.txt) trap "rm -f '$tmp'" EXIT echo "${AMIS[$r]}" > "$tmp" @@ -86,7 +84,7 @@ for r in "${!AMIS[@]}"; do OUT="${OUT}|${r}=${AMIS[$r]}" fi done -url=$(printf "$URL_FMT" "$BOARD" "$VER" "all") +url="$GS_URL/$BOARD/$VER/${IMAGE}_all.txt" tmp=$(mktemp --suffix=.txt) trap "rm -f '$tmp'" EXIT echo "$OUT" > "$tmp"