Merge pull request #310 from marineam/ami

AMI build updates

oem/ami/build_ebs_on_ec2.sh

@@ -61,8 +61,8 @@ do
     esac
 done
 
-if [[ $(id -u) -ne 0 ]]; then
+if [[ $(id -u) -eq 0 ]]; then
-    echo "$0: This command must be run as root!" >&2
+    echo "$0: This command should not be ran run as root!" >&2
     exit 1
 fi
 
@@ -139,14 +139,14 @@ echo "Writing image from $IMG_URL to $dev"
 # if it is on the local fs, just use it, otherwise try to download it
 if [[ -n "$IMG_PATH" ]]; then
     if [[ "$IMG_PATH" =~ \.bz2$ ]]; then
-        bunzip2 -c "$IMG_PATH" > $dev
+        bunzip2 -c "$IMG_PATH" | sudo dd of=$dev bs=1M
     else
-        dd if="$IMG_PATH" of=$dev bs=1M
+        sudo dd if="$IMG_PATH" of=$dev bs=1M
     fi
 elif [[ "$IMG_URL" == gs://* ]]; then
-    gsutil cat "$IMG_URL" | bunzip2 > $dev
+    gsutil cat "$IMG_URL" | bunzip2 | sudo dd of=$dev bs=1M
 else
-    curl --fail "$IMG_URL" | bunzip2 > $dev
+    curl --fail "$IMG_URL" | bunzip2 | sudo dd of=$dev bs=1M
 fi
 
 echo "Detaching $volumeid and creating snapshot"
@@ -160,9 +160,6 @@ while ec2-describe-snapshots "$snapshotid" | grep -q pending
 echo "Created snapshot $snapshotid, deleting $volumeid"
 ec2-delete-volume "$volumeid"
 
-echo "Sharing snapshot with Amazon"
-ec2-modify-snapshot-attribute "$snapshotid" -c --add 679593333241
-
 echo "Registering hvm AMI"
 hvm_amiid=$(ec2-register                              \
   --name "${name}-hvm"                                \
@@ -174,9 +171,6 @@ hvm_amiid=$(ec2-register                              \
   --block-device-mapping /dev/xvdb=ephemeral0         |
   cut -f2)
 
-echo "Making $hvm_amiid public"
-ec2-modify-image-attribute "$hvm_amiid" --launch-permission -a all
-
 echo "Registering paravirtual AMI"
 amiid=$(ec2-register                                  \
   --name "$name"                                      \
@@ -189,9 +183,6 @@ amiid=$(ec2-register                                  \
   --block-device-mapping /dev/xvdb=ephemeral0         |
   cut -f2)
 
-echo "Making $amiid public"
-ec2-modify-image-attribute "$amiid" --launch-permission -a all
-
 cat <<EOF
 $description
 architecture: $arch ($arch2)

oem/ami/copy_ami.sh

@@ -48,6 +48,10 @@ add_region() {
     REGIONS+=( "$1" )
 }
 
+clean_version() {
+    sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1"
+}
+
 while getopts "a:V:b:g:r:hv" OPTION
 do
     case $OPTION in
@@ -78,16 +82,17 @@ region=$(echo $zone | sed 's/.$//')
 export EC2_URL="http://ec2.${region}.amazonaws.com"
 
 if [[ -z "$AMI" ]]; then
-    AMI=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER" | grep -m1 ^IMAGE \
+    search_name=$(clean_version "CoreOS-$GROUP-$VER")
+    AMI=$(ec2-describe-images -F name="${search_name}" | grep -m1 ^IMAGE \
         | cut -f2) || true # Don't die silently, error messages are good
     if [[ -z "$AMI" ]]; then
-        echo "$0: Cannot find an AMI for CoreOS $GROUP $VER" >&2
+        echo "$0: Cannot find an AMI named $search_name" >&2
         exit 1
     fi
-    HVM=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER-hvm" \
+    HVM=$(ec2-describe-images -F name="${search_name}-hvm" \
         | grep -m1 ^IMAGE | cut -f2) || true
     if [[ -z "$HVM" ]]; then
-        echo "$0: Cannot find an AMI for CoreOS $GROUP $VER (HVM)" >&2
+        echo "$0: Cannot find an AMI named ${search_name}-hvm" >&2
         exit 1
     fi
 else
@@ -103,60 +108,61 @@ if [[ ${#REGIONS[@]} -eq 0 ]]; then
 fi
 
 # The name has a limited set of allowed characterrs
-name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$GROUP-$VER")
+name=$(clean_version "CoreOS-$GROUP-$VER")
 description="CoreOS $GROUP $VER"
 
 do_copy() {
     local r="$1"
     local virt_type="$2"
-    local r_amiid
+    local local_amiid="$3"
-    if [[ "$virt_type" == "hvm" ]]; then
+    local r_amiid r_name r_desc
-        r_amiid=$(ec2-copy-image                 \
-            --source-region "$region"            \
-            --source-ami-id "$HVM"               \
-            --name "${name}-hvm"                 \
-            --description "$description (HVM)"   \
-            --region "$r"                        |
-            cut -f2)
-    else
-        r_amiid=$(ec2-copy-image                 \
-            --source-region "$region"            \
-            --source-ami-id "$AMI"               \
-            --name "$name"                       \
-            --description "$description (PV)"    \
-            --region "$r"                        |
-        cut -f2)
-    fi
-    echo "AMI copy to $r as $r_amiid in progress"
 
-    local r_amidesc=$(ec2-describe-images "$r_amiid" --region="$r")
+    # run in a subshell, the -e flag doesn't get inherited
-    while grep -q pending <<<"$r_amidesc"; do
+    set -e
+
+    echo "Starting copy of $virt_type $local_amiid from $region to $r"
+    if [[ "$virt_type" == "hvm" ]]; then
+        r_name="${name}-hvm"
+        r_desc="${description} (HVM)"
+    else
+        r_name="${name}"
+        r_desc="${description} (PV)"
+    fi
+    r_amiid=$(ec2-copy-image \
+        --source-region "$region" --source-ami-id "$local_amiid" \
+        --name "$r_name" --description "$r_desc" --region "$r" |
+        cut -f2)
+    echo "AMI $virt_type copy to $r as $r_amiid in progress"
+
+    while ec2-describe-images "$r_amiid" --region="$r" | grep -q pending; do
         sleep 30
-        r_amidesc=$(ec2-describe-images "$r_amiid" --region="$r")
     done
     echo "AMI $virt_type copy to $r as $r_amiid in complete"
-
-    local r_snapshotid=$(echo "$r_amidesc" | \
-        grep '^BLOCKDEVICEMAPPING.*/dev/xvda' | cut -f5)
-    echo "Sharing snapshot $r_snapshotid in $r with Amazon"
-    ec2-modify-snapshot-attribute "$r_snapshotid" \
-        -c --add 679593333241 --region "$r"
-
-    echo "Making $r_amiid in $r public"
-    ec2-modify-image-attribute --region "$r" "$r_amiid" --launch-permission -a all
 }
 
+WAIT_PIDS=()
 for r in "${REGIONS[@]}"
 do
     [ "${r}" == "${region}" ] && continue
-    echo "Starting copy of pv $AMI from $region to $r"
+    do_copy "$r" pv "$AMI" &
-    do_copy "$r" pv &
+    WAIT_PIDS+=( $! )
     if [[ -n "$HVM" ]]; then
-        echo "Starting copy of hvm $AMI from $region to $r"
+        do_copy "$r" hvm "$HVM" &
-        do_copy "$r" hvm &
+        WAIT_PIDS+=( $! )
     fi
 done
 
-wait
+# wait for each subshell individually to report errors
+WAIT_FAILED=0
+for wait_pid in "${WAIT_PIDS[@]}"; do
+    if ! wait ${wait_pid}; then
+        : $(( WAIT_FAILED++ ))
+    fi
+done
+
+if [[ ${WAIT_FAILED} -ne 0 ]]; then
+    echo "${WAIT_FAILED} jobs failed :(" >&2
+    exit ${WAIT_FAILED}
+fi
 
 echo "Done"

oem/ami/master.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+DIR=/home/ec2-user/scripts
+URL="https://commondatastorage.googleapis.com/storage.core-os.net/coreos/amd64-usr/master"
+
+set -e
+eval $(curl -f "${URL}/version.txt")
+
+source $DIR/marineam-auth.sh
+args="-b amd64-usr -g master -V ${COREOS_VERSION}"
+$DIR/build_ebs_on_ec2.sh ${args} -u "${URL}/coreos_production_ami_image.bin.bz2"
+$DIR/test_ami.sh -v ${args}
+#$DIR/copy_ami.sh ${args}

oem/ami/prod.sh

@@ -11,9 +11,8 @@ if [ -z "$GROUP" -o -z "$VER" ]; then
 fi
 
 set -e
-args="-b $BOARD -g $GROUP -V $VER"
-sudo bash -c ". $DIR/marineam-auth.sh && $DIR/build_ebs_on_ec2.sh ${args}"
 source $DIR/marineam-auth.sh
+args="-b $BOARD -g $GROUP -V $VER"
+$DIR/build_ebs_on_ec2.sh ${args}
 $DIR/test_ami.sh -v ${args}
 $DIR/copy_ami.sh ${args}
-$DIR/upload_ami_txt.sh ${args}

oem/ami/publish_ami.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+#
+# Set pipefail along with -e in hopes that we catch more errors
+set -e -o pipefail
+
+REGIONS=(
+    us-east-1
+    us-west-1
+    us-west-2
+    eu-west-1
+    ap-southeast-1
+    ap-southeast-2
+    ap-northeast-1
+    sa-east-1
+)
+
+USAGE="Usage: $0 -V 100.0.0
+    -V VERSION  Find AMI by CoreOS version. (required)
+    -b BOARD    Set to the board name, default is amd64-usr
+    -g GROUP    Set the update group, default is alpha
+    -s STORAGE  GS URL for Google storage to upload to.
+    -h          this ;-)
+    -v          Verbose, see all the things!
+
+This script must be run from an ec2 host with the ec2 tools installed.
+"
+
+IMAGE="coreos_production_ami"
+GS_URL="gs://builds.release.core-os.net"
+AMI=
+VER=
+BOARD="amd64-usr"
+GROUP="alpha"
+
+clean_version() {
+    sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1"
+}
+
+while getopts "V:b:g:s:hv" OPTION
+do
+    case $OPTION in
+        V) VER="$OPTARG";;
+        b) BOARD="$OPTARG";;
+        g) GROUP="$OPTARG";;
+        s) GS_URL="$OPTARG";;
+        h) echo "$USAGE"; exit;;
+        v) set -x;;
+        *) exit 1;;
+    esac
+done
+
+if [[ $(id -u) -eq 0 ]]; then
+    echo "$0: This command should not be ran run as root!" >&2
+    exit 1
+fi
+
+if [[ ! -n "$VER" ]]; then
+    echo "$0: AMI version required via -V" >&2
+    echo "$USAGE" >&2
+    exit 1
+fi
+
+search_name=$(clean_version "CoreOS-$GROUP-$VER")
+declare -A AMIS HVM_AMIS
+for r in "${REGIONS[@]}"; do
+    AMI=$(ec2-describe-images --region=${r} -F name="${search_name}" \
+        | grep -m1 ^IMAGE | cut -f2) || true
+    if [[ -z "$AMI" ]]; then
+        echo "$0: Cannot find an AMI named ${search_name} in ${r}" >&2
+        exit 1
+    fi
+    AMIS[${r}]=$AMI
+    HVM=$(ec2-describe-images --region=${r} -F name="${search_name}-hvm" \
+        | grep -m1 ^IMAGE | cut -f2) || true
+    if [[ -z "$HVM" ]]; then
+        echo "$0: Cannot find an AMI named ${search_name}-hvm in ${r}" >&2
+        exit 1
+    fi
+    HVM_AMIS[${r}]=$HVM
+done
+
+# ignore this crap: /usr/lib64/python2.6/site-packages/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec.  You should rebuild using libgmp >= 5 to avoid timing attack vulnerability.
+upload_file() {
+    local name="$1"
+    local content="$2"
+    url="$GS_URL/$GROUP/boards/$BOARD/$VER/${IMAGE}_${name}.txt"
+    python -W "ignore:Not using mpz_powm_sec" \
+        `which gsutil` cp - "$url" <<<"$content"
+    echo "OK, ${url}=${content}"
+}
+
+publish_ami() {
+    local r="$1"
+    local virt_type="$2"
+    local r_amiid="$3"
+    local r_snapshotid=$(ec2-describe-images --region="$r" "$r_amiid" \
+        | grep '^BLOCKDEVICEMAPPING.*/dev/xvda' | cut -f5) || true
+
+    # run in a subshell, the -e flag doesn't get inherited
+    set -e
+
+    if [[ -z "${r_snapshotid}" ]]; then
+        echo "$0: Cannot find snapshot id for $r_amiid in $r" >&2
+        return 1
+    fi
+
+    echo "Sharing snapshot $r_snapshotid in $r with Amazon"
+    ec2-modify-snapshot-attribute --region "$r" \
+        "$r_snapshotid" -c --add 679593333241
+
+    echo "Making $r_amiid in $r public"
+    ec2-modify-image-attribute --region "$r" \
+        "$r_amiid" --launch-permission -a all
+
+    # compatibility name from before addition of hvm
+    if [[ "${virt_type}" == "pv" ]]; then
+        upload_file "$r" "$r_amiid"
+    fi
+
+    upload_file "${virt_type}_${r}" "$r_amiid"
+}
+
+WAIT_PIDS=()
+PV_ALL=""
+for r in "${!AMIS[@]}"; do
+    publish_ami "$r" pv "${AMIS[$r]}" &
+    WAIT_PIDS+=( $! )
+    PV_ALL+="|${r}=${AMIS[$r]}"
+done
+PV_ALL="${PV_ALL#|}"
+
+HVM_ALL=""
+for r in "${!HVM_AMIS[@]}"; do
+    publish_ami "$r" hvm "${HVM_AMIS[$r]}" &
+    WAIT_PIDS+=( $! )
+    HVM_ALL+="|${r}=${HVM_AMIS[$r]}"
+done
+HVM_ALL="${HVM_ALL#|}"
+
+# wait for each subshell individually to report errors
+WAIT_FAILED=0
+for wait_pid in "${WAIT_PIDS[@]}"; do
+    if ! wait ${wait_pid}; then
+        : $(( WAIT_FAILED++ ))
+    fi
+done
+
+if [[ ${WAIT_FAILED} -ne 0 ]]; then
+    echo "${WAIT_FAILED} jobs failed, aborting :(" >&2
+    exit ${WAIT_FAILED}
+fi
+
+upload_file "all" "${PV_ALL}"
+upload_file "pv" "${PV_ALL}"
+upload_file "hvm" "${HVM_ALL}"
+echo "Done"

oem/ami/test_ami.sh

@@ -29,6 +29,10 @@ VER=
 BOARD="amd64-usr"
 GROUP="alpha"
 
+clean_version() {
+    sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1"
+}
+
 while getopts "a:V:b:g:hv" OPTION
 do
     case $OPTION in
@@ -53,16 +57,17 @@ region=$(echo $zone | sed 's/.$//')
 export EC2_URL="http://ec2.${region}.amazonaws.com"
 
 if [[ -z "$AMI" && -n "$VER" ]]; then
-    AMI=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER" | grep -m1 ^IMAGE \
+    search_name=$(clean_version "CoreOS-$GROUP-$VER")
+    AMI=$(ec2-describe-images -F name="${search_name}" | grep -m1 ^IMAGE \
         | cut -f2) || true # Don't die silently, error messages are good
     if [[ -z "$AMI" ]]; then
-        echo "$0: Cannot find an AMI for CoreOS $GROUP $VER" >&2
+        echo "$0: Cannot find an AMI named $search_name" >&2
         exit 1
     fi
-    HVM=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER-hvm" \
+    HVM=$(ec2-describe-images -F name="${search_name}-hvm" \
         | grep -m1 ^IMAGE | cut -f2) || true
     if [[ -z "$HVM" ]]; then
-        echo "$0: Cannot find an AMI for CoreOS $GROUP $VER (HVM)" >&2
+        echo "$0: Cannot find an AMI named ${search_name}-hvm" >&2
         exit 1
     fi
 elif [[ -n "$AMI" ]]; then

oem/ami/upload_ami_txt.sh

@@ -1,106 +0,0 @@
-#!/bin/bash
-#
-# Set pipefail along with -e in hopes that we catch more errors
-set -e -o pipefail
-
-# we just use this for the list of regions
-# should a copy/paste from build_ebs_on_ec2.sh
-declare -A AKI
-AKI["us-east-1"]=aki-b4aa75dd
-AKI["us-west-1"]=aki-eb7e26ae
-AKI["us-west-2"]=aki-f837bac8
-AKI["eu-west-1"]=aki-8b655dff
-AKI["ap-southeast-1"]=aki-fa1354a8
-AKI["ap-southeast-2"]=aki-3d990e07
-AKI["ap-northeast-1"]=aki-40992841
-AKI["sa-east-1"]=aki-c88f51d5
-# AKI["gov-west-1"]=aki-75a4c056
-
-USAGE="Usage: $0 -V 100.0.0
-    -V VERSION  Find AMI by CoreOS version. (required)
-    -b BOARD    Set to the board name, default is amd64-usr
-    -g GROUP    Set the update group, default is alpha
-    -s STORAGE  GS URL for Google storage to upload to.
-    -h          this ;-)
-    -v          Verbose, see all the things!
-
-This script must be run from an ec2 host with the ec2 tools installed.
-"
-
-IMAGE="coreos_production_ami"
-GS_URL="gs://builds.release.core-os.net"
-AMI=
-VER=
-BOARD="amd64-usr"
-GROUP="alpha"
-
-while getopts "V:b:g:s:hv" OPTION
-do
-    case $OPTION in
-        V) VER="$OPTARG";;
-        b) BOARD="$OPTARG";;
-        g) GROUP="$OPTARG";;
-        s) GS_URL="$OPTARG";;
-        h) echo "$USAGE"; exit;;
-        v) set -x;;
-        *) exit 1;;
-    esac
-done
-
-if [[ $(id -u) -eq 0 ]]; then
-    echo "$0: This command should not be ran run as root!" >&2
-    exit 1
-fi
-
-if [[ ! -n "$VER" ]]; then
-    echo "$0: AMI version required via -V" >&2
-    echo "$USAGE" >&2
-    exit 1
-fi
-
-declare -A AMIS HVM_AMIS
-for r in "${!AKI[@]}"; do
-    AMI=$(ec2-describe-images --region=${r} -F name="CoreOS-$GROUP-$VER" \
-        | grep -m1 ^IMAGE \
-        | cut -f2) || true # Don't die silently, error messages are good
-    if [[ -z "$AMI" ]]; then
-        echo "$0: Cannot find ${r} AMI for CoreOS $GROUP $VER" >&2
-        continue
-    fi
-    AMIS[${r}]=$AMI
-    HVM=$(ec2-describe-images --region=${r} -F name="CoreOS-$GROUP-$VER-hvm" \
-        | grep -m1 ^IMAGE | cut -f2) || true
-    if [[ -z "$HVM" ]]; then
-        echo "$0: Cannot find ${r} AMI for CoreOS $GROUP $VER (HVM)" >&2
-        exit 1
-    fi
-    HVM_AMIS[${r}]=$HVM
-done
-
-upload_file() {
-    local name="$1"
-    local content="$2"
-    url="$GS_URL/$GROUP/boards/$BOARD/$VER/${IMAGE}_${name}.txt"
-    gsutil cp - "$url" <<<"$content"
-    echo "OK, ${url}=${content}"
-}
-
-PV_ALL=""
-for r in "${!AMIS[@]}"; do
-    upload_file "$r" "${AMIS[$r]}"
-    upload_file "pv_$r" "${AMIS[$r]}"
-    PV_ALL+="|${r}=${AMIS[$r]}"
-done
-PV_ALL="${PV_ALL#|}"
-
-HVM_ALL=""
-for r in "${!HVM_AMIS[@]}"; do
-    upload_file "hvm_$r" "${HVM_AMIS[$r]}"
-    HVM_ALL+="|${r}=${HVM_AMIS[$r]}"
-done
-HVM_ALL="${HVM_ALL#|}"
-
-upload_file "all" "${PV_ALL}"
-upload_file "pv" "${PV_ALL}"
-upload_file "hvm" "${HVM_ALL}"
-echo "Done"

oem/ami/user.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+DIR=/home/ec2-user/scripts
+USER=someone
+TYPE=production
+VERSION="367.0.0+2014-07-10-1613"
+URL="http://users.developer.core-os.net/${USER}/boards/amd64-usr/${VERSION}"
+
+set -e
+eval $(curl -f "${URL}/version.txt")
+
+source $DIR/marineam-auth.sh
+args="-b amd64-usr -g ${USER} -V ${VERSION}"
+$DIR/build_ebs_on_ec2.sh ${args} -u "${URL}/coreos_${TYPE}_ami_image.bin.bz2"
+$DIR/test_ami.sh -v ${args}
+#$DIR/copy_ami.sh ${args}