diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh index 360d5eae9d..9fbd707f53 100755 --- a/oem/ami/build_ebs_on_ec2.sh +++ b/oem/ami/build_ebs_on_ec2.sh @@ -61,8 +61,8 @@ do esac done -if [[ $(id -u) -ne 0 ]]; then - echo "$0: This command must be run as root!" >&2 +if [[ $(id -u) -eq 0 ]]; then + echo "$0: This command should not be ran run as root!" >&2 exit 1 fi @@ -139,14 +139,14 @@ echo "Writing image from $IMG_URL to $dev" # if it is on the local fs, just use it, otherwise try to download it if [[ -n "$IMG_PATH" ]]; then if [[ "$IMG_PATH" =~ \.bz2$ ]]; then - bunzip2 -c "$IMG_PATH" > $dev + bunzip2 -c "$IMG_PATH" | sudo dd of=$dev bs=1M else - dd if="$IMG_PATH" of=$dev bs=1M + sudo dd if="$IMG_PATH" of=$dev bs=1M fi elif [[ "$IMG_URL" == gs://* ]]; then - gsutil cat "$IMG_URL" | bunzip2 > $dev + gsutil cat "$IMG_URL" | bunzip2 | sudo dd of=$dev bs=1M else - curl --fail "$IMG_URL" | bunzip2 > $dev + curl --fail "$IMG_URL" | bunzip2 | sudo dd of=$dev bs=1M fi echo "Detaching $volumeid and creating snapshot" @@ -160,9 +160,6 @@ while ec2-describe-snapshots "$snapshotid" | grep -q pending echo "Created snapshot $snapshotid, deleting $volumeid" ec2-delete-volume "$volumeid" -echo "Sharing snapshot with Amazon" -ec2-modify-snapshot-attribute "$snapshotid" -c --add 679593333241 - echo "Registering hvm AMI" hvm_amiid=$(ec2-register \ --name "${name}-hvm" \ @@ -174,9 +171,6 @@ hvm_amiid=$(ec2-register \ --block-device-mapping /dev/xvdb=ephemeral0 | cut -f2) -echo "Making $hvm_amiid public" -ec2-modify-image-attribute "$hvm_amiid" --launch-permission -a all - echo "Registering paravirtual AMI" amiid=$(ec2-register \ --name "$name" \ @@ -189,9 +183,6 @@ amiid=$(ec2-register \ --block-device-mapping /dev/xvdb=ephemeral0 | cut -f2) -echo "Making $amiid public" -ec2-modify-image-attribute "$amiid" --launch-permission -a all - cat <&2 + echo "$0: Cannot find an AMI named $search_name" >&2 exit 1 fi - HVM=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER-hvm" \ + HVM=$(ec2-describe-images -F name="${search_name}-hvm" \ | grep -m1 ^IMAGE | cut -f2) || true if [[ -z "$HVM" ]]; then - echo "$0: Cannot find an AMI for CoreOS $GROUP $VER (HVM)" >&2 + echo "$0: Cannot find an AMI named ${search_name}-hvm" >&2 exit 1 fi else @@ -103,60 +108,61 @@ if [[ ${#REGIONS[@]} -eq 0 ]]; then fi # The name has a limited set of allowed characterrs -name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$GROUP-$VER") +name=$(clean_version "CoreOS-$GROUP-$VER") description="CoreOS $GROUP $VER" do_copy() { local r="$1" local virt_type="$2" - local r_amiid - if [[ "$virt_type" == "hvm" ]]; then - r_amiid=$(ec2-copy-image \ - --source-region "$region" \ - --source-ami-id "$HVM" \ - --name "${name}-hvm" \ - --description "$description (HVM)" \ - --region "$r" | - cut -f2) - else - r_amiid=$(ec2-copy-image \ - --source-region "$region" \ - --source-ami-id "$AMI" \ - --name "$name" \ - --description "$description (PV)" \ - --region "$r" | - cut -f2) - fi - echo "AMI copy to $r as $r_amiid in progress" + local local_amiid="$3" + local r_amiid r_name r_desc - local r_amidesc=$(ec2-describe-images "$r_amiid" --region="$r") - while grep -q pending <<<"$r_amidesc"; do + # run in a subshell, the -e flag doesn't get inherited + set -e + + echo "Starting copy of $virt_type $local_amiid from $region to $r" + if [[ "$virt_type" == "hvm" ]]; then + r_name="${name}-hvm" + r_desc="${description} (HVM)" + else + r_name="${name}" + r_desc="${description} (PV)" + fi + r_amiid=$(ec2-copy-image \ + --source-region "$region" --source-ami-id "$local_amiid" \ + --name "$r_name" --description "$r_desc" --region "$r" | + cut -f2) + echo "AMI $virt_type copy to $r as $r_amiid in progress" + + while ec2-describe-images "$r_amiid" --region="$r" | grep -q pending; do sleep 30 - r_amidesc=$(ec2-describe-images "$r_amiid" --region="$r") done echo "AMI $virt_type copy to $r as $r_amiid in complete" - - local r_snapshotid=$(echo "$r_amidesc" | \ - grep '^BLOCKDEVICEMAPPING.*/dev/xvda' | cut -f5) - echo "Sharing snapshot $r_snapshotid in $r with Amazon" - ec2-modify-snapshot-attribute "$r_snapshotid" \ - -c --add 679593333241 --region "$r" - - echo "Making $r_amiid in $r public" - ec2-modify-image-attribute --region "$r" "$r_amiid" --launch-permission -a all } +WAIT_PIDS=() for r in "${REGIONS[@]}" do [ "${r}" == "${region}" ] && continue - echo "Starting copy of pv $AMI from $region to $r" - do_copy "$r" pv & + do_copy "$r" pv "$AMI" & + WAIT_PIDS+=( $! ) if [[ -n "$HVM" ]]; then - echo "Starting copy of hvm $AMI from $region to $r" - do_copy "$r" hvm & + do_copy "$r" hvm "$HVM" & + WAIT_PIDS+=( $! ) fi done -wait +# wait for each subshell individually to report errors +WAIT_FAILED=0 +for wait_pid in "${WAIT_PIDS[@]}"; do + if ! wait ${wait_pid}; then + : $(( WAIT_FAILED++ )) + fi +done + +if [[ ${WAIT_FAILED} -ne 0 ]]; then + echo "${WAIT_FAILED} jobs failed :(" >&2 + exit ${WAIT_FAILED} +fi echo "Done" diff --git a/oem/ami/master.sh b/oem/ami/master.sh new file mode 100755 index 0000000000..327dcfe559 --- /dev/null +++ b/oem/ami/master.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +DIR=/home/ec2-user/scripts +URL="https://commondatastorage.googleapis.com/storage.core-os.net/coreos/amd64-usr/master" + +set -e +eval $(curl -f "${URL}/version.txt") + +source $DIR/marineam-auth.sh +args="-b amd64-usr -g master -V ${COREOS_VERSION}" +$DIR/build_ebs_on_ec2.sh ${args} -u "${URL}/coreos_production_ami_image.bin.bz2" +$DIR/test_ami.sh -v ${args} +#$DIR/copy_ami.sh ${args} diff --git a/oem/ami/run.sh b/oem/ami/prod.sh similarity index 73% rename from oem/ami/run.sh rename to oem/ami/prod.sh index fe53216be2..8d4c690404 100755 --- a/oem/ami/run.sh +++ b/oem/ami/prod.sh @@ -11,9 +11,8 @@ if [ -z "$GROUP" -o -z "$VER" ]; then fi set -e -args="-b $BOARD -g $GROUP -V $VER" -sudo bash -c ". $DIR/marineam-auth.sh && $DIR/build_ebs_on_ec2.sh ${args}" source $DIR/marineam-auth.sh +args="-b $BOARD -g $GROUP -V $VER" +$DIR/build_ebs_on_ec2.sh ${args} $DIR/test_ami.sh -v ${args} $DIR/copy_ami.sh ${args} -$DIR/upload_ami_txt.sh ${args} diff --git a/oem/ami/publish_ami.sh b/oem/ami/publish_ami.sh new file mode 100755 index 0000000000..e3cfbc89c8 --- /dev/null +++ b/oem/ami/publish_ami.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Set pipefail along with -e in hopes that we catch more errors +set -e -o pipefail + +REGIONS=( + us-east-1 + us-west-1 + us-west-2 + eu-west-1 + ap-southeast-1 + ap-southeast-2 + ap-northeast-1 + sa-east-1 +) + +USAGE="Usage: $0 -V 100.0.0 + -V VERSION Find AMI by CoreOS version. (required) + -b BOARD Set to the board name, default is amd64-usr + -g GROUP Set the update group, default is alpha + -s STORAGE GS URL for Google storage to upload to. + -h this ;-) + -v Verbose, see all the things! + +This script must be run from an ec2 host with the ec2 tools installed. +" + +IMAGE="coreos_production_ami" +GS_URL="gs://builds.release.core-os.net" +AMI= +VER= +BOARD="amd64-usr" +GROUP="alpha" + +clean_version() { + sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1" +} + +while getopts "V:b:g:s:hv" OPTION +do + case $OPTION in + V) VER="$OPTARG";; + b) BOARD="$OPTARG";; + g) GROUP="$OPTARG";; + s) GS_URL="$OPTARG";; + h) echo "$USAGE"; exit;; + v) set -x;; + *) exit 1;; + esac +done + +if [[ $(id -u) -eq 0 ]]; then + echo "$0: This command should not be ran run as root!" >&2 + exit 1 +fi + +if [[ ! -n "$VER" ]]; then + echo "$0: AMI version required via -V" >&2 + echo "$USAGE" >&2 + exit 1 +fi + +search_name=$(clean_version "CoreOS-$GROUP-$VER") +declare -A AMIS HVM_AMIS +for r in "${REGIONS[@]}"; do + AMI=$(ec2-describe-images --region=${r} -F name="${search_name}" \ + | grep -m1 ^IMAGE | cut -f2) || true + if [[ -z "$AMI" ]]; then + echo "$0: Cannot find an AMI named ${search_name} in ${r}" >&2 + exit 1 + fi + AMIS[${r}]=$AMI + HVM=$(ec2-describe-images --region=${r} -F name="${search_name}-hvm" \ + | grep -m1 ^IMAGE | cut -f2) || true + if [[ -z "$HVM" ]]; then + echo "$0: Cannot find an AMI named ${search_name}-hvm in ${r}" >&2 + exit 1 + fi + HVM_AMIS[${r}]=$HVM +done + +# ignore this crap: /usr/lib64/python2.6/site-packages/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability. +upload_file() { + local name="$1" + local content="$2" + url="$GS_URL/$GROUP/boards/$BOARD/$VER/${IMAGE}_${name}.txt" + python -W "ignore:Not using mpz_powm_sec" \ + `which gsutil` cp - "$url" <<<"$content" + echo "OK, ${url}=${content}" +} + +publish_ami() { + local r="$1" + local virt_type="$2" + local r_amiid="$3" + local r_snapshotid=$(ec2-describe-images --region="$r" "$r_amiid" \ + | grep '^BLOCKDEVICEMAPPING.*/dev/xvda' | cut -f5) || true + + # run in a subshell, the -e flag doesn't get inherited + set -e + + if [[ -z "${r_snapshotid}" ]]; then + echo "$0: Cannot find snapshot id for $r_amiid in $r" >&2 + return 1 + fi + + echo "Sharing snapshot $r_snapshotid in $r with Amazon" + ec2-modify-snapshot-attribute --region "$r" \ + "$r_snapshotid" -c --add 679593333241 + + echo "Making $r_amiid in $r public" + ec2-modify-image-attribute --region "$r" \ + "$r_amiid" --launch-permission -a all + + # compatibility name from before addition of hvm + if [[ "${virt_type}" == "pv" ]]; then + upload_file "$r" "$r_amiid" + fi + + upload_file "${virt_type}_${r}" "$r_amiid" +} + +WAIT_PIDS=() +PV_ALL="" +for r in "${!AMIS[@]}"; do + publish_ami "$r" pv "${AMIS[$r]}" & + WAIT_PIDS+=( $! ) + PV_ALL+="|${r}=${AMIS[$r]}" +done +PV_ALL="${PV_ALL#|}" + +HVM_ALL="" +for r in "${!HVM_AMIS[@]}"; do + publish_ami "$r" hvm "${HVM_AMIS[$r]}" & + WAIT_PIDS+=( $! ) + HVM_ALL+="|${r}=${HVM_AMIS[$r]}" +done +HVM_ALL="${HVM_ALL#|}" + +# wait for each subshell individually to report errors +WAIT_FAILED=0 +for wait_pid in "${WAIT_PIDS[@]}"; do + if ! wait ${wait_pid}; then + : $(( WAIT_FAILED++ )) + fi +done + +if [[ ${WAIT_FAILED} -ne 0 ]]; then + echo "${WAIT_FAILED} jobs failed, aborting :(" >&2 + exit ${WAIT_FAILED} +fi + +upload_file "all" "${PV_ALL}" +upload_file "pv" "${PV_ALL}" +upload_file "hvm" "${HVM_ALL}" +echo "Done" diff --git a/oem/ami/test_ami.sh b/oem/ami/test_ami.sh index 7570f22539..65b99fcdc4 100755 --- a/oem/ami/test_ami.sh +++ b/oem/ami/test_ami.sh @@ -29,6 +29,10 @@ VER= BOARD="amd64-usr" GROUP="alpha" +clean_version() { + sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1" +} + while getopts "a:V:b:g:hv" OPTION do case $OPTION in @@ -53,16 +57,17 @@ region=$(echo $zone | sed 's/.$//') export EC2_URL="http://ec2.${region}.amazonaws.com" if [[ -z "$AMI" && -n "$VER" ]]; then - AMI=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER" | grep -m1 ^IMAGE \ + search_name=$(clean_version "CoreOS-$GROUP-$VER") + AMI=$(ec2-describe-images -F name="${search_name}" | grep -m1 ^IMAGE \ | cut -f2) || true # Don't die silently, error messages are good if [[ -z "$AMI" ]]; then - echo "$0: Cannot find an AMI for CoreOS $GROUP $VER" >&2 + echo "$0: Cannot find an AMI named $search_name" >&2 exit 1 fi - HVM=$(ec2-describe-images -F name="CoreOS-$GROUP-$VER-hvm" \ + HVM=$(ec2-describe-images -F name="${search_name}-hvm" \ | grep -m1 ^IMAGE | cut -f2) || true if [[ -z "$HVM" ]]; then - echo "$0: Cannot find an AMI for CoreOS $GROUP $VER (HVM)" >&2 + echo "$0: Cannot find an AMI named ${search_name}-hvm" >&2 exit 1 fi elif [[ -n "$AMI" ]]; then diff --git a/oem/ami/upload_ami_txt.sh b/oem/ami/upload_ami_txt.sh deleted file mode 100755 index 7bea511699..0000000000 --- a/oem/ami/upload_ami_txt.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -# -# Set pipefail along with -e in hopes that we catch more errors -set -e -o pipefail - -# we just use this for the list of regions -# should a copy/paste from build_ebs_on_ec2.sh -declare -A AKI -AKI["us-east-1"]=aki-b4aa75dd -AKI["us-west-1"]=aki-eb7e26ae -AKI["us-west-2"]=aki-f837bac8 -AKI["eu-west-1"]=aki-8b655dff -AKI["ap-southeast-1"]=aki-fa1354a8 -AKI["ap-southeast-2"]=aki-3d990e07 -AKI["ap-northeast-1"]=aki-40992841 -AKI["sa-east-1"]=aki-c88f51d5 -# AKI["gov-west-1"]=aki-75a4c056 - -USAGE="Usage: $0 -V 100.0.0 - -V VERSION Find AMI by CoreOS version. (required) - -b BOARD Set to the board name, default is amd64-usr - -g GROUP Set the update group, default is alpha - -s STORAGE GS URL for Google storage to upload to. - -h this ;-) - -v Verbose, see all the things! - -This script must be run from an ec2 host with the ec2 tools installed. -" - -IMAGE="coreos_production_ami" -GS_URL="gs://builds.release.core-os.net" -AMI= -VER= -BOARD="amd64-usr" -GROUP="alpha" - -while getopts "V:b:g:s:hv" OPTION -do - case $OPTION in - V) VER="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - s) GS_URL="$OPTARG";; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -if [[ ! -n "$VER" ]]; then - echo "$0: AMI version required via -V" >&2 - echo "$USAGE" >&2 - exit 1 -fi - -declare -A AMIS HVM_AMIS -for r in "${!AKI[@]}"; do - AMI=$(ec2-describe-images --region=${r} -F name="CoreOS-$GROUP-$VER" \ - | grep -m1 ^IMAGE \ - | cut -f2) || true # Don't die silently, error messages are good - if [[ -z "$AMI" ]]; then - echo "$0: Cannot find ${r} AMI for CoreOS $GROUP $VER" >&2 - continue - fi - AMIS[${r}]=$AMI - HVM=$(ec2-describe-images --region=${r} -F name="CoreOS-$GROUP-$VER-hvm" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$HVM" ]]; then - echo "$0: Cannot find ${r} AMI for CoreOS $GROUP $VER (HVM)" >&2 - exit 1 - fi - HVM_AMIS[${r}]=$HVM -done - -upload_file() { - local name="$1" - local content="$2" - url="$GS_URL/$GROUP/boards/$BOARD/$VER/${IMAGE}_${name}.txt" - gsutil cp - "$url" <<<"$content" - echo "OK, ${url}=${content}" -} - -PV_ALL="" -for r in "${!AMIS[@]}"; do - upload_file "$r" "${AMIS[$r]}" - upload_file "pv_$r" "${AMIS[$r]}" - PV_ALL+="|${r}=${AMIS[$r]}" -done -PV_ALL="${PV_ALL#|}" - -HVM_ALL="" -for r in "${!HVM_AMIS[@]}"; do - upload_file "hvm_$r" "${HVM_AMIS[$r]}" - HVM_ALL+="|${r}=${HVM_AMIS[$r]}" -done -HVM_ALL="${HVM_ALL#|}" - -upload_file "all" "${PV_ALL}" -upload_file "pv" "${PV_ALL}" -upload_file "hvm" "${HVM_ALL}" -echo "Done" diff --git a/oem/ami/user.sh b/oem/ami/user.sh new file mode 100755 index 0000000000..0e70eea1b9 --- /dev/null +++ b/oem/ami/user.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +DIR=/home/ec2-user/scripts +USER=someone +TYPE=production +VERSION="367.0.0+2014-07-10-1613" +URL="http://users.developer.core-os.net/${USER}/boards/amd64-usr/${VERSION}" + +set -e +eval $(curl -f "${URL}/version.txt") + +source $DIR/marineam-auth.sh +args="-b amd64-usr -g ${USER} -V ${VERSION}" +$DIR/build_ebs_on_ec2.sh ${args} -u "${URL}/coreos_${TYPE}_ami_image.bin.bz2" +$DIR/test_ami.sh -v ${args} +#$DIR/copy_ami.sh ${args}