diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh index 2955ce65ab..e8af97b577 100755 --- a/oem/ami/build_ebs_on_ec2.sh +++ b/oem/ami/build_ebs_on_ec2.sh @@ -176,23 +176,6 @@ ec2-modify-image-attribute "$amiid" --launch-permission -a all ec2-delete-volume "$volumeid" -# hack job to copy AMIs -export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0 -export JAVA_HOME=/usr -for r in "${!AKI[@]}" -do - [ "${r}" == "${region}" ] && continue - r_amiid=$($EC2_HOME/bin/ec2-copy-image \ - --source-region "$region" \ - --source-ami-id "$amiid" \ - --name "$description" \ - --region "$r" | - cut -f2) - ec2-modify-image-attribute --region "$r" "$amiid" --launch-permission -a all - echo "$r $r_amiid" -done - - cat <&2 + exit 1 +fi + +if [[ -z "$VER" ]]; then + echo "$0: Providing the verison via -V is required." >&2 + exit 1 +fi + +if [[ -z "$AMI" ]]; then + AMI=$(ec2-describe-images -F name="CoreOS-$VER" | grep -m1 ^IMAGE \ + | cut -f2) || true # Don't die silently, error messages are good + if [[ -z "$AMI" ]]; then + echo "$0: Cannot find an AMI for CoreOS $VER" >&2 + exit 1 + fi +else + # check to make sure this is a valid image + if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then + echo "$0: Unknown image: $AMI" >&2 + exit 1 + fi +fi + +# The name has a limited set of allowed characterrs +name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VER") +description="CoreOS $VER" + +zoneurl=http://instance-data/latest/meta-data/placement/availability-zone +zone=$(curl --fail -s $zoneurl) +region=$(echo $zone | sed 's/.$//') + +# hack job to copy AMIs +export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0 +export JAVA_HOME=/usr + +do_copy() { + local r="$1" + r_amiid=$($EC2_HOME/bin/ec2-copy-image \ + --source-region "$region" \ + --source-ami-id "$AMI" \ + --name "$name" \ + --description "$description" \ + --region "$r" | + cut -f2) + echo "AMI copy to $r as $r_amiid in progress" + + while ec2-describe-images "$r_amiid" --region="$r" | grep -q pending; do + sleep 30 + done + echo "AMI copy to $r as $r_amiid in complete" + + # Not sure if this is needed, permissions seem to be copied + echo "Making $r_amiid in $r public" + ec2-modify-image-attribute --region "$r" "$r_amiid" --launch-permission -a all + + # TODO: Add the awsmarket permissions to the snapshot backing the AMI which + # certainly isn't copied. Need to parse ec2-describe-images or something. +} + +for r in "${!AKI[@]}" +do + [ "${r}" == "${region}" ] && continue + echo "Starting copy of $AMI from $region to $r" + do_copy "$r" & + sleep 15 +done + +wait + +echo "Done" diff --git a/oem/ami/run.sh b/oem/ami/run.sh new file mode 100755 index 0000000000..a294bb056b --- /dev/null +++ b/oem/ami/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +VER="$1" +DIR=/home/ubuntu/official + +if [ -z "$VER" ]; then + echo "Usage: $0 1.2.3" >&2 + exit 1 +fi + +set -e +sudo $DIR/build_ebs_on_ec2.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem +$DIR/test_ami2.sh -v -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem +$DIR/copy_ami.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem +$DIR/upload_ami_txt.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem diff --git a/oem/ami/test_ami2.sh b/oem/ami/test_ami2.sh new file mode 100755 index 0000000000..6b9b6b7c7d --- /dev/null +++ b/oem/ami/test_ami2.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# This expects to run on an EC2 instance. +# +# mad props to Eric Hammond for the initial script +# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami + +# This script will launch three ec2 nodes with shared user-data, and then +# then test of the cluster is bootstrapped + +# Set pipefail along with -e in hopes that we catch more errors +set -e -o pipefail + +USAGE="Usage: $0 -a ami-id + -a ami-id ID of the AMI to be tests (required) + -K KEY Path to Amazon API private key. + -C CERT Path to Amazon API key certificate. + -h this ;-) + -v Verbose, see all the things! + +This script must be run from an ec2 host with the ec2 tools installed. +" + +AMI= +VER= + +while getopts "a:V:K:C:hv" OPTION +do + case $OPTION in + a) AMI="$OPTARG";; + V) VER="$OPTARG";; + K) export EC2_PRIVATE_KEY="$OPTARG";; + C) export EC2_CERT="$OPTARG";; + h) echo "$USAGE"; exit;; + v) set -x;; + *) exit 1;; + esac +done + +if [[ $(id -u) -eq 0 ]]; then + echo "$0: This command should not be ran run as root!" >&2 + exit 1 +fi + +if [[ -z "$AMI" && -n "$VER" ]]; then + AMI=$(ec2-describe-images -F name="CoreOS-$VER" | grep -m1 ^IMAGE \ + | cut -f2) || true # Don't die silently, error messages are good + if [[ -z "$AMI" ]]; then + echo "$0: Cannot find an AMI for CoreOS $VER" >&2 + exit 1 + fi +elif [[ -n "$AMI" ]]; then + # check to make sure this is a valid image + if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then + echo "$0: Unknown image: $AMI" >&2 + exit 1 + fi +else + echo "$0: AMI id or version required (-a or -V options)" >&2 + echo "$USAGE" >&2 + exit 1 +fi + +echo -n "Creating keys and security group... " +key_name="autotest-`date +%s`" +key_file="/tmp/$key_name" +ec2-create-keypair $key_name | grep -v KEYPAIR > $key_file +chmod 600 $key_file + +sg_name=$key_name +sg=$(ec2-create-group $sg_name --description "$sg_name" | cut -f2) +ec2-authorize "$sg_name" -P tcp -p 4001 > /dev/null +ec2-authorize "$sg_name" -P tcp -p 7001 > /dev/null +ec2-authorize "$sg_name" -P tcp -p 22 > /dev/null +echo "OK ($key_name)" + +# might be needed later for multi-zone tests +zoneurl=http://instance-data/latest/meta-data/placement/availability-zone +zone=$(curl --fail -s $zoneurl) +region=$(echo $zone | sed 's/.$//') + +token=$(uuidgen) + +echo -n "Booting instances... " +instances=$(ec2-run-instances \ + --user-data "$token" \ + --instance-type "t1.micro" \ + --instance-count 3 \ + --group "$sg_name" \ + --key "$key_name" $AMI | \ + grep INSTANCE | cut -f2) +# little hack to create a describe instances command that only +# pulls data for these instances +ec2_cmd=$(echo $instances | sed 's/ / --filter instance-id=/g') +ec2_cmd="ec2-describe-instances --filter instance-id=$ec2_cmd" + +while $ec2_cmd | grep INSTANCE | grep -q pending + do sleep 10; done + +declare -a ips=($($ec2_cmd | grep INSTANCE | cut -f4)) + +# sleep until all the sockets we need come up +for host in ${ips[@]}; do + for port in 22 4001 7001; do + timeout 90 perl -MIO::Socket::INET -e " + until(new IO::Socket::INET('$host:$port')){sleep 1}" + done +done +echo "OK ($instances)" + +echo -n "Testing etcd... " +test_key="v1/keys/test" +# XXX: the sleep *should never* be required, this is a bug in etcd +sleep 1 +curl --fail -s -L "${ips[0]}:4001/$test_key" -d value="$token" > /dev/null +sleep 1 +for host in ${ips[@]}; do + if ! curl --fail -s -L "${host}:4001/$test_key" | grep -q $token; then + echo "etcd bootstrap appears to have failed for $host" >&2 + exit 1 + fi +done +echo "OK" + +echo -n "Cleaning up environment... " +ec2-terminate-instances $instances > /dev/null +while ! $ec2_cmd | grep INSTANCE | grep -q terminated + do sleep 10; done + +ec2-delete-group $sg_name > /dev/null +ec2-delete-keypair $key_name > /dev/null +rm $key_file +echo "OK" diff --git a/oem/ami/upload_ami_txt.sh b/oem/ami/upload_ami_txt.sh index 7b9ecd2cde..0ad1f15030 100755 --- a/oem/ami/upload_ami_txt.sh +++ b/oem/ami/upload_ami_txt.sh @@ -16,7 +16,7 @@ AKI["ap-northeast-1"]=aki-40992841 AKI["sa-east-1"]=aki-c88f51d5 # AKI["gov-west-1"]=aki-75a4c056 -USAGE="Usage: $0 -a ami-id +USAGE="Usage: $0 -V 100.0.0 -V VERSION Find AMI by CoreOS version. (required) -K KEY Path to Amazon API private key. -C CERT Path to Amazon API key certificate.