mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-12 15:36:58 +02:00
add(copy_ami): Move ami copying step to new script for easier testing
Fix some copying issues: - Don't set AMI permissions until it is out of the pending state - Set name and description properly - Handle each region in parallel, mostly (these Java apps use lots of CPU for some reason so parallelism is limited, hence the sleeps). Less important but also included here: - Add run.sh and test_ami2.sh which are currently used in my release process. The alternate test script is used because the autotest stuff in the other script is broken right now.
This commit is contained in:
Michael Marineau
parent
af9c881f72
commit
aecedce889
@ -176,23 +176,6 @@ ec2-modify-image-attribute "$amiid" --launch-permission -a all
|
||||
|
||||
ec2-delete-volume "$volumeid"
|
||||
|
||||
# hack job to copy AMIs
|
||||
export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0
|
||||
export JAVA_HOME=/usr
|
||||
for r in "${!AKI[@]}"
|
||||
do
|
||||
[ "${r}" == "${region}" ] && continue
|
||||
r_amiid=$($EC2_HOME/bin/ec2-copy-image \
|
||||
--source-region "$region" \
|
||||
--source-ami-id "$amiid" \
|
||||
--name "$description" \
|
||||
--region "$r" |
|
||||
cut -f2)
|
||||
ec2-modify-image-attribute --region "$r" "$amiid" --launch-permission -a all
|
||||
echo "$r $r_amiid"
|
||||
done
|
||||
|
||||
|
||||
cat <<EOF
|
||||
AMI: $amiid $region $arch2
|
||||
|
||||
|
||||
124
oem/ami/copy_ami.sh
Executable file
124
oem/ami/copy_ami.sh
Executable file
@ -0,0 +1,124 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# This expects to run on an EC2 instance.
|
||||
#
|
||||
# mad props to Eric Hammond for the initial script
|
||||
# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami
|
||||
|
||||
# AKI ids from:
|
||||
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html
|
||||
# we need pv-grub-hd00 x86_64
|
||||
|
||||
# Set pipefail along with -e in hopes that we catch more errors
|
||||
set -e -o pipefail
|
||||
|
||||
declare -A AKI
|
||||
AKI["us-east-1"]=aki-b4aa75dd
|
||||
AKI["us-west-1"]=aki-eb7e26ae
|
||||
AKI["us-west-2"]=aki-f837bac8
|
||||
AKI["eu-west-1"]=aki-8b655dff
|
||||
AKI["ap-southeast-1"]=aki-fa1354a8
|
||||
AKI["ap-southeast-2"]=aki-3d990e07
|
||||
AKI["ap-northeast-1"]=aki-40992841
|
||||
AKI["sa-east-1"]=aki-c88f51d5
|
||||
# AKI["gov-west-1"]=aki-75a4c056
|
||||
|
||||
USAGE="Usage: $0 -a ami-id
|
||||
-a ami-id ID of the AMI to be coppied.
|
||||
-V VERSION Find AMI by CoreOS version.
|
||||
-K KEY Path to Amazon API private key.
|
||||
-C CERT Path to Amazon API key certificate.
|
||||
-h this ;-)
|
||||
-v Verbose, see all the things!
|
||||
|
||||
This script must be run from an ec2 host with the ec2 tools installed.
|
||||
"
|
||||
|
||||
AMI=
|
||||
VER=
|
||||
|
||||
while getopts "a:V:K:C:hv" OPTION
|
||||
do
|
||||
case $OPTION in
|
||||
a) AMI="$OPTARG";;
|
||||
V) VER="$OPTARG";;
|
||||
K) export EC2_PRIVATE_KEY="$OPTARG";;
|
||||
C) export EC2_CERT="$OPTARG";;
|
||||
h) echo "$USAGE"; exit;;
|
||||
v) set -x;;
|
||||
*) exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $(id -u) -eq 0 ]]; then
|
||||
echo "$0: This command should not be ran run as root!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$VER" ]]; then
|
||||
echo "$0: Providing the verison via -V is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$AMI" ]]; then
|
||||
AMI=$(ec2-describe-images -F name="CoreOS-$VER" | grep -m1 ^IMAGE \
|
||||
| cut -f2) || true # Don't die silently, error messages are good
|
||||
if [[ -z "$AMI" ]]; then
|
||||
echo "$0: Cannot find an AMI for CoreOS $VER" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# check to make sure this is a valid image
|
||||
if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then
|
||||
echo "$0: Unknown image: $AMI" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# The name has a limited set of allowed characterrs
|
||||
name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VER")
|
||||
description="CoreOS $VER"
|
||||
|
||||
zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
|
||||
zone=$(curl --fail -s $zoneurl)
|
||||
region=$(echo $zone | sed 's/.$//')
|
||||
|
||||
# hack job to copy AMIs
|
||||
export EC2_HOME=/home/ubuntu/ec2/ec2-api-tools-1.6.10.0
|
||||
export JAVA_HOME=/usr
|
||||
|
||||
do_copy() {
|
||||
local r="$1"
|
||||
r_amiid=$($EC2_HOME/bin/ec2-copy-image \
|
||||
--source-region "$region" \
|
||||
--source-ami-id "$AMI" \
|
||||
--name "$name" \
|
||||
--description "$description" \
|
||||
--region "$r" |
|
||||
cut -f2)
|
||||
echo "AMI copy to $r as $r_amiid in progress"
|
||||
|
||||
while ec2-describe-images "$r_amiid" --region="$r" | grep -q pending; do
|
||||
sleep 30
|
||||
done
|
||||
echo "AMI copy to $r as $r_amiid in complete"
|
||||
|
||||
# Not sure if this is needed, permissions seem to be copied
|
||||
echo "Making $r_amiid in $r public"
|
||||
ec2-modify-image-attribute --region "$r" "$r_amiid" --launch-permission -a all
|
||||
|
||||
# TODO: Add the awsmarket permissions to the snapshot backing the AMI which
|
||||
# certainly isn't copied. Need to parse ec2-describe-images or something.
|
||||
}
|
||||
|
||||
for r in "${!AKI[@]}"
|
||||
do
|
||||
[ "${r}" == "${region}" ] && continue
|
||||
echo "Starting copy of $AMI from $region to $r"
|
||||
do_copy "$r" &
|
||||
sleep 15
|
||||
done
|
||||
|
||||
wait
|
||||
|
||||
echo "Done"
|
||||
15
oem/ami/run.sh
Executable file
15
oem/ami/run.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
VER="$1"
|
||||
DIR=/home/ubuntu/official
|
||||
|
||||
if [ -z "$VER" ]; then
|
||||
echo "Usage: $0 1.2.3" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
sudo $DIR/build_ebs_on_ec2.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
|
||||
$DIR/test_ami2.sh -v -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
|
||||
$DIR/copy_ami.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
|
||||
$DIR/upload_ami_txt.sh -V $VER -K $DIR/aws-pk.pem -C $DIR/aws-cert.pem
|
||||
133
oem/ami/test_ami2.sh
Executable file
133
oem/ami/test_ami2.sh
Executable file
@ -0,0 +1,133 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# This expects to run on an EC2 instance.
|
||||
#
|
||||
# mad props to Eric Hammond for the initial script
|
||||
# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami
|
||||
|
||||
# This script will launch three ec2 nodes with shared user-data, and then
|
||||
# then test of the cluster is bootstrapped
|
||||
|
||||
# Set pipefail along with -e in hopes that we catch more errors
|
||||
set -e -o pipefail
|
||||
|
||||
USAGE="Usage: $0 -a ami-id
|
||||
-a ami-id ID of the AMI to be tests (required)
|
||||
-K KEY Path to Amazon API private key.
|
||||
-C CERT Path to Amazon API key certificate.
|
||||
-h this ;-)
|
||||
-v Verbose, see all the things!
|
||||
|
||||
This script must be run from an ec2 host with the ec2 tools installed.
|
||||
"
|
||||
|
||||
AMI=
|
||||
VER=
|
||||
|
||||
while getopts "a:V:K:C:hv" OPTION
|
||||
do
|
||||
case $OPTION in
|
||||
a) AMI="$OPTARG";;
|
||||
V) VER="$OPTARG";;
|
||||
K) export EC2_PRIVATE_KEY="$OPTARG";;
|
||||
C) export EC2_CERT="$OPTARG";;
|
||||
h) echo "$USAGE"; exit;;
|
||||
v) set -x;;
|
||||
*) exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $(id -u) -eq 0 ]]; then
|
||||
echo "$0: This command should not be ran run as root!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$AMI" && -n "$VER" ]]; then
|
||||
AMI=$(ec2-describe-images -F name="CoreOS-$VER" | grep -m1 ^IMAGE \
|
||||
| cut -f2) || true # Don't die silently, error messages are good
|
||||
if [[ -z "$AMI" ]]; then
|
||||
echo "$0: Cannot find an AMI for CoreOS $VER" >&2
|
||||
exit 1
|
||||
fi
|
||||
elif [[ -n "$AMI" ]]; then
|
||||
# check to make sure this is a valid image
|
||||
if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then
|
||||
echo "$0: Unknown image: $AMI" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "$0: AMI id or version required (-a or -V options)" >&2
|
||||
echo "$USAGE" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -n "Creating keys and security group... "
|
||||
key_name="autotest-`date +%s`"
|
||||
key_file="/tmp/$key_name"
|
||||
ec2-create-keypair $key_name | grep -v KEYPAIR > $key_file
|
||||
chmod 600 $key_file
|
||||
|
||||
sg_name=$key_name
|
||||
sg=$(ec2-create-group $sg_name --description "$sg_name" | cut -f2)
|
||||
ec2-authorize "$sg_name" -P tcp -p 4001 > /dev/null
|
||||
ec2-authorize "$sg_name" -P tcp -p 7001 > /dev/null
|
||||
ec2-authorize "$sg_name" -P tcp -p 22 > /dev/null
|
||||
echo "OK ($key_name)"
|
||||
|
||||
# might be needed later for multi-zone tests
|
||||
zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
|
||||
zone=$(curl --fail -s $zoneurl)
|
||||
region=$(echo $zone | sed 's/.$//')
|
||||
|
||||
token=$(uuidgen)
|
||||
|
||||
echo -n "Booting instances... "
|
||||
instances=$(ec2-run-instances \
|
||||
--user-data "$token" \
|
||||
--instance-type "t1.micro" \
|
||||
--instance-count 3 \
|
||||
--group "$sg_name" \
|
||||
--key "$key_name" $AMI | \
|
||||
grep INSTANCE | cut -f2)
|
||||
# little hack to create a describe instances command that only
|
||||
# pulls data for these instances
|
||||
ec2_cmd=$(echo $instances | sed 's/ / --filter instance-id=/g')
|
||||
ec2_cmd="ec2-describe-instances --filter instance-id=$ec2_cmd"
|
||||
|
||||
while $ec2_cmd | grep INSTANCE | grep -q pending
|
||||
do sleep 10; done
|
||||
|
||||
declare -a ips=($($ec2_cmd | grep INSTANCE | cut -f4))
|
||||
|
||||
# sleep until all the sockets we need come up
|
||||
for host in ${ips[@]}; do
|
||||
for port in 22 4001 7001; do
|
||||
timeout 90 perl -MIO::Socket::INET -e "
|
||||
until(new IO::Socket::INET('$host:$port')){sleep 1}"
|
||||
done
|
||||
done
|
||||
echo "OK ($instances)"
|
||||
|
||||
echo -n "Testing etcd... "
|
||||
test_key="v1/keys/test"
|
||||
# XXX: the sleep *should never* be required, this is a bug in etcd
|
||||
sleep 1
|
||||
curl --fail -s -L "${ips[0]}:4001/$test_key" -d value="$token" > /dev/null
|
||||
sleep 1
|
||||
for host in ${ips[@]}; do
|
||||
if ! curl --fail -s -L "${host}:4001/$test_key" | grep -q $token; then
|
||||
echo "etcd bootstrap appears to have failed for $host" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
echo "OK"
|
||||
|
||||
echo -n "Cleaning up environment... "
|
||||
ec2-terminate-instances $instances > /dev/null
|
||||
while ! $ec2_cmd | grep INSTANCE | grep -q terminated
|
||||
do sleep 10; done
|
||||
|
||||
ec2-delete-group $sg_name > /dev/null
|
||||
ec2-delete-keypair $key_name > /dev/null
|
||||
rm $key_file
|
||||
echo "OK"
|
||||
@ -16,7 +16,7 @@ AKI["ap-northeast-1"]=aki-40992841
|
||||
AKI["sa-east-1"]=aki-c88f51d5
|
||||
# AKI["gov-west-1"]=aki-75a4c056
|
||||
|
||||
USAGE="Usage: $0 -a ami-id
|
||||
USAGE="Usage: $0 -V 100.0.0
|
||||
-V VERSION Find AMI by CoreOS version. (required)
|
||||
-K KEY Path to Amazon API private key.
|
||||
-C CERT Path to Amazon API key certificate.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user