mirror of
https://github.com/flatcar/scripts.git
synced 2025-09-23 14:41:31 +02:00
feat(build_ebs_on_ec2): Fetch prod images by version, misc cleanup.
The build host will start generating production ami disk images so to simplify the next step this script can automatically fetch them from that location by version. The default sticks with the existing 'master' versioning scheme. Added logging and turned off -x by default to make the output log more readable. Removing the zip_and_ship script since it isn't useful with officially built disk images and only works with locally built images and a very particular ec2 host. A different long term automation scheme will have to be found.
This commit is contained in:
Michael Marineau
parent
c5cd245603
commit
37c56b64de
@ -1,4 +1,4 @@
|
||||
#!/bin/bash -ex
|
||||
#!/bin/bash
|
||||
#
|
||||
# This expects to run on an EC2 instance.
|
||||
#
|
||||
@ -9,6 +9,9 @@
|
||||
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html
|
||||
# we need pv-grub-hd00 x86_64
|
||||
|
||||
# Set pipefail along with -e in hopes that we catch more errors
|
||||
set -e -o pipefail
|
||||
|
||||
declare -A AKI
|
||||
AKI["us-east-1"]=aki-b4aa75dd
|
||||
AKI["us-west-1"]=aki-eb7e26ae
|
||||
@ -21,43 +24,90 @@ AKI["sa-east-1"]=aki-c88f51d5
|
||||
# AKI["gov-west-1"]=aki-75a4c056
|
||||
|
||||
readonly COREOS_EPOCH=1372636800
|
||||
TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
|
||||
EXTRA_VERSION=$(date +%H-%M)
|
||||
VERSION="master"
|
||||
IMAGE="coreos_production_ami_image.bin.bz2"
|
||||
URL_FMT="http://storage.core-os.net/coreos/amd64-generic/%s/$IMAGE"
|
||||
IMG_URL=""
|
||||
IMG_PATH=""
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
echo "usage: $0 [http|path/to/bin.bz2]"
|
||||
USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2]
|
||||
Options:
|
||||
-V VERSION Set the version of this AMI, default is 'master'
|
||||
-p PATH Path to compressed disk image, overrides -u
|
||||
-u URL URL to compressed disk image, derived from -V if unset.
|
||||
-K KEY Path to Amazon API private key.
|
||||
-C CERT Path to Amazon API key certificate.
|
||||
-h this ;-)
|
||||
-v Verbose, see all the things!
|
||||
|
||||
This script must be run from an ec2 host with the ec2 tools installed.
|
||||
"
|
||||
|
||||
while getopts "V:p:u:K:C:hv" OPTION
|
||||
do
|
||||
case $OPTION in
|
||||
V) VERSION="$OPTARG";;
|
||||
p) IMG_PATH="$OPTARG";;
|
||||
u) IMG_URL="$OPTARG";;
|
||||
K) export EC2_PRIVATE_KEY="$OPTARG";;
|
||||
C) export EC2_CERT="$OPTARG";;
|
||||
h) echo "$USAGE"; exit;;
|
||||
v) set -x;;
|
||||
*) exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ $(id -u) -ne 0 ]]; then
|
||||
echo "$0: This command must be run as root!" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
binurl=$1
|
||||
# Quick sanity check that the image exists
|
||||
if [[ -n "$IMG_PATH" ]]; then
|
||||
if [[ ! -f "$IMG_PATH" ]]; then
|
||||
echo "$0: Image path does not exist: $IMG_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
IMG_URL=$(basename "$IMG_PATH")
|
||||
else
|
||||
if [[ -z "$IMG_URL" ]]; then
|
||||
IMG_URL=$(printf "$URL_FMT" "$VERSION")
|
||||
fi
|
||||
if ! curl -s --head "$IMG_URL" >/dev/null; then
|
||||
echo "$0: Image URL unavailable: $IMG_URL" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$VERSION" == "master" ]]; then
|
||||
# Come up with something more descriptive and timestamped
|
||||
TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
|
||||
VERSION="master-${TODAYS_VERSION}-$(date +%H-%M)"
|
||||
fi
|
||||
|
||||
|
||||
# Size of AMI file system
|
||||
# TODO: Perhaps define size and arch in a metadata file image_to_vm creates?
|
||||
size=8 # GB
|
||||
|
||||
arch=x86_64
|
||||
arch2=amd64
|
||||
ephemeraldev=/dev/sdb
|
||||
version="master-$TODAYS_VERSION-$EXTRA_VERSION"
|
||||
|
||||
#TBD
|
||||
name="CoreOS-$version"
|
||||
description="CoreOS master"
|
||||
|
||||
export EC2_CERT=$(echo /tmp/*cert*.pem)
|
||||
export EC2_PRIVATE_KEY=$(echo /tmp/*pk*.pem)
|
||||
# The name has a limited set of allowed characterrs
|
||||
name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$VERSION")
|
||||
description="CoreOS $VERSION"
|
||||
|
||||
zoneurl=http://instance-data/latest/meta-data/placement/availability-zone
|
||||
zone=$(curl -s $zoneurl)
|
||||
region=$(echo $zone | sed 's/.$//')
|
||||
|
||||
# this is defined in: build_library/ami_constants.sh
|
||||
akiid=${AKI[$region]}
|
||||
|
||||
if [ -z "$akiid" ]; then
|
||||
echo "$0: Can't identify AKI, using region: $region";
|
||||
echo "$0: Can't identify AKI, using region: $region" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TODO: Once we sort out a long-term scheme for generating AMIs this likely
|
||||
# will go away. What if we want to generate AMIs from CoreOS?!?!?
|
||||
if [ -z "$(which ec2-attach-volume)" ]; then
|
||||
# Update and install Ubuntu packages
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
@ -69,12 +119,14 @@ if [ -z "$(which ec2-attach-volume)" ]; then
|
||||
ec2-ami-tools
|
||||
fi
|
||||
|
||||
export EC2_URL=http://ec2.$region.amazonaws.com
|
||||
export EC2_URL="http://ec2.${region}.amazonaws.com"
|
||||
echo "Building AMI in zone $zone, region id $akiid"
|
||||
|
||||
# Create and mount temporary EBS volume with file system to hold new AMI image
|
||||
volumeid=$(ec2-create-volume --size $size --availability-zone $zone |
|
||||
cut -f2)
|
||||
instanceid=$(wget -qO- http://instance-data/latest/meta-data/instance-id)
|
||||
instanceid=$(curl -s http://instance-data/latest/meta-data/instance-id)
|
||||
echo "Attaching new volume $volumeid locally (instance $instanceid)"
|
||||
ec2-attach-volume --device /dev/sdi --instance "$instanceid" "$volumeid"
|
||||
while [ ! -e /dev/sdi -a ! -e /dev/xvdi ]
|
||||
do sleep 3; done
|
||||
@ -84,13 +136,21 @@ else
|
||||
dev=/dev/sdi
|
||||
fi
|
||||
|
||||
echo "Attached volume $volumeid as $dev"
|
||||
echo "Writing image from $IMG_URL to $dev"
|
||||
|
||||
# if it is on the local fs, just use it, otherwise try to download it
|
||||
if [ -e "$binurl" ]; then
|
||||
bunzip2 -c $binurl | dd of=$dev bs=128M
|
||||
if [[ -n "$IMG_PATH" ]]; then
|
||||
if [[ "$IMG_PATH" =~ \.bz2$ ]]; then
|
||||
bunzip2 -c "$IMG_PATH" | dd of=$dev bs=1M
|
||||
else
|
||||
dd if="$IMG_PATH" of=$dev bs=1M
|
||||
fi
|
||||
else
|
||||
curl -s $binurl | bunzip2 | dd of=$dev bs=128M
|
||||
curl "$IMG_URL" | bunzip2 | dd of=$dev bs=1M
|
||||
fi
|
||||
|
||||
echo "Detaching $volumeid and creating snapshot"
|
||||
ec2-detach-volume "$volumeid"
|
||||
while ec2-describe-volumes "$volumeid" | grep -q ATTACHMENT
|
||||
do sleep 3; done
|
||||
@ -98,7 +158,7 @@ snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2)
|
||||
while ec2-describe-snapshots "$snapshotid" | grep -q pending
|
||||
do sleep 30; done
|
||||
|
||||
# Register the snapshot as a new AMI
|
||||
echo "Created snapshot $snapshotid, registering as a new AMI"
|
||||
amiid=$(ec2-register \
|
||||
--name "$name" \
|
||||
--description "$description" \
|
||||
@ -111,9 +171,9 @@ amiid=$(ec2-register \
|
||||
ec2-delete-volume "$volumeid"
|
||||
|
||||
cat <<EOF
|
||||
AMI: $amiid $codename $region $arch2
|
||||
AMI: $amiid $region $arch2
|
||||
|
||||
CoreOS $version
|
||||
CoreOS $VERSION
|
||||
architecture: $arch ($arch2)
|
||||
region: $region ($zone)
|
||||
aki id: $akiid
|
||||
@ -122,11 +182,11 @@ description: $description
|
||||
EBS volume: $volumeid (deleted)
|
||||
EBS snapshot: $snapshotid
|
||||
AMI id: $amiid
|
||||
bin url: $binurl
|
||||
bin url: $IMG_URL
|
||||
|
||||
Test the new AMI using something like:
|
||||
|
||||
export EC2_URL=http://ec2.$region.amazonaws.com
|
||||
export EC2_URL=http://ec2.${region}.amazonaws.com
|
||||
ec2-run-instances \\
|
||||
--key \$USER \\
|
||||
--instance-type t1.micro \\
|
||||
|
||||
@ -1,10 +0,0 @@
|
||||
#!/bin/bash -xe
|
||||
#
|
||||
# this needs more refactoring, but is used to build the latest image
|
||||
|
||||
IMG="../build/images/amd64-generic/latest/coreos_ami_image.bin"
|
||||
ssh -i ~/.ssh/coreos-images.pem ubuntu@23.22.1.1 "mkdir -p /mnt/tmp"
|
||||
|
||||
bzip2 $IMG
|
||||
scp -i ~/.ssh/coreos-images.pem build_ebs_on_ec2.sh $IMG.bz2 ubuntu@23.22.1.1:/mnt/tmp
|
||||
ssh -i ~/.ssh/coreos-images.pem ubuntu@23.22.1.1 "sudo /mnt/tmp/build_ebs_on_ec2.sh /mnt/tmp/coreos_ami_image.bin.bz2"
|
||||
Loading…
x
Reference in New Issue
Block a user