From 1508b44722505ce9317b09f2aec0e02de0a34421 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Mon, 20 Apr 2015 13:43:59 -0700 Subject: [PATCH] oem/ami: add new import script, replaces build_ebs_on_ec2.sh This script uses the EC2 volume import tools instead of attaching and writing to an EBS volume. This mechanism will be useful for creating AMIs in isolated EC2 regions and can be run from any host with API access and the EC2 tools. TODO: Allow region to be specified and automatically create region-local S3 buckets as needed. This version hard codes a bucket only usable by our dev AWS account, not prod. Later on: move to a more compact disk format like VMDK. --- oem/ami/import.sh | 203 ++++++++++++++++++++++++++++++++++++++++++++++ oem/ami/master.sh | 2 +- oem/ami/prod.sh | 2 +- oem/ami/user.sh | 2 +- 4 files changed, 206 insertions(+), 3 deletions(-) create mode 100755 oem/ami/import.sh diff --git a/oem/ami/import.sh b/oem/ami/import.sh new file mode 100755 index 0000000000..dad09fba93 --- /dev/null +++ b/oem/ami/import.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# +# This expects to run on an EC2 instance. +# +# mad props to Eric Hammond for the initial script +# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami + +# Set pipefail along with -e in hopes that we catch more errors +set -e -o pipefail + +DIR=$(dirname $0) +source $DIR/regions.sh + +readonly COREOS_EPOCH=1372636800 +VERSION="master" +BOARD="amd64-usr" +GROUP="alpha" +IMAGE="coreos_production_ami_image.bin.bz2" +GS_URL="gs://builds.release.core-os.net" +IMG_URL="" +IMG_PATH="" + +USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2] +Options: + -V VERSION Set the version of this AMI, default is 'master' + -b BOARD Set to the board name, default is amd64-usr + -g GROUP Set the update group, default is alpha or master + -p PATH Path to compressed disk image, overrides -u + -u URL URL to compressed disk image, derived from -V if unset. + -s STORAGE GS URL for Google storage (used to generate URL) + -h this ;-) + -v Verbose, see all the things! + +This script must be run from an ec2 host with the ec2 tools installed. +" + +while getopts "V:b:g:p:u:s:hv" OPTION +do + case $OPTION in + V) VERSION="$OPTARG";; + b) BOARD="$OPTARG";; + g) GROUP="$OPTARG";; + p) IMG_PATH="$OPTARG";; + u) IMG_URL="$OPTARG";; + s) GS_URL="$OPTARG";; + t) export TMPDIR="$OPTARG";; + h) echo "$USAGE"; exit;; + v) set -x;; + *) exit 1;; + esac +done + +if [[ $(id -u) -eq 0 ]]; then + echo "$0: This command should not be ran run as root!" >&2 + exit 1 +fi + +# Quick sanity check that the image exists +if [[ -n "$IMG_PATH" ]]; then + if [[ ! -f "$IMG_PATH" ]]; then + echo "$0: Image path does not exist: $IMG_PATH" >&2 + exit 1 + fi + IMG_URL=$(basename "$IMG_PATH") +else + if [[ -z "$IMG_URL" ]]; then + IMG_URL="$GS_URL/$GROUP/boards/$BOARD/$VERSION/$IMAGE" + fi + if [[ "$IMG_URL" == gs://* ]]; then + if ! gsutil -q stat "$IMG_URL"; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi + else + if ! curl --fail -s --head "$IMG_URL" >/dev/null; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi + fi +fi + +if [[ "$VERSION" == "master" ]]; then + # Come up with something more descriptive and timestamped + TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) + VERSION="${TODAYS_VERSION}-$(date +%H-%M)" + GROUP="master" +fi + +# Size of AMI file system +# TODO: Perhaps define size and arch in a metadata file image_to_vm creates? +size=8 # GB +arch=x86_64 +# The name has a limited set of allowed characterrs +name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$GROUP-$VERSION") +description="CoreOS $GROUP $VERSION" + +zoneurl=http://instance-data/latest/meta-data/placement/availability-zone +zone=$(curl --fail -s $zoneurl) +region=$(echo $zone | sed 's/.$//') +akiid=${ALL_AKIS[$region]} + +if [ -z "$akiid" ]; then + echo "$0: Can't identify AKI, using region: $region" >&2 + exit 1 +fi + +export EC2_URL="http://ec2.${region}.amazonaws.com" +echo "Building AMI in zone $zone, region id $akiid" + +tmpimg=$(mktemp) +trap "rm -f '${dldir}'" EXIT + +# if it is on the local fs, just use it, otherwise try to download it +if [[ -n "$IMG_PATH" ]]; then + if [[ "$IMG_PATH" =~ \.bz2$ ]]; then + bunzip2 -c "$IMG_PATH" >"${tmpimg}" + else + rm -f "${tmpimg}" + trap - EXIT + tmpimg="$IMG_PATH" + fi +elif [[ "$IMG_URL" == gs://* ]]; then + gsutil cat "$IMG_URL" | bunzip2 >"${tmpimg}" +else + curl --fail "$IMG_URL" | bunzip2 >"${tmpimg}" +fi + +#aws s3 mb s3://marineam-import-testing --region $region +importid=$(ec2-import-volume "${tmpimg}" \ + -f raw -s $size -x 2 -z $zone \ + -b marineam-import-testing \ + -o "${AWS_ACCESS_KEY}" \ + -w "${AWS_SECRET_KEY}" \ + --no-upload | awk '/IMPORTVOLUME/{print $4}') +ec2-resume-import "${tmpimg}" \ + -t "${importid}" -x 2 \ + -o "${AWS_ACCESS_KEY}" \ + -w "${AWS_SECRET_KEY}" + +echo "Waiting on import task ${importid}" +importstat=$(ec2-describe-conversion-tasks "${importid}" | grep IMPORTVOLUME) +while $(grep -qv completed <<<"${importstat}"); do + sed -e 's/.*StatusMessage/Status:/' <<<"${importstat}" + sleep 30 + importstat=$(ec2-describe-conversion-tasks "${importid}" | grep IMPORTVOLUME) +done + +volumeid=$(ec2-describe-conversion-tasks "${importid}" | \ + grep DISKIMAGE | sed -e 's%.*\(vol-[a-z0-9]*\).*%\1%') + +while ! ec2-describe-volumes "$volumeid" | grep -q available + do sleep 1; done + +echo "Volume ${volumeid} ready, deleting upload from S3..." +ec2-delete-disk-image \ + -t "${importid}" \ + -o "${AWS_ACCESS_KEY}" \ + -w "${AWS_SECRET_KEY}" + +echo "Creating snapshot..." +snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2) +echo "Waiting on snapshot ${snapshotid}" +while ec2-describe-snapshots "$snapshotid" | grep -q pending + do sleep 30; done + +echo "Created snapshot $snapshotid, deleting $volumeid" +ec2-delete-volume "$volumeid" + +echo "Registering hvm AMI" +hvm_amiid=$(ec2-register \ + --name "${name}-hvm" \ + --description "$description (HVM)" \ + --architecture "$arch" \ + --virtualization-type hvm \ + --root-device-name /dev/xvda \ + --block-device-mapping /dev/xvda=$snapshotid::true \ + --block-device-mapping /dev/xvdb=ephemeral0 | + cut -f2) + +echo "Registering paravirtual AMI" +amiid=$(ec2-register \ + --name "$name" \ + --description "$description (PV)" \ + --architecture "$arch" \ + --virtualization-type paravirtual \ + --kernel "$akiid" \ + --root-device-name /dev/sda \ + --block-device-mapping /dev/sda=$snapshotid::true \ + --block-device-mapping /dev/sdb=ephemeral0 | + cut -f2) + +cat <