diff --git a/oem/ami/import.sh b/oem/ami/import.sh new file mode 100755 index 0000000000..dad09fba93 --- /dev/null +++ b/oem/ami/import.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# +# This expects to run on an EC2 instance. +# +# mad props to Eric Hammond for the initial script +# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami + +# Set pipefail along with -e in hopes that we catch more errors +set -e -o pipefail + +DIR=$(dirname $0) +source $DIR/regions.sh + +readonly COREOS_EPOCH=1372636800 +VERSION="master" +BOARD="amd64-usr" +GROUP="alpha" +IMAGE="coreos_production_ami_image.bin.bz2" +GS_URL="gs://builds.release.core-os.net" +IMG_URL="" +IMG_PATH="" + +USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2] +Options: + -V VERSION Set the version of this AMI, default is 'master' + -b BOARD Set to the board name, default is amd64-usr + -g GROUP Set the update group, default is alpha or master + -p PATH Path to compressed disk image, overrides -u + -u URL URL to compressed disk image, derived from -V if unset. + -s STORAGE GS URL for Google storage (used to generate URL) + -h this ;-) + -v Verbose, see all the things! + +This script must be run from an ec2 host with the ec2 tools installed. +" + +while getopts "V:b:g:p:u:s:hv" OPTION +do + case $OPTION in + V) VERSION="$OPTARG";; + b) BOARD="$OPTARG";; + g) GROUP="$OPTARG";; + p) IMG_PATH="$OPTARG";; + u) IMG_URL="$OPTARG";; + s) GS_URL="$OPTARG";; + t) export TMPDIR="$OPTARG";; + h) echo "$USAGE"; exit;; + v) set -x;; + *) exit 1;; + esac +done + +if [[ $(id -u) -eq 0 ]]; then + echo "$0: This command should not be ran run as root!" >&2 + exit 1 +fi + +# Quick sanity check that the image exists +if [[ -n "$IMG_PATH" ]]; then + if [[ ! -f "$IMG_PATH" ]]; then + echo "$0: Image path does not exist: $IMG_PATH" >&2 + exit 1 + fi + IMG_URL=$(basename "$IMG_PATH") +else + if [[ -z "$IMG_URL" ]]; then + IMG_URL="$GS_URL/$GROUP/boards/$BOARD/$VERSION/$IMAGE" + fi + if [[ "$IMG_URL" == gs://* ]]; then + if ! gsutil -q stat "$IMG_URL"; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi + else + if ! curl --fail -s --head "$IMG_URL" >/dev/null; then + echo "$0: Image URL unavailable: $IMG_URL" >&2 + exit 1 + fi + fi +fi + +if [[ "$VERSION" == "master" ]]; then + # Come up with something more descriptive and timestamped + TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) + VERSION="${TODAYS_VERSION}-$(date +%H-%M)" + GROUP="master" +fi + +# Size of AMI file system +# TODO: Perhaps define size and arch in a metadata file image_to_vm creates? +size=8 # GB +arch=x86_64 +# The name has a limited set of allowed characterrs +name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$GROUP-$VERSION") +description="CoreOS $GROUP $VERSION" + +zoneurl=http://instance-data/latest/meta-data/placement/availability-zone +zone=$(curl --fail -s $zoneurl) +region=$(echo $zone | sed 's/.$//') +akiid=${ALL_AKIS[$region]} + +if [ -z "$akiid" ]; then + echo "$0: Can't identify AKI, using region: $region" >&2 + exit 1 +fi + +export EC2_URL="http://ec2.${region}.amazonaws.com" +echo "Building AMI in zone $zone, region id $akiid" + +tmpimg=$(mktemp) +trap "rm -f '${dldir}'" EXIT + +# if it is on the local fs, just use it, otherwise try to download it +if [[ -n "$IMG_PATH" ]]; then + if [[ "$IMG_PATH" =~ \.bz2$ ]]; then + bunzip2 -c "$IMG_PATH" >"${tmpimg}" + else + rm -f "${tmpimg}" + trap - EXIT + tmpimg="$IMG_PATH" + fi +elif [[ "$IMG_URL" == gs://* ]]; then + gsutil cat "$IMG_URL" | bunzip2 >"${tmpimg}" +else + curl --fail "$IMG_URL" | bunzip2 >"${tmpimg}" +fi + +#aws s3 mb s3://marineam-import-testing --region $region +importid=$(ec2-import-volume "${tmpimg}" \ + -f raw -s $size -x 2 -z $zone \ + -b marineam-import-testing \ + -o "${AWS_ACCESS_KEY}" \ + -w "${AWS_SECRET_KEY}" \ + --no-upload | awk '/IMPORTVOLUME/{print $4}') +ec2-resume-import "${tmpimg}" \ + -t "${importid}" -x 2 \ + -o "${AWS_ACCESS_KEY}" \ + -w "${AWS_SECRET_KEY}" + +echo "Waiting on import task ${importid}" +importstat=$(ec2-describe-conversion-tasks "${importid}" | grep IMPORTVOLUME) +while $(grep -qv completed <<<"${importstat}"); do + sed -e 's/.*StatusMessage/Status:/' <<<"${importstat}" + sleep 30 + importstat=$(ec2-describe-conversion-tasks "${importid}" | grep IMPORTVOLUME) +done + +volumeid=$(ec2-describe-conversion-tasks "${importid}" | \ + grep DISKIMAGE | sed -e 's%.*\(vol-[a-z0-9]*\).*%\1%') + +while ! ec2-describe-volumes "$volumeid" | grep -q available + do sleep 1; done + +echo "Volume ${volumeid} ready, deleting upload from S3..." +ec2-delete-disk-image \ + -t "${importid}" \ + -o "${AWS_ACCESS_KEY}" \ + -w "${AWS_SECRET_KEY}" + +echo "Creating snapshot..." +snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2) +echo "Waiting on snapshot ${snapshotid}" +while ec2-describe-snapshots "$snapshotid" | grep -q pending + do sleep 30; done + +echo "Created snapshot $snapshotid, deleting $volumeid" +ec2-delete-volume "$volumeid" + +echo "Registering hvm AMI" +hvm_amiid=$(ec2-register \ + --name "${name}-hvm" \ + --description "$description (HVM)" \ + --architecture "$arch" \ + --virtualization-type hvm \ + --root-device-name /dev/xvda \ + --block-device-mapping /dev/xvda=$snapshotid::true \ + --block-device-mapping /dev/xvdb=ephemeral0 | + cut -f2) + +echo "Registering paravirtual AMI" +amiid=$(ec2-register \ + --name "$name" \ + --description "$description (PV)" \ + --architecture "$arch" \ + --virtualization-type paravirtual \ + --kernel "$akiid" \ + --root-device-name /dev/sda \ + --block-device-mapping /dev/sda=$snapshotid::true \ + --block-device-mapping /dev/sdb=ephemeral0 | + cut -f2) + +cat <