diff --git a/oem/ami/build_ebs_on_ec2.sh b/oem/ami/build_ebs_on_ec2.sh deleted file mode 100755 index 7f2e2ddcca..0000000000 --- a/oem/ami/build_ebs_on_ec2.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash -# -# This expects to run on an EC2 instance. -# -# mad props to Eric Hammond for the initial script -# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami - -# Set pipefail along with -e in hopes that we catch more errors -set -e -o pipefail - -DIR=$(dirname $0) -source $DIR/regions.sh - -readonly COREOS_EPOCH=1372636800 -VERSION="master" -BOARD="amd64-usr" -GROUP="alpha" -IMAGE="coreos_production_ami_image.bin.bz2" -GS_URL="gs://builds.release.core-os.net" -IMG_URL="" -IMG_PATH="" - -USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2] -Options: - -V VERSION Set the version of this AMI, default is 'master' - -b BOARD Set to the board name, default is amd64-usr - -g GROUP Set the update group, default is alpha or master - -p PATH Path to compressed disk image, overrides -u - -u URL URL to compressed disk image, derived from -V if unset. - -s STORAGE GS URL for Google storage (used to generate URL) - -h this ;-) - -v Verbose, see all the things! - -This script must be run from an ec2 host with the ec2 tools installed. -" - -while getopts "V:b:g:p:u:s:hv" OPTION -do - case $OPTION in - V) VERSION="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - p) IMG_PATH="$OPTARG";; - u) IMG_URL="$OPTARG";; - s) GS_URL="$OPTARG";; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -# Quick sanity check that the image exists -if [[ -n "$IMG_PATH" ]]; then - if [[ ! -f "$IMG_PATH" ]]; then - echo "$0: Image path does not exist: $IMG_PATH" >&2 - exit 1 - fi - IMG_URL=$(basename "$IMG_PATH") -else - if [[ -z "$IMG_URL" ]]; then - IMG_URL="$GS_URL/$GROUP/boards/$BOARD/$VERSION/$IMAGE" - fi - if [[ "$IMG_URL" == gs://* ]]; then - if ! gsutil -q stat "$IMG_URL"; then - echo "$0: Image URL unavailable: $IMG_URL" >&2 - exit 1 - fi - else - if ! curl --fail -s --head "$IMG_URL" >/dev/null; then - echo "$0: Image URL unavailable: $IMG_URL" >&2 - exit 1 - fi - fi -fi - -if [[ "$VERSION" == "master" ]]; then - # Come up with something more descriptive and timestamped - TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) - VERSION="${TODAYS_VERSION}-$(date +%H-%M)" - GROUP="master" -fi - -# Size of AMI file system -# TODO: Perhaps define size and arch in a metadata file image_to_vm creates? -size=8 # GB -arch=x86_64 -arch2=amd64 -# The name has a limited set of allowed characterrs -name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$GROUP-$VERSION") -description="CoreOS $GROUP $VERSION" - -zoneurl=http://instance-data/latest/meta-data/placement/availability-zone -zone=$(curl --fail -s $zoneurl) -region=$(echo $zone | sed 's/.$//') -akiid=${ALL_AKIS[$region]} - -if [ -z "$akiid" ]; then - echo "$0: Can't identify AKI, using region: $region" >&2 - exit 1 -fi - -export EC2_URL="http://ec2.${region}.amazonaws.com" -echo "Building AMI in zone $zone, region id $akiid" - -# Create and mount temporary EBS volume with file system to hold new AMI image -volumeid=$(ec2-create-volume --size $size --availability-zone $zone | - cut -f2) -while ! ec2-describe-volumes "$volumeid" | grep -q available - do sleep 1; done -instanceid=$(curl --fail -s http://instance-data/latest/meta-data/instance-id) -echo "Attaching new volume $volumeid locally (instance $instanceid)" -ec2-attach-volume --device /dev/sdi --instance "$instanceid" "$volumeid" -while [ ! -e /dev/sdi -a ! -e /dev/xvdi ] - do sleep 3; done -if [ -e /dev/xvdi ]; then - dev=/dev/xvdi -else - dev=/dev/sdi -fi - -echo "Attached volume $volumeid as $dev" -echo "Writing image from $IMG_URL to $dev" - -# if it is on the local fs, just use it, otherwise try to download it -if [[ -n "$IMG_PATH" ]]; then - if [[ "$IMG_PATH" =~ \.bz2$ ]]; then - bunzip2 -c "$IMG_PATH" | sudo dd of=$dev bs=1M - else - sudo dd if="$IMG_PATH" of=$dev bs=1M - fi -elif [[ "$IMG_URL" == gs://* ]]; then - gsutil cat "$IMG_URL" | bunzip2 | sudo dd of=$dev bs=1M -else - curl --fail "$IMG_URL" | bunzip2 | sudo dd of=$dev bs=1M -fi - -echo "Detaching $volumeid and creating snapshot" -ec2-detach-volume "$volumeid" -while ec2-describe-volumes "$volumeid" | grep -q ATTACHMENT - do sleep 3; done -snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2) -while ec2-describe-snapshots "$snapshotid" | grep -q pending - do sleep 30; done - -echo "Created snapshot $snapshotid, deleting $volumeid" -ec2-delete-volume "$volumeid" - -echo "Registering hvm AMI" -hvm_amiid=$(ec2-register \ - --name "${name}-hvm" \ - --description "$description (HVM)" \ - --architecture "$arch" \ - --virtualization-type hvm \ - --root-device-name /dev/xvda \ - --block-device-mapping /dev/xvda=$snapshotid::true \ - --block-device-mapping /dev/xvdb=ephemeral0 | - cut -f2) - -echo "Registering paravirtual AMI" -amiid=$(ec2-register \ - --name "$name" \ - --description "$description (PV)" \ - --architecture "$arch" \ - --virtualization-type paravirtual \ - --kernel "$akiid" \ - --root-device-name /dev/sda \ - --block-device-mapping /dev/sda=$snapshotid::true \ - --block-device-mapping /dev/sdb=ephemeral0 | - cut -f2) - -cat <&2; - exit 1 - fi - REGIONS+=( "$1" ) -} - -clean_version() { - sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1" -} - -while getopts "a:V:b:g:l:r:hv" OPTION -do - case $OPTION in - a) AMI="$OPTARG";; - V) VER="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - l) GRANT_LAUNCH="${OPTARG}";; - r) add_region "$OPTARG";; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -if [[ -z "$VER" ]]; then - echo "$0: Providing the verison via -V is required." >&2 - exit 1 -fi - -zoneurl=http://instance-data/latest/meta-data/placement/availability-zone -zone=$(curl --fail -s $zoneurl) -region=$(echo $zone | sed 's/.$//') -export EC2_URL="https://ec2.${region}.amazonaws.com" - -if [[ -z "$AMI" ]]; then - search_name=$(clean_version "CoreOS-$GROUP-$VER") - AMI=$(ec2-describe-images -F name="${search_name}" | grep -m1 ^IMAGE \ - | cut -f2) || true # Don't die silently, error messages are good - if [[ -z "$AMI" ]]; then - echo "$0: Cannot find an AMI named $search_name" >&2 - exit 1 - fi - HVM=$(ec2-describe-images -F name="${search_name}-hvm" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$HVM" ]]; then - echo "$0: Cannot find an AMI named ${search_name}-hvm" >&2 - exit 1 - fi -else - # check to make sure this is a valid image - if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then - echo "$0: Unknown image: $AMI" >&2 - exit 1 - fi -fi - -if [[ ${#REGIONS[@]} -eq 0 ]]; then - REGIONS=( "${MAIN_REGIONS[@]}" ) -fi - -# The name has a limited set of allowed characterrs -name=$(clean_version "CoreOS-$GROUP-$VER") -description="CoreOS $GROUP $VER" - -do_copy() { - local r="$1" - local virt_type="$2" - local local_amiid="$3" - local r_amiid r_name r_desc - - # run in a subshell, the -e flag doesn't get inherited - set -e - - echo "Starting copy of $virt_type $local_amiid from $region to $r" - if [[ "$virt_type" == "hvm" ]]; then - r_name="${name}-hvm" - r_desc="${description} (HVM)" - else - r_name="${name}" - r_desc="${description} (PV)" - fi - r_amiid=$(ec2-copy-image \ - --source-region "$region" --source-ami-id "$local_amiid" \ - --name "$r_name" --description "$r_desc" --region "$r" | - cut -f2) - echo "AMI $virt_type copy to $r as $r_amiid in progress" - - while ec2-describe-images "$r_amiid" --region="$r" | grep -q pending; do - sleep 30 - done - - if [[ -n "${GRANT_LAUNCH}" ]]; then - echo "Granting launch permission to ${GRANT_LAUNCH} for $r_amiid in $r" - ec2-modify-image-attribute --region="$r" "${r_amiid}" \ - --launch-permission --add "${GRANT_LAUNCH}" - fi - - echo "AMI $virt_type copy to $r as $r_amiid in complete" -} - -WAIT_PIDS=() -for r in "${REGIONS[@]}" -do - [ "${r}" == "${region}" ] && continue - do_copy "$r" pv "$AMI" & - WAIT_PIDS+=( $! ) -done - -# wait for each subshell individually to report errors -WAIT_FAILED=0 -for wait_pid in "${WAIT_PIDS[@]}"; do - if ! wait ${wait_pid}; then - : $(( WAIT_FAILED++ )) - fi -done - -if [[ ${WAIT_FAILED} -ne 0 ]]; then - echo "${WAIT_FAILED} jobs failed :(" >&2 - exit ${WAIT_FAILED} -fi - -WAIT_PIDS=() -for r in "${REGIONS[@]}" -do - [ "${r}" == "${region}" ] && continue - if [[ -n "$HVM" ]]; then - do_copy "$r" hvm "$HVM" & - WAIT_PIDS+=( $! ) - fi -done - -# wait for each subshell individually to report errors -WAIT_FAILED=0 -for wait_pid in "${WAIT_PIDS[@]}"; do - if ! wait ${wait_pid}; then - : $(( WAIT_FAILED++ )) - fi -done - -if [[ ${WAIT_FAILED} -ne 0 ]]; then - echo "${WAIT_FAILED} jobs failed :(" >&2 - exit ${WAIT_FAILED} -fi - -echo "Done" diff --git a/oem/ami/import.sh b/oem/ami/import.sh deleted file mode 100755 index 2f85002b40..0000000000 --- a/oem/ami/import.sh +++ /dev/null @@ -1,252 +0,0 @@ -#!/bin/bash -# -# This expects to run on an EC2 instance. -# -# mad props to Eric Hammond for the initial script -# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami - -# Set pipefail along with -e in hopes that we catch more errors -set -e -o pipefail - -DIR=$(dirname $0) -source $DIR/regions.sh - -readonly COREOS_EPOCH=1372636800 -VERSION="master" -BOARD="amd64-usr" -GROUP="alpha" -IMAGE="coreos_production_ami_image.bin.bz2" -GS_URL="gs://builds.release.core-os.net" -IMG_URL="" -IMG_PATH="" -GRANT_LAUNCH="" -USE_GPG=1 -# accepted via the environment -: ${EC2_IMPORT_BUCKET:=} -: ${EC2_IMPORT_ZONE:=} - -USAGE="Usage: $0 [-V 1.2.3] [-p path/image.bz2 | -u http://foo/image.bz2] -Options: - -V VERSION Set the version of this AMI, default is 'master' - -b BOARD Set to the board name, default is amd64-usr - -g GROUP Set the update group, default is alpha or master - -p PATH Path to compressed disk image, overrides -u - -u URL URL to compressed disk image, derived from -V if unset. - -s STORAGE GS URL for Google storage (used to generate URL) - -B BUCKET S3 bucket to use for temporary storage. - -Z ZONE EC2 availability zone to use. - -l ACCOUNT Grant launch permission to a given AWS account ID. - -X Disable GPG verification of downloads. - -h this ;-) - -v Verbose, see all the things! - -This script must be run from an ec2 host with the ec2 tools installed. -" - -while getopts "V:b:g:p:u:s:t:l:B:Z:Xhv" OPTION -do - case $OPTION in - V) VERSION="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - p) IMG_PATH="$OPTARG";; - u) IMG_URL="$OPTARG";; - s) GS_URL="$OPTARG";; - B) EC2_IMPORT_BUCKET="${OPTARG}";; - Z) EC2_IMPORT_ZONE="${OPTARG}";; - l) GRANT_LAUNCH="${OPTARG}";; - t) export TMPDIR="$OPTARG";; - X) USE_GPG=0;; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -if [[ -z "${EC2_IMPORT_BUCKET}" ]]; then - echo "$0: -B or \$EC2_IMPORT_BUCKET must be set!" >&2 - exit 1 -fi - -# Quick sanity check that the image exists -if [[ -n "$IMG_PATH" ]]; then - if [[ ! -f "$IMG_PATH" ]]; then - echo "$0: Image path does not exist: $IMG_PATH" >&2 - exit 1 - fi - IMG_URL=$(basename "$IMG_PATH") -else - if [[ -z "$IMG_URL" ]]; then - IMG_URL="$GS_URL/$GROUP/boards/$BOARD/$VERSION/$IMAGE" - fi - if [[ "$IMG_URL" == gs://* ]]; then - if ! gsutil -q stat "$IMG_URL"; then - echo "$0: Image URL unavailable: $IMG_URL" >&2 - exit 1 - fi - else - if ! curl --fail -s --head "$IMG_URL" >/dev/null; then - echo "$0: Image URL unavailable: $IMG_URL" >&2 - exit 1 - fi - fi -fi - -if [[ "$VERSION" == "master" ]]; then - # Come up with something more descriptive and timestamped - TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) - VERSION="${TODAYS_VERSION}-$(date +%H-%M)" - GROUP="master" -fi - -# Size of AMI file system -# TODO: Perhaps define size and arch in a metadata file image_to_vm creates? -size=8 # GB -arch=x86_64 -# The name has a limited set of allowed characterrs -name=$(sed -e "s%[^A-Za-z0-9()\\./_-]%_%g" <<< "CoreOS-$GROUP-$VERSION") -description="CoreOS $GROUP $VERSION" - -if [[ -z "${EC2_IMPORT_ZONE}" ]]; then - zoneurl=http://instance-data/latest/meta-data/placement/availability-zone - EC2_IMPORT_ZONE=$(curl --fail -s $zoneurl) -fi -region=$(echo "${EC2_IMPORT_ZONE}" | sed 's/.$//') -akiid=${ALL_AKIS[$region]} - -if [ -z "$akiid" ]; then - echo "$0: Can't identify AKI, using region: $region" >&2 - exit 1 -fi - -export EC2_URL="https://ec2.${region}.amazonaws.com" -echo "Building AMI in zone ${EC2_IMPORT_ZONE}" - -tmpdir=$(mktemp --directory --tmpdir=/var/tmp) -trap "rm -rf '${tmpdir}'" EXIT - -# if it is on the local fs, just use it, otherwise try to download it -if [[ -z "$IMG_PATH" ]]; then - IMG_PATH="${tmpdir}/${IMG_URL##*/}" - if [[ "$IMG_URL" == gs://* ]]; then - gsutil cp "$IMG_URL" "$IMG_PATH" - if [[ "$USE_GPG" != 0 ]]; then - gsutil cp "${IMG_URL}.sig" "${IMG_PATH}.sig" - fi - else - curl --fail "$IMG_URL" > "$IMG_PATH" - if [[ "$USE_GPG" != 0 ]]; then - curl --fail "${IMG_URL}.sig" > "${IMG_PATH}.sig" - fi - fi -fi - -if [[ "$USE_GPG" != 0 ]]; then - gpg --verify "${IMG_PATH}.sig" -fi - -echo "Bunzipping...." -tmpimg="${tmpdir}/img" -bunzip2 -c "$IMG_PATH" >"${tmpimg}" - -imgfmt=ponies -case "$IMG_PATH" in - *_image.bin*) imgfmt=raw;; - *_image.vmdk*) imgfmt=vmdk;; - *_image.vhd*) imgfmt=vhd;; - *) - echo "$0: Cannot guess image format from image path!" - exit 1 - ;; -esac - -importid=$(ec2-import-volume "${tmpimg}" \ - -f $imgfmt -s $size -x 2 \ - -z "${EC2_IMPORT_ZONE}" \ - -b "${EC2_IMPORT_BUCKET}" \ - -o "${AWS_ACCESS_KEY}" \ - -w "${AWS_SECRET_KEY}" \ - --no-upload | awk '/IMPORTVOLUME/{print $4}') -ec2-resume-import "${tmpimg}" \ - -t "${importid}" -x 2 \ - -o "${AWS_ACCESS_KEY}" \ - -w "${AWS_SECRET_KEY}" - -echo "Waiting on import task ${importid}" -importstat=$(ec2-describe-conversion-tasks "${importid}" | grep IMPORTVOLUME) -while $(grep -qv completed <<<"${importstat}"); do - sed -e 's/.*StatusMessage/Status:/' <<<"${importstat}" - sleep 30 - importstat=$(ec2-describe-conversion-tasks "${importid}" | grep IMPORTVOLUME) -done - -volumeid=$(ec2-describe-conversion-tasks "${importid}" | \ - grep DISKIMAGE | sed -e 's%.*\(vol-[a-z0-9]*\).*%\1%') - -while ! ec2-describe-volumes "$volumeid" | grep -q available - do sleep 1; done - -echo "Volume ${volumeid} ready, deleting upload from S3..." -ec2-delete-disk-image \ - -t "${importid}" \ - -o "${AWS_ACCESS_KEY}" \ - -w "${AWS_SECRET_KEY}" - -echo "Creating snapshot..." -snapshotid=$(ec2-create-snapshot --description "$name" "$volumeid" | cut -f2) -echo "Waiting on snapshot ${snapshotid}" -while ec2-describe-snapshots "$snapshotid" | grep -q pending - do sleep 30; done - -echo "Created snapshot $snapshotid, deleting $volumeid" -ec2-delete-volume "$volumeid" - -echo "Registering hvm AMI" -hvm_amiid=$(ec2-register \ - --name "${name}-hvm" \ - --description "$description (HVM)" \ - --architecture "$arch" \ - --virtualization-type hvm \ - --root-device-name /dev/xvda \ - --block-device-mapping /dev/xvda=$snapshotid::true \ - --sriov simple \ - --block-device-mapping /dev/xvdb=ephemeral0 | - cut -f2) - -echo "Registering paravirtual AMI" -amiid=$(ec2-register \ - --name "$name" \ - --description "$description (PV)" \ - --architecture "$arch" \ - --virtualization-type paravirtual \ - --kernel "$akiid" \ - --root-device-name /dev/sda \ - --block-device-mapping /dev/sda=$snapshotid::true \ - --block-device-mapping /dev/sdb=ephemeral0 | - cut -f2) - -if [[ -n "${GRANT_LAUNCH}" ]]; then - echo "Granting launch permission to ${GRANT_LAUNCH}" - ec2-modify-image-attribute "${hvm_amiid}" \ - --launch-permission --add "${GRANT_LAUNCH}" - ec2-modify-image-attribute "${amiid}" \ - --launch-permission --add "${GRANT_LAUNCH}" -fi - -cat <&2 - exit 1 -fi - -if [[ -z "${EC2_IMPORT_BUCKET}" ]]; then - echo "$0: -B or \$EC2_IMPORT_BUCKET must be set!" >&2 - exit 1 -fi - -if [[ -z "${EC2_IMPORT_ZONE}" ]]; then - zoneurl=http://instance-data/latest/meta-data/placement/availability-zone - EC2_IMPORT_ZONE=$(curl --fail -s $zoneurl) -fi -region=$(echo "${EC2_IMPORT_ZONE}" | sed 's/.$//') - -# The AWS cli uses slightly different vars than the EC2 cli... -export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY}" -export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_KEY}" -aws s3 mb "s3://${EC2_IMPORT_BUCKET}" --region "$region" diff --git a/oem/ami/master.sh b/oem/ami/master.sh deleted file mode 100755 index 9c4ce0200d..0000000000 --- a/oem/ami/master.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -DIR=/home/ec2-user/scripts/oem/ami -URL="https://commondatastorage.googleapis.com/storage.core-os.net/coreos/amd64-usr/master" - -set -e -eval $(curl -f "${URL}/version.txt") - -source $DIR/marineam-auth.sh -args="-b amd64-usr -g master -V ${COREOS_VERSION}" -$DIR/import.sh ${args} -u "${URL}/coreos_production_ami_image.bin.bz2" -$DIR/test_ami.sh -v ${args} -#$DIR/copy_ami.sh ${args} diff --git a/oem/ami/prod-publish.sh b/oem/ami/prod-publish.sh deleted file mode 100755 index 6badf03527..0000000000 --- a/oem/ami/prod-publish.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -BOARD="amd64-usr" -GROUP="$1" -VER="$2" -DIR=/home/ec2-user/scripts/oem/ami - -if [ -z "$GROUP" -o -z "$VER" ]; then - echo "Usage: $0 alpha 1.2.3" >&2 - exit 1 -fi - -$DIR/publish_ami.sh -b $BOARD -g $GROUP -V $VER diff --git a/oem/ami/prod.sh b/oem/ami/prod.sh deleted file mode 100755 index a872391f50..0000000000 --- a/oem/ami/prod.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -BOARD="amd64-usr" -GROUP="$1" -VER="$2" -DIR=/home/ec2-user/scripts/oem/ami - -if [ -z "$GROUP" -o -z "$VER" ]; then - echo "Usage: $0 alpha 1.2.3" >&2 - exit 1 -fi - -set -e -source $DIR/marineam-auth.sh -args="-b $BOARD -g $GROUP -V $VER" -$DIR/import.sh -l 477645798544 ${args} -$DIR/test_ami.sh -v ${args} -$DIR/copy_ami.sh -l 477645798544 ${args} - -source $DIR/ami-builder-us-gov-auth.sh -$DIR/import.sh ${args} - -$DIR/update_json.sh ${args} diff --git a/oem/ami/publish_ami.sh b/oem/ami/publish_ami.sh deleted file mode 100755 index dff084b4d8..0000000000 --- a/oem/ami/publish_ami.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash -# -# Set pipefail along with -e in hopes that we catch more errors -set -e -o pipefail - -DIR=$(dirname $0) -source $DIR/regions.sh - -USAGE="Usage: $0 -V 100.0.0 - -V VERSION Find AMI by CoreOS version. (required) - -b BOARD Set to the board name, default is amd64-usr - -g GROUP Set the update group, default is alpha - -h this ;-) - -v Verbose, see all the things! - -This script must be run from an ec2 host with the ec2 tools installed. -" - -IMAGE="coreos_production_ami" -AMI= -VER= -BOARD="amd64-usr" -GROUP="alpha" - -clean_version() { - sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1" -} - -while getopts "V:b:g:s:hv" OPTION -do - case $OPTION in - V) VER="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -if [[ ! -n "$VER" ]]; then - echo "$0: AMI version required via -V" >&2 - echo "$USAGE" >&2 - exit 1 -fi - -search_name=$(clean_version "CoreOS-$GROUP-$VER") -declare -A AMIS HVM_AMIS -for r in "${ALL_REGIONS[@]}"; do - # Hacky but avoids writing an indirection layer to handle auth... - if [[ "${r}" == "us-gov-west-1" ]]; then - source $DIR/ami-builder-us-gov-auth.sh - else - source $DIR/marineam-auth.sh - fi - - AMI=$(ec2-describe-images --region=${r} -F name="${search_name}" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$AMI" ]]; then - echo "$0: Cannot find an AMI named ${search_name} in ${r}" >&2 - exit 1 - fi - AMIS[${r}]=$AMI - HVM=$(ec2-describe-images --region=${r} -F name="${search_name}-hvm" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$HVM" ]]; then - echo "$0: Cannot find an AMI named ${search_name}-hvm in ${r}" >&2 - exit 1 - fi - HVM_AMIS[${r}]=$HVM -done - -publish_ami() { - local r="$1" - local virt_type="$2" - local r_amiid="$3" - - if [[ "${r}" == "us-gov-west-1" ]]; then - source $DIR/ami-builder-us-gov-auth.sh - else - source $DIR/marineam-auth.sh - fi - - local r_snapshotid=$(ec2-describe-images --region="$r" "$r_amiid" \ - | grep -E '^BLOCKDEVICEMAPPING.*/dev/(xv|s)da' | cut -f5) || true - - if [[ -z "${r_snapshotid}" ]]; then - echo "$0: Cannot find snapshot id for $r_amiid in $r" >&2 - return 1 - fi - - echo "Making $r_snapshotid in $r public" - ec2-modify-snapshot-attribute --region "$r" \ - "$r_snapshotid" --create-volume-permission --add all - - echo "Making $r_amiid in $r public" - ec2-modify-image-attribute --region "$r" \ - "$r_amiid" --launch-permission -a all -} - -for r in "${!AMIS[@]}"; do - publish_ami "$r" pv "${AMIS[$r]}" -done - -for r in "${!HVM_AMIS[@]}"; do - publish_ami "$r" hvm "${HVM_AMIS[$r]}" -done diff --git a/oem/ami/regions.sh b/oem/ami/regions.sh deleted file mode 100644 index 7421300c44..0000000000 --- a/oem/ami/regions.sh +++ /dev/null @@ -1,26 +0,0 @@ -# AKI ids from: -# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/UserProvidedkernels.html -# These are pv-grub-hd0_1.04-x86_64 - -declare -A ALL_AKIS -ALL_AKIS["us-east-1"]=aki-919dcaf8 -ALL_AKIS["us-east-2"]=aki-da055ebf -ALL_AKIS["us-west-1"]=aki-880531cd -ALL_AKIS["us-west-2"]=aki-fc8f11cc -ALL_AKIS["eu-west-1"]=aki-52a34525 -ALL_AKIS["eu-west-2"]=aki-8b6369ef -ALL_AKIS["eu-central-1"]=aki-184c7a05 -ALL_AKIS["ap-south-1"]=aki-a7305ac8 -ALL_AKIS["ap-southeast-1"]=aki-503e7402 -ALL_AKIS["ap-southeast-2"]=aki-c362fff9 -ALL_AKIS["ap-northeast-1"]=aki-176bf516 -ALL_AKIS["ap-northeast-2"]=aki-01a66b6f -ALL_AKIS["sa-east-1"]=aki-5553f448 -ALL_AKIS["ca-central-1"]=aki-320ebd56 - -MAIN_REGIONS=( "${!ALL_AKIS[@]}" ) - -# The following are isolated regions -ALL_AKIS["us-gov-west-1"]=aki-1de98d3e - -ALL_REGIONS=( "${!ALL_AKIS[@]}" ) diff --git a/oem/ami/test_ami.sh b/oem/ami/test_ami.sh deleted file mode 100755 index 0b233cdaa4..0000000000 --- a/oem/ami/test_ami.sh +++ /dev/null @@ -1,212 +0,0 @@ -#!/bin/bash -# -# This expects to run on an EC2 instance. -# -# mad props to Eric Hammond for the initial script -# https://github.com/alestic/alestic-hardy-ebs/blob/master/bin/alestic-hardy-ebs-build-ami - -# This script will launch three ec2 nodes with shared user-data, and then -# then test of the cluster is bootstrapped - -# Set pipefail along with -e in hopes that we catch more errors -set -e -o pipefail - -USAGE="Usage: $0 -a ami-id - -a ami-id ID of the AMI to be tests - -V VERSION Find AMI by CoreOS version. - -b BOARD Set to the board name, default is amd64-usr - -g GROUP Set the update group, default is alpha - -h this ;-) - -v Verbose, see all the things! - -The AMI to test must be specified by -a or -V. -This script must be run from an ec2 host with the ec2 tools installed. -" - -AMI= -HVM= -VER= -BOARD="amd64-usr" -GROUP="alpha" - -clean_version() { - sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1" -} - -while getopts "a:V:b:g:hv" OPTION -do - case $OPTION in - a) AMI="$OPTARG";; - V) VER="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -zoneurl=http://instance-data/latest/meta-data/placement/availability-zone -zone=$(curl --fail -s $zoneurl) -region=$(echo $zone | sed 's/.$//') -export EC2_URL="https://ec2.${region}.amazonaws.com" - -if [[ -z "$AMI" && -n "$VER" ]]; then - search_name=$(clean_version "CoreOS-$GROUP-$VER") - AMI=$(ec2-describe-images -F name="${search_name}" | grep -m1 ^IMAGE \ - | cut -f2) || true # Don't die silently, error messages are good - if [[ -z "$AMI" ]]; then - echo "$0: Cannot find an AMI named $search_name" >&2 - exit 1 - fi - HVM=$(ec2-describe-images -F name="${search_name}-hvm" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$HVM" ]]; then - echo "$0: Cannot find an AMI named ${search_name}-hvm" >&2 - exit 1 - fi -elif [[ -n "$AMI" ]]; then - # check to make sure this is a valid image - if ! ec2-describe-images -F image-id="$AMI" | grep -q "$AMI"; then - echo "$0: Unknown image: $AMI" >&2 - exit 1 - fi -else - echo "$0: AMI id or version required (-a or -V options)" >&2 - echo "$USAGE" >&2 - exit 1 -fi - -echo -n "Creating keys and security group... " -key_name="autotest-`date +%s`" -key_file="/tmp/$key_name" -ec2-create-keypair $key_name | grep -v KEYPAIR > $key_file -chmod 600 $key_file - -sg_name=$key_name -sg=$(ec2-create-group $sg_name --description "$sg_name" | cut -f2) -ec2-authorize "$sg_name" -P tcp -p 4001 > /dev/null -ec2-authorize "$sg_name" -P tcp -p 7001 > /dev/null -ec2-authorize "$sg_name" -P tcp -p 22 > /dev/null -echo "OK ($key_name)" - -discovery=$(curl --fail -s https://discovery.etcd.io/new) -userdata="#cloud-config - -coreos: - etcd: - discovery: $discovery - addr: \$public_ipv4:4001 - peer-addr: \$public_ipv4:7001 - units: - - name: etcd.service - command: start - - name: fleet.service - command: start -" - -echo -n "Booting instances... " -# Add in 1 HVM instance if available. -if [[ -z "$HVM" ]]; then - instances=$(ec2-run-instances \ - --user-data "$userdata" \ - --instance-type "t1.micro" \ - --instance-count 3 \ - --group "$sg_name" \ - --key "$key_name" $AMI | \ - grep INSTANCE | cut -f2) -else - instances=$(ec2-run-instances \ - --user-data "$userdata" \ - --instance-type "t1.micro" \ - --instance-count 2 \ - --group "$sg_name" \ - --key "$key_name" $AMI | \ - grep INSTANCE | cut -f2) - instances+=" " - instances+=$(ec2-run-instances \ - --user-data "$userdata" \ - --instance-type "m3.medium" \ - --instance-count 1 \ - --group "$sg_name" \ - --key "$key_name" $HVM | \ - grep INSTANCE | cut -f2) -fi -# little hack to create a describe instances command that only -# pulls data for these instances -ec2_cmd=$(echo $instances | sed 's/ / --filter instance-id=/g') -ec2_cmd="ec2-describe-instances --filter instance-id=$ec2_cmd" - -while $ec2_cmd | grep INSTANCE | grep -q pending - do sleep 10; done - -declare -a ips=($($ec2_cmd | grep INSTANCE | cut -f4)) - -# sleep until all the sockets we need come up -for host in ${ips[@]}; do - for port in 22 4001 7001; do - timeout 600 perl -MIO::Socket::INET -e " - until(new IO::Socket::INET('$host:$port')){sleep 1}" - done -done -echo "OK ($instances)" - -echo "Letting etcd settle..." -sleep 10 - -echo "Running coretest..." -for host in ${ips[@]}; do - if ! ssh -i "$key_file" -l core -o StrictHostKeyChecking=no "$host" \ - coretest -test.v=true -test.parallel=8 - then - echo "coretest failed for $host" >&2 - exit 1 - fi -done -echo "OK" - -echo -n "Testing etcd... " -test_key="v2/keys/test" -token=$(uuidgen) -# XXX: the sleep *should never* be required, this is a bug in etcd -sleep 5 -curl --fail -s -L "${ips[0]}:4001/$test_key" -d value="$token" > /dev/null -sleep 5 -for host in ${ips[@]}; do - if ! curl --fail -s -L "${host}:4001/$test_key" | grep -q $token; then - echo "etcd bootstrap appears to have failed for $host" >&2 - exit 1 - fi -done -echo "OK" - -echo "Checking disk GUID... " -for host in ${ips[@]}; do - if ! ssh -i "$key_file" -l core -o StrictHostKeyChecking=no "$host" \ - sudo sgdisk --print /dev/xvda | \ - grep "^Disk identifier" | \ - grep -v 00000000-0000-0000-0000-000000000001 - then - echo "disk guid unset on $host" >&2 - exit 1 - fi -done -echo "OK" - - -echo -n "Cleaning up environment... " -ec2-terminate-instances $instances > /dev/null -while ! $ec2_cmd | grep INSTANCE | grep -q terminated - do sleep 10; done - -# The security group may take a little longer to free up -while ! ec2-delete-group $sg_name > /dev/null - do sleep 10; done -ec2-delete-keypair $key_name > /dev/null -rm $key_file -echo "OK" diff --git a/oem/ami/update_json.sh b/oem/ami/update_json.sh deleted file mode 100755 index 0ac05a00f3..0000000000 --- a/oem/ami/update_json.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash -# -# Set pipefail along with -e in hopes that we catch more errors -set -e -o pipefail - -DIR=$(dirname $0) -source $DIR/regions.sh - -USAGE="Usage: $0 -V 100.0.0 - -V VERSION Find AMI by CoreOS version. (required) - -b BOARD Set to the board name, default is amd64-usr - -g GROUP Set the update group, default is alpha - -s STORAGE GS URL for Google storage to upload to. - -h this ;-) - -v Verbose, see all the things! - -This script must be run from an ec2 host with the ec2 tools installed. -" - -IMAGE="coreos_production_ami" -GS_URL="gs://builds.release.core-os.net" -AMI= -VER= -BOARD="amd64-usr" -GROUP="alpha" - -clean_version() { - sed -e 's%[^A-Za-z0-9()\\./_-]%_%g' <<< "$1" -} - -while getopts "V:b:g:s:hv" OPTION -do - case $OPTION in - V) VER="$OPTARG";; - b) BOARD="$OPTARG";; - g) GROUP="$OPTARG";; - s) GS_URL="$OPTARG";; - h) echo "$USAGE"; exit;; - v) set -x;; - *) exit 1;; - esac -done - -if [[ $(id -u) -eq 0 ]]; then - echo "$0: This command should not be ran run as root!" >&2 - exit 1 -fi - -if [[ ! -n "$VER" ]]; then - echo "$0: AMI version required via -V" >&2 - echo "$USAGE" >&2 - exit 1 -fi - -search_name=$(clean_version "CoreOS-$GROUP-$VER") -declare -A AMIS HVM_AMIS -for r in "${ALL_REGIONS[@]}"; do - # Hacky but avoids writing an indirection layer to handle auth... - if [[ "${r}" == "us-gov-west-1" ]]; then - source $DIR/ami-builder-us-gov-auth.sh - else - source $DIR/marineam-auth.sh - fi - - AMI=$(ec2-describe-images --region=${r} -F name="${search_name}" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$AMI" ]]; then - echo "$0: Cannot find an AMI named ${search_name} in ${r}" >&2 - exit 1 - fi - AMIS[${r}]=$AMI - HVM=$(ec2-describe-images --region=${r} -F name="${search_name}-hvm" \ - | grep -m1 ^IMAGE | cut -f2) || true - if [[ -z "$HVM" ]]; then - echo "$0: Cannot find an AMI named ${search_name}-hvm in ${r}" >&2 - exit 1 - fi - HVM_AMIS[${r}]=$HVM -done - -# ignore this crap: /usr/lib64/python2.6/site-packages/Crypto/Util/number.py:57: PowmInsecureWarning: Not using mpz_powm_sec. You should rebuild using libgmp >= 5 to avoid timing attack vulnerability. -upload_file() { - local name="$1" - local content="$2" - url="$GS_URL/$GROUP/boards/$BOARD/$VER/${IMAGE}_${name}" - echo -e "$content" \ - | python -W "ignore:Not using mpz_powm_sec" \ - `which gsutil` cp - "$url" - echo "OK, ${url}=${content}" -} - -publish_ami() { - local r="$1" - local virt_type="$2" - local r_amiid="$3" - - # compatibility name from before addition of hvm - if [[ "${virt_type}" == "pv" ]]; then - upload_file "${r}.txt" "$r_amiid" - fi - - upload_file "${virt_type}_${r}.txt" "$r_amiid" -} - -PV_ALL="" -for r in "${!AMIS[@]}"; do - publish_ami "$r" pv "${AMIS[$r]}" - PV_ALL+="|${r}=${AMIS[$r]}" -done -PV_ALL="${PV_ALL#|}" - -HVM_ALL="" -for r in "${!HVM_AMIS[@]}"; do - publish_ami "$r" hvm "${HVM_AMIS[$r]}" - HVM_ALL+="|${r}=${HVM_AMIS[$r]}" -done -HVM_ALL="${HVM_ALL#|}" - -AMI_ALL="{\n \"amis\": [" -for r in "${ALL_REGIONS[@]}"; do - AMI_ALL+="\n {" - AMI_ALL+="\n \"name\": \"${r}\"," - AMI_ALL+="\n \"pv\": \"${AMIS[$r]}\"," - AMI_ALL+="\n \"hvm\": \"${HVM_AMIS[$r]}\"" - AMI_ALL+="\n }," -done -AMI_ALL="${AMI_ALL%,}" -AMI_ALL+="\n ]\n}" - -upload_file "all.txt" "${PV_ALL}" -upload_file "pv.txt" "${PV_ALL}" -upload_file "hvm.txt" "${HVM_ALL}" -upload_file "all.json" "${AMI_ALL}" -echo "Done" diff --git a/oem/ami/user.sh b/oem/ami/user.sh deleted file mode 100755 index f825a8220f..0000000000 --- a/oem/ami/user.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -DIR=/home/ec2-user/scripts/oem/ami -USER=someone -TYPE=production -VERSION="367.0.0+2014-07-10-1613" -URL="http://users.developer.core-os.net/${USER}/boards/amd64-usr/${VERSION}" - -set -e -eval $(curl -f "${URL}/version.txt") - -source $DIR/marineam-auth.sh -args="-b amd64-usr -g ${USER} -V ${VERSION}" -$DIR/import.sh ${args} -u "${URL}/coreos_${TYPE}_ami_image.bin.bz2" -$DIR/test_ami.sh -v ${args} -#$DIR/copy_ami.sh ${args}