Merge pull request #425 from polvi/ami-cloudinit2

feat(ami): refactor to use cloudinit

sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-cloudinit/coreos-cloudinit-9999.ebuild

@@ -9,7 +9,7 @@ CROS_WORKON_REPO="git://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64"
 else
-	CROS_WORKON_COMMIT="9bcd56222f08d7d1ecb23686a33238a8e89bded1" 
+	CROS_WORKON_COMMIT="a5e5505b512d22b3b650cd76542ebdaee8155645" 
 	KEYWORDS="amd64"
 fi
 

sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild

@@ -52,7 +52,6 @@ src_install() {
 	systemd_enable_service basic.target coreos-startup.target
 
 	# Services!
-	systemd_enable_service default.target coreos-c10n.service
 	systemd_enable_service default.target coreos-cloudinit-oem.service
 	systemd_enable_service default.target local-enable.service
 	systemd_enable_service default.target sshd-keygen.service

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/cloud-config.yml

@@ -0,0 +1,52 @@
+#cloud-config
+
+coreos:
+    units:
+      - name: ec2-ssh-key.service
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Sets SSH key from metadata
+
+          [Service]
+          Type=oneshot
+          StandardOutput=journal+console
+          ExecStart=/usr/share/oem/usr/bin/ec2-ssh-key
+      - name: ec2-cloudinit.service
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Cloudinit from EC2 metadata
+
+          [Service]
+          Type=oneshot
+          ExecStart=/usr/bin/coreos-cloudinit -from-url=http://169.254.169.254/latest/user-data
+      - name: ec2-c10n.service
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Legacy c10n support from EC2 metadata
+          Requires=ec2-cloudinit.service
+          After=ec2-cloudinit.service
+
+          [Service]
+          Type=oneshot
+          RemainAfterExit=no
+          ExecStart=/usr/share/oem/usr/bin/coreos-c10n
+      - name: etcd.service
+        runtime: yes
+        content: |
+          [Unit]
+          Description=etcd with ec2 auto-bootstrap
+          Requires=ec2-c10n.service
+          After=ec2-c10n.service
+
+          [Service]
+          User=etcd
+          PermissionsStartOnly=true
+          ExecStart=/usr/share/oem/usr/bin/etcd-bootstrap
+          Restart=always
+          RestartSec=10s
+
+          [Install]
+          WantedBy=multi-user.target

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/coreos-c10n

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+depwarn() {
+	DOCS="https://github.com/coreos/coreos-cloudinit#cloud-config"
+	echo "DEPRECATION WARNING! $1 See docs for the new cloud-config format here: $DOCS" 1>&2;
+}
+
+
+C10N_ENDPOINT=https://core-api.appspot.com/v1/c10n/group
+META_URL="http://169.254.169.254/latest"
+
+BOOTSTRAP_PEERS="/var/run/etcd/bootstrap.config"
+BOOTSTRAP_DISCOVERY="/var/run/etcd/bootstrap.disco"
+
+/usr/bin/block-until-url $META_URL
+
+# Attempt to download userdata, exiting on failure or no user-data is found
+USER_DATA=$(curl -s --fail $META_URL/user-data)
+retcode=$?
+
+# exit code 22 is http code 404, meaning user-data is blank
+if [[ $retcode -ne 0 && $retcode -ne 22 ]]; then
+	echo "Unable to communicate with EC2 metadata service"
+	exit 1
+elif [[ $retcode -eq 22 || -z "${USER_DATA}" ]]; then
+	echo "No user-data provided"
+	exit 0
+fi
+
+# Preserve the original user-data contents to use as the c10n token in case
+# a discovery URL is not provided.
+C10N_TOKEN="${USER_DATA}"
+
+# If user-data is an HTTPS url, follow it
+if echo "${USER_DATA}" | grep -q '^https://'; then
+	depwarn "Resolving user-data as a URL is deprecated."
+
+	# Backwards compatibility.  If we have a GitHub gist that doesn't end in /raw, we'll append it to before grabbing the gist
+	if echo "${USER_DATA}" | grep -e '^https://gist.github.com' | grep -v -e 'raw$'; then
+		USER_DATA="${USER_DATA}/raw"
+	fi
+
+	echo "Downloading contents of URL: ${USER_DATA}"
+
+	USER_DATA="$(curl -s $USER_DATA)"
+fi
+
+# Create temporary file that gets cleaned up on exit
+TMP=$(mktemp)
+trap "rm -f ${TMP}" EXIT
+
+# Write user-data to disk for future validation
+echo "${USER_DATA}" > $TMP
+
+# If user-data is a script, execute it
+if  head -n 1 $TMP | grep -q '^#!'; then
+	if grep -q '^ETCD_DISCOVERY_URL=' $TMP; then
+		depwarn "The ETCD_DISCOVERY_URL variable is deprecated."
+		eval $(grep  '^ETCD_DISCOVERY_URL=' $TMP | tail -n1)
+	fi
+
+	if grep -q '^START_FLEET=1' $TMP; then
+		depwarn "The START_FLEET variable is deprecated."
+		START_FLEET=1
+	fi
+
+# If user-data is a cloud-config, hand it off to the new hotness
+elif head -n 1 $TMP | grep -q '^#cloud-config'; then
+	# If an etcd discovery URL was found, we can assume the new hotness
+	# will handle bootstrapping, allowing us to clear the c10n token
+	if grep -q 'discovery_url:' $TMP; then
+		C10N_TOKEN=""
+	fi
+
+# Validate the user-data as an SSH key, installing it if so
+else
+	depwarn "Interpreting user-data as an SSH key is deprecated."
+
+	# This just validates the key
+	ssh-keygen -l -f $TMP > /dev/null 2>&1
+
+	if [ $? -eq 0 ]; then
+		update-ssh-keys -a c10n $TMP
+	fi
+fi
+
+# After all that, if a script happened to set the discovery URL, write it to disk for etcd-bootstrap
+if [ -n "${ETCD_DISCOVERY_URL}"  ]; then
+	echo "Using '${ETCD_DISCOVERY_URL}' as etcd discovery url"
+	echo "${ETCD_DISCOVERY_URL}" > $BOOTSTRAP_DISCOVERY
+# ...otherwise, we treat the provided user-data as a legacy c10n URL
+elif [ -n "${C10N_TOKEN}" ]; then
+	depwarn "Bootstrapping etcd with the c10n service is deprecated."
+
+	echo "Using '$C10N_TOKEN' as c10n token"
+
+	# Assert we have networking up and able to access the c10n service
+	/usr/bin/block-until-url $C10N_ENDPOINT
+
+	IP=$(curl -s $META_URL/meta-data/local-ipv4)
+	curl -s $C10N_ENDPOINT -d "c10n_url=$C10N_TOKEN" -d"ip_list=$IP" > $BOOTSTRAP_PEERS
+fi
+
+if [ "$START_FLEET" = "1" ]; then
+	echo "Starting fleet"
+	systemctl start --no-block fleet
+fi

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/coreos-setup-environment

@@ -0,0 +1,23 @@
+#!/bin/bash +x
+
+ENV=$1
+
+if [ -z "$ENV" ]; then
+        echo usage: $0 /etc/environment
+        exit 1
+fi
+# test for rw
+touch $ENV
+if [ $? -ne 0 ]; then
+        echo exiting, unable to modify: $ENV
+        exit 1
+fi
+
+# get public ip from metadata
+PUB_IP_URL=http://169.254.169.254/latest/meta-data/public-ipv4
+IP=$(/usr/bin/curl --fail -s "$PUB_IP_URL")
+echo COREOS_PUBLIC_IPV4=${IP} >> $ENV
+
+PRIV_IP_URL=http://169.254.169.254/latest/meta-data/local-ipv4
+IP=$(/usr/bin/curl --fail -s "$PRIV_IP_URL")
+echo COREOS_PRIVATE_IPV4=${IP} >> $ENV

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/ec2-ssh-key

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+
+KEY_URL="http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key"
+
+/usr/bin/block-until-url "$KEY_URL"
+/usr/bin/curl --fail -s "$KEY_URL" | /usr/bin/update-ssh-keys -a ec2

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/etcd-bootstrap

@@ -3,16 +3,10 @@
 #TODO: Generating a config file rather than args so etcd-bootstrap can run as a
 #      completely separate unit from etcd
 
-VIRT=$(coreos-detect-virt)
 STATE=/var/lib/etcd
 ARGS="-f -data-dir $STATE -bind-addr 0.0.0.0"
 
-if [ "${VIRT}" != "ec2" ]; then
-	echo "Detected environment \"${VIRT}\", just starting solo master..."
-	exec /usr/bin/etcd ${ARGS} -n ${HOSTNAME}
-else
-	ARGS="${ARGS} -peer-election-timeout 1200"
-fi
+ARGS="${ARGS} -peer-election-timeout 1200"
 
 BOOTSTRAP_PEERS="/var/run/etcd/bootstrap.config"
 BOOTSTRAP_DISCO="/var/run/etcd/bootstrap.disco"

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/oem-release

@@ -1 +0,0 @@
-ec2

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/run

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-set -e
-
-KEY_URL="http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key"
-
-block-until-url "$KEY_URL"
-curl --fail -s "$KEY_URL" | update-ssh-keys -a ec2

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/oem-ami-0.0.1.ebuild

@@ -16,9 +16,12 @@ IUSE=""
 S="${WORKDIR}"
 
 src_install() {
-	exeinto "/"
-	doexe ${FILESDIR}/run
+	into "/usr"
+	dobin ${FILESDIR}/ec2-ssh-key
+	dobin ${FILESDIR}/coreos-setup-environment
+	dobin ${FILESDIR}/coreos-c10n
+	dobin ${FILESDIR}/etcd-bootstrap
 
 	insinto "/"
-	doins ${FILESDIR}/oem-release
+	doins ${FILESDIR}/cloud-config.yml
 }

sdk_container/src/third_party/coreos-overlay/dev-db/etcd/etcd-0.3.0-r5.ebuild

@@ -29,8 +29,6 @@ src_compile() {
 
 src_install() {
 	dobin ${S}/bin/${PN}
-	dobin ${FILESDIR}/etcd-bootstrap
-	dobin ${FILESDIR}/etcd-pre-exec
 
 	systemd_dounit "${FILESDIR}"/${PN}.service
 	systemd_enable_service multi-user.target ${PN}.service

sdk_container/src/third_party/coreos-overlay/dev-db/etcd/etcd-99999.ebuild

@@ -28,8 +28,6 @@ src_compile() {
 
 src_install() {
 	dobin ${S}/bin/${PN}
-	dobin ${FILESDIR}/etcd-bootstrap
-	dobin ${FILESDIR}/etcd-pre-exec
 
 	systemd_dounit "${FILESDIR}"/${PN}.service
 	systemd_enable_service multi-user.target ${PN}.service

sdk_container/src/third_party/coreos-overlay/dev-db/etcd/files/etcd-pre-exec

@@ -1,11 +0,0 @@
-#!/bin/sh -e
-
-# Fixup the old state directory
-OLD_STATE=/media/state/etcd
-STATE=/var/lib/etcd
-
-if [ -d /media/state/etcd ]; then
-  cp -R ${OLD_STATE}/. ${STATE}
-  rm -R ${OLD_STATE}
-  chown -R etcd:etcd ${STATE}
-fi

sdk_container/src/third_party/coreos-overlay/dev-db/etcd/files/etcd.service

@@ -1,13 +1,14 @@
 [Unit]
 Description=etcd
-Requires=coreos-c10n.service
-After=coreos-c10n.service
+RequiresMountsFor=/usr/share/oem
+# oem will setup etcd for auto bootstrapping
+ConditionPathExists=!/usr/share/oem/cloud-config.yml
 
 [Service]
 User=etcd
 PermissionsStartOnly=true
-ExecStartPre=/usr/bin/etcd-pre-exec
-ExecStart=/usr/bin/etcd-bootstrap
+# Default, this will be overwritten with a cloud-config
+ExecStart=/usr/bin/etcd -f -data-dir /var/lib/etcd -bind-addr 0.0.0.0
 
 Restart=always
 # Set a longish timeout in case this machine isn't behaving