Merge pull request #483 from philips/gce-python

Google Compute Engine agent

sdk_container/src/third_party/coreos-overlay/app-emulation/google-compute-daemon/files/0001-Fixes-authorized_keys-file-permissions.patch

@@ -0,0 +1,40 @@
+From 6b9c8b8ae50dd3530c80fdd26fbeef7fc9ae8dff Mon Sep 17 00:00:00 2001
+From: Ferran Rodenas <frodenas@gmail.com>
+Date: Tue, 18 Feb 2014 00:21:17 -0800
+Subject: [PATCH] Fixes authorized_keys file permissions
+
+Owner and group id are not inherit when calling shutil move. This commit
+moves setting permissions logic for the authorized_keys file to be sure
+that the final destination file has the proper permissions.
+---
+ google-daemon/usr/share/google/google_daemon/accounts.py | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/google-daemon/usr/share/google/google_daemon/accounts.py b/google-daemon/usr/share/google/google_daemon/accounts.py
+index 4c572ca..6f970c4 100755
+--- a/google-daemon/usr/share/google/google_daemon/accounts.py
++++ b/google-daemon/usr/share/google/google_daemon/accounts.py
+@@ -378,10 +378,6 @@ class Accounts(object):
+       user_lines = [
+           lines[i] for i in range(len(lines)) if i not in google_added_ixs]
+ 
+-      # Make sure the keys_file has the right perms (u+rw).
+-      self.os.fchmod(keys_file.fileno(), 0600)
+-      self.os.fchown(keys_file.fileno(), uid, gid)
+-
+       # First write user's entries.
+       for user_line in user_lines:
+         keys_file.write(EnsureTrailingNewline(user_line))
+@@ -394,5 +390,9 @@ class Accounts(object):
+     # Override the old authorized keys file with the new one.
+     self.system.MoveFile(new_keys_path, authorized_keys_file)
+ 
++    # Make sure the authorized_keys_file has the right perms (u+rw).
++    self.os.chmod(authorized_keys_file, 0600)
++    self.os.chown(authorized_keys_file, uid, gid)
++
+     # Set SELinux context, if applicable to this system
+     self.SetSELinuxContext(authorized_keys_file)
+-- 
+1.8.5.2 (Apple Git-48)
+

sdk_container/src/third_party/coreos-overlay/app-emulation/google-compute-daemon/files/0001-fix-google-daemon-use-for-the-passwd-not.patch

@@ -0,0 +1,29 @@
+From 0243ab89cd941f15344ef03b63c96f3f0e1ffa45 Mon Sep 17 00:00:00 2001
+From: Brandon Philips <brandon@ifup.co>
+Date: Tue, 1 Apr 2014 11:51:13 -0700
+Subject: [PATCH] fix(google-daemon): use * for the passwd not !
+
+useradd defaults to using ! as the "locked" password marker.
+Unfortunatly, openssh interprets this to mean that it shouldn't let the
+user in via ssh if PAM is missing. Work around this by using the *
+marker which also means locked but is allowed by openssh.
+---
+ google-daemon/usr/share/google/google_daemon/utils.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/google-daemon/usr/share/google/google_daemon/utils.py b/google-daemon/usr/share/google/google_daemon/utils.py
+index 0f17f34..371034d 100755
+--- a/google-daemon/usr/share/google/google_daemon/utils.py
++++ b/google-daemon/usr/share/google/google_daemon/utils.py
+@@ -69,7 +69,7 @@ class System(object):
+   def UserAdd(self, user, groups):
+     logging.info('Creating account %s', user)
+     result = self.RunCommand([
+-        '/usr/sbin/useradd', user, '-m', '-s', '/bin/bash', '-G',
++        '/usr/sbin/useradd', user, '-m', '-s', '/bin/bash', '-p', '*', '-G',
+         ','.join(groups)])
+     if self.RunCommandFailed(result, 'Could not create user %s', user):
+       return False
+-- 
+1.8.5.2 (Apple Git-48)
+

sdk_container/src/third_party/coreos-overlay/app-emulation/google-compute-daemon/files/0001-hack-address_manager-use-CoreOS-names-and-locations.patch

@@ -0,0 +1,92 @@
+From 265f951bc8d55bfb988050dda6332511b58e72e8 Mon Sep 17 00:00:00 2001
+From: Brandon Philips <brandon@ifup.co>
+Date: Tue, 1 Apr 2014 15:18:20 -0700
+Subject: [PATCH] hack(address_manager): use CoreOS names and locations
+
+ip is in /usr/bin/ip and the network interface is ens4v1 via
+http://www.freedesktop.org/wiki/Software/systemd/PredictableNetworkInterfaceNames/
+---
+ .../share/google/google_daemon/address_manager.py  | 24 +++++++++++-----------
+ 1 file changed, 12 insertions(+), 12 deletions(-)
+
+diff --git a/google-daemon/usr/share/google/google_daemon/address_manager.py b/google-daemon/usr/share/google/google_daemon/address_manager.py
+index 1b3997c..568720d 100644
+--- a/google-daemon/usr/share/google/google_daemon/address_manager.py
++++ b/google-daemon/usr/share/google/google_daemon/address_manager.py
+@@ -15,16 +15,16 @@
+ """Manage extra network interface addresses on a GCE instance.
+ 
+ Fetch a list of public endpoint IPs from the metadata server, compare it with
+-what's configured on eth0, and add/remove addresses from eth0 to make them
++what's configured on ens4v1, and add/remove addresses from ens4v1 to make them
+ match.  Only remove those which match our proto code.
+ 
+ This must be run by root. If it reads any malformed data, it will take no
+ action.
+ 
+ Command used to add ips:
+-  ip route add to local $IP/32 dev eth0 proto 66
++  ip route add to local $IP/32 dev ens4v1 proto 66
+ Command used to fetch list of configured IPs:
+-  ip route ls table local type local dev eth0 scope host proto 66
++  ip route ls table local type local dev ens4v1 scope host proto 66
+ """
+ 
+ 
+@@ -103,8 +103,8 @@ class AddressManager(object):
+     return self.ParseIPAddrs(addrs_data)
+ 
+   def ReadLocalConfiguredAddrs(self):
+-    """Fetch list of addresses we've configured on eth0 already."""
+-    cmd = ('/sbin/ip route ls table local type local dev eth0 scope host ' +
++    """Fetch list of addresses we've configured on ens4v1 already."""
++    cmd = ('/usr/bin/ip route ls table local type local dev ens4v1 scope host ' +
+            'proto %d' % GOOGLE_PROTO_ID)
+     result = self.system.RunCommand(cmd.split())
+     if self.IPCommandFailed(result, cmd):
+@@ -132,27 +132,27 @@ class AddressManager(object):
+             to_remove or None))
+ 
+   def AddAddresses(self, to_add):
+-    """Configure new addresses on eth0."""
++    """Configure new addresses on ens4v1."""
+     for addr in to_add:
+        self.AddOneAddress(addr)
+ 
+   def AddOneAddress(self, addr):
+-    """Configure one address on eth0."""
+-    cmd = '/sbin/ip route add to local %s/32 dev eth0 proto %d' % (
++    """Configure one address on ens4v1."""
++    cmd = '/usr/bin/ip route add to local %s/32 dev ens4v1 proto %d' % (
+         addr, GOOGLE_PROTO_ID)
+     result = self.system.RunCommand(cmd.split())
+     self.IPCommandFailed(result, cmd)  # Ignore return code
+ 
+   def DeleteAddresses(self, to_remove):
+-    """Un-configure a list of addresses from eth0."""
++    """Un-configure a list of addresses from ens4v1."""
+     for addr in to_remove:
+       self.DeleteOneAddress(addr)
+ 
+   def DeleteOneAddress(self, addr):
+-    """Delete one address from eth0."""
++    """Delete one address from ens4v1."""
+     # This will fail if it doesn't match exactly the specs listed.
+     # That'll help ensure we don't remove one added by someone else.
+-    cmd = '/sbin/ip route delete to local %s/32 dev eth0 proto %d' % (
++    cmd = '/usr/bin/ip route delete to local %s/32 dev ens4v1 proto %d' % (
+         addr, GOOGLE_PROTO_ID)
+     result = self.system.RunCommand(cmd.split())
+     self.IPCommandFailed(result, cmd)  # Ignore return code
+@@ -168,7 +168,7 @@ class AddressManager(object):
+     return addrs
+ 
+   def IPCommandFailed(self, result, cmd):
+-    """If an /sbin/ip command failed, log and return True."""
++    """If an /usr/bin/ip command failed, log and return True."""
+     if self.system.RunCommandFailed(
+         result, 'Non-zero exit status from: "%s"' % cmd):
+       return True
+-- 
+1.8.5.2 (Apple Git-48)
+

sdk_container/src/third_party/coreos-overlay/app-emulation/google-compute-daemon/google-compute-daemon-1.1.1.ebuild

@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2014 CoreOS, Inc. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+#
+
+EAPI=5
+inherit toolchain-funcs systemd
+
+DESCRIPTION="Google Daemon for Compute Engine"
+HOMEPAGE="https://github.com/GoogleCloudPlatform/compute-image-packages"
+SRC_URI="https://github.com/GoogleCloudPlatform/compute-image-packages/releases/download/${PV}/google-daemon-${PV}.tar.gz"
+
+LICENSE="MIT"
+SLOT="0"
+KEYWORDS="amd64 x86"
+IUSE=""
+
+RDEPEND="dev-lang/python-oem"
+
+S="${WORKDIR}"
+
+src_prepare() {
+	epatch "${FILESDIR}"/0001-Fixes-authorized_keys-file-permissions.patch
+	epatch "${FILESDIR}"/0001-fix-google-daemon-use-for-the-passwd-not.patch
+	epatch "${FILESDIR}"/0001-hack-address_manager-use-CoreOS-names-and-locations.patch
+}
+
+src_install() {
+	mkdir -p ${D}/usr/share/oem/google-compute-daemon/
+	cp -Ra ${WORKDIR}/usr/share/google/google_daemon/. ${D}/usr/share/oem/google-compute-daemon/ || die
+}

sdk_container/src/third_party/coreos-overlay/app-emulation/google-startup-scripts/files/0001-fix-google-startup-scripts-use-GOOGLE_STARTUP_SCRIPT.patch

@@ -0,0 +1,63 @@
+From 2748f99730e699b2597653ddef1c4109cae976b2 Mon Sep 17 00:00:00 2001
+From: Brandon Philips <brandon@ifup.co>
+Date: Tue, 1 Apr 2014 16:35:28 -0700
+Subject: [PATCH] fix(google-startup-scripts): use
+ ${GOOGLE_STARTUP_SCRIPTS_PREFIX} in onboot
+
+We can't install these scripts into /usr/share/google. So, take a PREFIX
+environment variable.
+---
+ google-startup-scripts/usr/share/google/onboot | 14 +++++++-------
+ 1 file changed, 7 insertions(+), 7 deletions(-)
+
+diff --git a/google-startup-scripts/usr/share/google/onboot b/google-startup-scripts/usr/share/google/onboot
+index d740d07..f65b5ab 100755
+--- a/google-startup-scripts/usr/share/google/onboot
++++ b/google-startup-scripts/usr/share/google/onboot
+@@ -33,7 +33,7 @@ else
+     declare -r LOGGER=/bin/logger
+ fi
+ 
+-declare -r BOTO_SETUP_SCRIPT=/usr/share/google/boto/boot_setup.py
++declare -r BOTO_SETUP_SCRIPT=${GOOGLE_STARTUP_SCRIPTS_PREFIX}/boto/boot_setup.py
+ 
+ declare -r CURL_RETRY_LIMIT=10
+ declare -r CURL_TIMEOUT=10
+@@ -105,13 +105,13 @@ function download_url() {
+ 
+ function get_metadata_value() {
+   local readonly varname=$1
+-  /usr/share/google/get_metadata_value ${varname}
++  ${GOOGLE_STARTUP_SCRIPTS_PREFIX}/get_metadata_value ${varname}
+   return $?
+ }
+ 
+ function get_metadata_attribute() {
+   local readonly varname=$1
+-  /usr/share/google/get_metadata_value "attributes/${varname}"
++  ${GOOGLE_STARTUP_SCRIPTS_PREFIX}/get_metadata_value "attributes/${varname}"
+   return $?
+ }
+ 
+@@ -129,14 +129,14 @@ function run_command_with_retry() {
+ }
+ 
+ function first_boot() {
+-  if [[ -x /usr/share/google/first-boot ]]; then
+-    /usr/share/google/first-boot
++  if [[ -x ${GOOGLE_STARTUP_SCRIPTS_PREFIX}/first-boot ]]; then
++    ${GOOGLE_STARTUP_SCRIPTS_PREFIX}/first-boot
+   fi
+ }
+ 
+ function virtionet_irq_affinity() {
+-  if [[ -x /usr/share/google/virtionet-irq-affinity ]]; then
+-    /usr/share/google/virtionet-irq-affinity
++  if [[ -x ${GOOGLE_STARTUP_SCRIPTS_PREFIX}/virtionet-irq-affinity ]]; then
++    ${GOOGLE_STARTUP_SCRIPTS_PREFIX}/virtionet-irq-affinity
+   fi
+ }
+ 
+-- 
+1.8.5.2 (Apple Git-48)
+

sdk_container/src/third_party/coreos-overlay/app-emulation/google-startup-scripts/google-startup-scripts-1.1.1.ebuild

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2014 CoreOS, Inc. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+#
+
+EAPI=5
+inherit toolchain-funcs systemd
+
+DESCRIPTION="Google Startup Scripts for Compute Engine"
+HOMEPAGE="https://github.com/GoogleCloudPlatform/compute-image-packages"
+SRC_URI="https://github.com/GoogleCloudPlatform/compute-image-packages/releases/download/${PV}/${P}.tar.gz"
+
+LICENSE="MIT"
+SLOT="0"
+KEYWORDS="amd64 x86"
+IUSE=""
+
+RDEPEND="dev-lang/python-oem"
+
+S="${WORKDIR}"
+
+src_prepare() {
+	epatch "${FILESDIR}"/0001-fix-google-startup-scripts-use-GOOGLE_STARTUP_SCRIPT.patch
+}
+
+src_install() {
+	mkdir -p ${D}/usr/share/oem/google-startup-scripts/usr/share/google/
+	cp -Ra ${WORKDIR}/usr/share/google/. ${D}/usr/share/oem/google-startup-scripts/ || die
+}

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-gce/files/cloud-config.yml

@@ -2,6 +2,15 @@
 
 coreos:
     units:
+      - name: gce-add-metadata-host.service
+        command: start
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Setup metadata in /etc/hosts
+
+          [Service]
+          ExecStart=/usr/share/oem/bin/gce-add-metadata-host
       - name: gce-coreos-cloudinit.service
         command: restart
         runtime: yes
@@ -11,6 +20,7 @@ coreos:
           After=gce-setup-environment.service
 
           [Service]
+          Type=oneshot
           EnvironmentFile=/etc/environment
           ExecStart=/usr/share/oem/bin/gce-coreos-cloudinit
       - name: gce-setup-environment.service
@@ -35,3 +45,50 @@ coreos:
           Type=oneshot
           StandardOutput=journal+console
           ExecStart=/usr/share/oem/bin/gce-ssh-key
+      - name: google-accounts-manager.service
+        command: start
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Google Compute Engine User Accounts Manager Daemon
+          After=network.target
+          Before=sshd.service
+          Requires=network.target
+ 
+          [Service]
+          Type=simple
+          ExecStart=/usr/share/oem/python/bin/python2.7 /usr/share/oem/google-compute-daemon/manage_accounts.py
+
+          [Install]
+          WantedBy=multi-user.target
+      - name: google-address-manager.service
+        command: start
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Google Compute Engine Address Manager Daemon
+          After=network.target
+          Requires=network.target
+
+          [Service]
+          Type=simple
+          ExecStart=/usr/share/oem/python/bin/python2.7 /usr/share/oem/google-compute-daemon/manage_addresses.py
+
+          [Install]
+          WantedBy=multi-user.target
+      - name: google-startup-scripts.service
+        command: start
+        runtime: yes
+        content: |
+          [Unit]
+          Description=Google Compute Engine Startup Scripts
+          After=network.target
+          Requires=network.target
+
+          [Service]
+          Type=oneshot
+          Environment=GOOGLE_STARTUP_SCRIPTS_PREFIX=/usr/share/oem/google-startup-scripts
+          ExecStart=/usr/share/oem/google-startup-scripts/onboot
+
+          [Install]
+          WantedBy=multi-user.target

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-gce/files/gce-add-metadata-host

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+entry="169.254.169.254 metadata"
+
+grep -q "${entry}" /etc/hosts
+
+if [ $? -ne 0 ]; then
+	echo ${entry} >> /etc/hosts
+fi

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-gce/oem-gce-0.0.2-r2.ebuild

@@ -15,11 +15,17 @@ IUSE=""
 # no source directory
 S="${WORKDIR}"
 
+RDEPEND="
+	app-emulation/google-compute-daemon
+	app-emulation/google-startup-scripts
+"
+
 src_install() {
 	into "/usr/share/oem"
 	dobin ${FILESDIR}/gce-ssh-key
 	dobin ${FILESDIR}/gce-coreos-cloudinit
 	dobin ${FILESDIR}/gce-setup-environment
+	dobin ${FILESDIR}/gce-add-metadata-host
 
 	insinto "/usr/share/oem"
 	doins ${FILESDIR}/cloud-config.yml