Merge pull request #1083 from flatcar/tormath1/oem

sysext: port AWS OEM to systemd sysext image

build_library/vm_image_util.sh

@@ -221,11 +221,13 @@ IMG_parallels_DISK_LAYOUT=vm
 IMG_parallels_CONF_FORMAT=pvs
 
 ## ami
-IMG_ami_OEM_PACKAGE=oem-ec2-compat
-IMG_ami_OEM_USE=ec2
 IMG_ami_vmdk_DISK_FORMAT=vmdk_stream
-IMG_ami_vmdk_OEM_PACKAGE=oem-ec2-compat
+IMG_ami_vmdk_OEM_USE=ami
-IMG_ami_vmdk_OEM_USE=ec2
+IMG_ami_vmdk_OEM_PACKAGE=common-oem-files
+IMG_ami_vmdk_SYSEXT=oem-ami
+IMG_ami_OEM_USE=ami
+IMG_ami_OEM_PACKAGE=common-oem-files
+IMG_ami_OEM_SYSEXT=oem-ami
 
 ## openstack, supports ec2's metadata format so use oem-ec2-compat
 IMG_openstack_DISK_FORMAT=qcow2

changelog/changes/2023-08-30-sysext-for-aws-oem.md

@@ -0,0 +1 @@
+- AWS OEM images now use a systemd-sysext image for layering additional platform-specific software on top of `/usr`

sdk_container/src/third_party/coreos-overlay/app-emulation/amazon-ssm-agent/amazon-ssm-agent-3.2.985.0-r2.ebuild

@@ -5,7 +5,7 @@ EAPI=7
 COREOS_GO_PACKAGE="${GITHUB_URI}"
 COREOS_GO_VERSION="go1.19"
 
-inherit coreos-go-depend golang-vcs-snapshot
+inherit coreos-go-depend golang-vcs-snapshot systemd
 
 EGO_PN="github.com/aws/${PN}"
 DESCRIPTION="AWS Systems Manager Agent"
@@ -48,10 +48,10 @@ src_compile() {
 }
 
 src_install() {
-	into "/oem"
 	dobin bin/amazon-ssm-agent bin/ssm-cli bin/ssm-document-worker bin/ssm-session-logger bin/ssm-session-worker
-	# files used by ignition on a first run
+	insinto "/usr/share/amazon/ssm"
-	insinto "/oem/ssm"
 	newins seelog_unix.xml seelog.xml.template
 	doins amazon-ssm-agent.json.template
+
+	systemd_dounit packaging/linux/amazon-ssm-agent.service
 }

sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r1.ebuild

@@ -4,6 +4,7 @@
 EAPI=8
 
 OEMIDS=(
+    ami
     azure
     qemu
     vmware
@@ -28,7 +29,7 @@ BDEPEND="
 "
 
 src_compile() {
-    local oemid package ebuild version name homepage lines
+    local oemid package ebuild version name homepage lines oemid_cmdline
 
     for oemid in "${OEMIDS[@]}"; do
         if use "${oemid}"; then break; fi
@@ -69,10 +70,19 @@ src_compile() {
         fi
     } >"${T}/oem-release"
 
+    oemid_cmdline="${oemid}"
+
+    # In this specific case, the OEM ID from the oem-release file ('ami')
+    # is different from the OEM ID kernel command line parameter ('ec2')
+    # because some services like Afterburn or Ignition expects 'ec2|aws' value.
+    if [[ "${oemid}" == "ami" ]]; then
+        oemid_cmdline="ec2"
+    fi
+
     lines=(
         '# Flatcar GRUB settings'
         ''
-        "set oem_id=\"${oemid}\""
+        "set oem_id=\"${oemid_cmdline}\""
     )
     {
         printf '%s\n' "${lines[@]}"

sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/ami/grub.cfg.frag

@@ -1,6 +1,4 @@
-# Flatcar GRUB settings for EC2
+# Flatcar GRUB settings for AMI
-
-set oem_id="ec2"
 
 # Blacklist the Xen framebuffer module so it doesn't get loaded at boot
 # Disable `ens3` style names, so eth0 is used for both ixgbevf or xen.

sdk_container/src/third_party/coreos-overlay/coreos-base/flatcar-eks/files/bootstrap.patch

@@ -1,6 +1,6 @@
---- orig/bootstrap.sh	2021-02-02 14:04:27.121358890 +0100
+--- orig/bootstrap.sh
-+++ flatcar/bootstrap.sh	2021-02-02 14:07:15.175175277 +0100
++++ flatcar/bootstrap.sh
-@@ -268,6 +268,9 @@
+@@ -268,6 +268,9 @@ if [ -z "$CLUSTER_NAME" ]; then
      exit  1
  fi
  
@@ -10,77 +10,75 @@
  
  TOKEN=$(get_token)
  AWS_DEFAULT_REGION=$(get_meta_data 'latest/dynamic/instance-identity/document' | jq .region -r)
-@@ -284,7 +287,8 @@
+@@ -285,6 +288,8 @@ PAUSE_CONTAINER="$PAUSE_CONTAINER_IMAGE:$PAUSE_CONTAINER_VERSION"
- PAUSE_CONTAINER="$PAUSE_CONTAINER_IMAGE:$PAUSE_CONTAINER_VERSION"
  
  ### kubelet kubeconfig
--
+ 
 +shopt -s expand_aliases
 +alias aws="docker run --rm --network host amazon/aws-cli"
  CA_CERTIFICATE_DIRECTORY=/etc/kubernetes/pki
  CA_CERTIFICATE_FILE_PATH=$CA_CERTIFICATE_DIRECTORY/ca.crt
  mkdir -p $CA_CERTIFICATE_DIRECTORY
-@@ -324,9 +328,9 @@
+@@ -324,9 +329,9 @@ fi
  
  echo $B64_CLUSTER_CA | base64 -d > $CA_CERTIFICATE_FILE_PATH
  
 -sed -i s,CLUSTER_NAME,$CLUSTER_NAME,g /var/lib/kubelet/kubeconfig
 -sed -i s,MASTER_ENDPOINT,$APISERVER_ENDPOINT,g /var/lib/kubelet/kubeconfig
 -sed -i s,AWS_REGION,$AWS_DEFAULT_REGION,g /var/lib/kubelet/kubeconfig
-+sed -i s,CLUSTER_NAME,$CLUSTER_NAME,g /oem/eks/kubelet-kubeconfig
++sed -i s,CLUSTER_NAME,$CLUSTER_NAME,g /usr/share/amazon/eks/kubelet-kubeconfig
-+sed -i s,MASTER_ENDPOINT,$APISERVER_ENDPOINT,g /oem/eks/kubelet-kubeconfig
++sed -i s,MASTER_ENDPOINT,$APISERVER_ENDPOINT,g /usr/share/amazon/eks/kubelet-kubeconfig
-+sed -i s,AWS_REGION,$AWS_DEFAULT_REGION,g /oem/eks/kubelet-kubeconfig
++sed -i s,AWS_REGION,$AWS_DEFAULT_REGION,g /usr/share/amazon/eks/kubelet-kubeconfig
  ### kubelet.service configuration
  
  if [[ -z "${DNS_CLUSTER_IP}" ]]; then
-@@ -345,7 +349,7 @@
+@@ -345,7 +350,7 @@ else
    DNS_CLUSTER_IP="${DNS_CLUSTER_IP}"
  fi
  
 -KUBELET_CONFIG=/etc/kubernetes/kubelet/kubelet-config.json
-+KUBELET_CONFIG=/oem/eks/kubelet-config.json
++KUBELET_CONFIG=/usr/share/amazon/eks/kubelet-config.json
  echo "$(jq ".clusterDNS=[\"$DNS_CLUSTER_IP\"]" $KUBELET_CONFIG)" > $KUBELET_CONFIG
  
  INTERNAL_IP=$(get_meta_data 'latest/meta-data/local-ipv4')
-@@ -357,7 +361,7 @@
+@@ -357,7 +362,7 @@ INSTANCE_TYPE=$(get_meta_data 'latest/meta-data/instance-type')
  # with this formula when scheduling pods: Allocatable = Capacity - Reserved - Eviction Threshold.
  
  #calculate the max number of pods per instance type
 -MAX_PODS_FILE="/etc/eks/eni-max-pods.txt"
-+MAX_PODS_FILE="/oem/eks/eni-max-pods.txt"
++MAX_PODS_FILE="/usr/share/amazon/eks/eni-max-pods.txt"
  set +o pipefail
  MAX_PODS=$(cat $MAX_PODS_FILE | awk "/^${INSTANCE_TYPE:-unset}/"' { print $2 }')
  set -o pipefail
-@@ -382,6 +386,8 @@
+@@ -382,6 +387,8 @@ if [[ "$USE_MAX_PODS" = "true" ]]; then
      fi
  fi
  
-+cp /oem/eks/kubelet.service /etc/systemd/system/
++cp /usr/share/amazon/eks/kubelet.service /etc/systemd/system/
 +
  mkdir -p /etc/systemd/system/kubelet.service.d
  
  cat <<EOF > /etc/systemd/system/kubelet.service.d/10-kubelet-args.conf
-@@ -396,10 +402,16 @@
+@@ -397,9 +404,14 @@ EOF
- EOF
  fi
  
-+
  # Replace with custom docker config contents.
 +mkdir -p /etc/docker
  if [[ -n "$DOCKER_CONFIG_JSON" ]]; then
      echo "$DOCKER_CONFIG_JSON" > /etc/docker/daemon.json
      systemctl restart docker
 +else
-+    # Copy the docker config shipped in /oem
++    # Copy the docker config shipped in /usr/share/amazon/eks
-+    cp /oem/eks/docker-daemon.json /etc/docker/daemon.json
++    cp /usr/share/amazon/eks/docker-daemon.json /etc/docker/daemon.json
 +    systemctl restart docker
  fi
  
  if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then
-@@ -409,7 +421,19 @@
+@@ -408,8 +420,20 @@ if [[ "$ENABLE_DOCKER_BRIDGE" = "true" ]]; then
+     echo "$(jq '.bridge="docker0" | ."live-restore"=false' /etc/docker/daemon.json)" > /etc/docker/daemon.json
      systemctl restart docker
  fi
- 
++ 
 +# sysctl tweaking
 +cat <<EOF | sudo tee -a /etc/sysctl.d/99-kubelet.conf
 +# Needed for protectKernelDefaults=true
@@ -91,7 +89,7 @@
 +fs.inotify.max_user_watches=524288
 +vm.max_map_count=524288
 +EOF
-+
+ 
  systemctl daemon-reload
 +systemctl restart systemd-sysctl
  systemctl enable kubelet

sdk_container/src/third_party/coreos-overlay/coreos-base/flatcar-eks/files/kubelet.service

@@ -7,12 +7,12 @@ After=docker.service iptables-restore.service
 
 [Service]
 EnvironmentFile=/run/metadata/flatcar
-ExecStartPre=/oem/eks/download-kubelet.sh
+ExecStartPre=/usr/share/amazon/eks/download-kubelet.sh
 ExecStartPre=/sbin/iptables -P FORWARD ACCEPT -w 5
-ExecStart=/opt/eks/kubelet \
+ExecStart=/usr/share/amazon/eks/kubelet \
     --cloud-provider aws \
-    --config /oem/eks/kubelet-config.json \
+    --config /usr/share/amazon/eks/kubelet-config.json \
-    --kubeconfig /oem/eks/kubelet-kubeconfig \
+    --kubeconfig /usr/share/amazon/eks/kubelet-kubeconfig \
     --container-runtime docker \
     --network-plugin cni \
     --cni-bin-dir=/opt/cni/bin \

sdk_container/src/third_party/coreos-overlay/coreos-base/flatcar-eks/flatcar-eks-0.0.1-r2.ebuild

@@ -31,7 +31,7 @@ src_prepare() {
 }
 
 src_install() {
-    insinto /oem/eks
+    insinto /usr/share/amazon/eks
     doins "${WORKDIR}/bootstrap.sh"
 
     # These files are based on the ones found on the amazon-eks-ami repository,
@@ -53,5 +53,5 @@ src_install() {
     # necessary files to run the kubelet on the node.
     doins "${FILESDIR}/download-kubelet.sh"
 
-    chmod +x "${D}/oem/eks/bootstrap.sh" "${D}/oem/eks/download-kubelet.sh"
+    chmod +x "${D}/usr/share/amazon/eks/bootstrap.sh" "${D}/usr/share/amazon/eks/download-kubelet.sh"
 }

sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/files/oems/ami

@@ -0,0 +1,11 @@
+/etc/eks
+/etc/systemd/system/amazon-ssm-agent.service
+/etc/systemd/system/multi-user.target.requires/coreos-metadata-sshkeys@core.service
+/etc/systemd/system/multi-user.target.wants/amazon-ssm-agent.service
+/etc/systemd/system/multi-user.target.wants/nvidia.service
+/etc/systemd/system/nvidia.service
+/oem/base/
+/oem/bin/
+/oem/eks/
+/oem/ssm
+/oem/units

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/10-oem-ami.conf

@@ -0,0 +1,2 @@
+[Unit]
+Upholds=amazon-ssm-agent.service setup-oem.service

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/files/setup-oem.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Setup OEM
+Before=amazon-ssm-agent.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStartPre=/usr/bin/mkdir --parents /etc/amazon/ssm/ /etc/eks
+ExecStartPre=-/usr/bin/ln --symbolic /usr/share/amazon/ssm/amazon-ssm-agent.json.template /etc/amazon/ssm/amazon-ssm-agent.json
+ExecStartPre=-/usr/bin/ln --symbolic /usr/share/amazon/ssm/seelog.xml.template /etc/amazon/ssm/seelog.xml
+ExecStart=-/usr/bin/ln --symbolic /usr/share/amazon/eks/bootstrap.sh /etc/eks/bootstrap.sh
+[Install]
+WantedBy=multi-user.target

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/metadata.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "http://www.gentoo.org/dtd/metadata.dtd">
+<pkgmetadata>
+</pkgmetadata>

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ami/oem-ami-3.2.985.0.ebuild

@@ -0,0 +1,33 @@
+# Copyright (c) 2013 CoreOS, Inc.. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+inherit systemd
+
+DESCRIPTION="OEM suite for Amazon Machine Images"
+HOMEPAGE="http://aws.amazon.com/ec2/"
+SRC_URI=""
+
+LICENSE="Apache-2.0"
+SLOT="0"
+KEYWORDS="amd64 arm64"
+IUSE=""
+
+RDEPEND="
+  ~app-emulation/amazon-ssm-agent-${PV}
+  coreos-base/flatcar-eks
+"
+
+# for coreos-base/common-oem-files
+OEM_NAME="Amazon EC2"
+
+S="${WORKDIR}"
+
+src_install() {
+	systemd_dounit "${FILESDIR}/setup-oem.service"
+
+	insinto "$(systemd_get_systemunitdir)/multi-user.target.d"
+	doins "${FILESDIR}/10-oem-ami.conf"
+
+}

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ec2-compat/files/base/base-ec2.ign

@@ -1,46 +0,0 @@
-{
-  "ignition": {
-    "version": "2.1.0"
-  },
-  "systemd": {
-    "units": [
-      {
-        "name": "coreos-metadata-sshkeys@.service",
-        "enabled": true
-      },
-      {
-        "name": "amazon-ssm-agent.service",
-        "enabled": true,
-        "contents": "[Unit]\nDescription=amazon-ssm-agent\nAfter=network-online.target\n\n[Service]\nType=simple\nWorkingDirectory=/oem\nExecStart=/oem/bin/amazon-ssm-agent\nKillMode=process\nRestart=on-failure\nRestartForceExitStatus=SIGPIPE\nRestartSec=15min\n\n[Install]\nWantedBy=multi-user.target\n"
-      }
-    ]
-  },
-  "storage": {
-    "files": [
-      {
-        "filesystem": "root",
-        "path": "/etc/amazon/ssm/amazon-ssm-agent.json",
-        "contents": {
-          "source": "oem:///ssm/amazon-ssm-agent.json.template"
-        },
-        "mode": 292
-      },
-      {
-        "filesystem": "root",
-        "path": "/etc/amazon/ssm/seelog.xml",
-        "contents": {
-          "source": "oem:///ssm/seelog.xml.template"
-        },
-        "mode": 292
-      },
-      {
-        "filesystem": "root",
-        "path": "/etc/eks/bootstrap.sh",
-        "contents": {
-          "source": "oem:///eks/bootstrap.sh"
-        },
-        "mode": 493
-      }
-    ]
-  }
-}

sdk_container/src/third_party/coreos-overlay/coreos-base/oem-ec2-compat/oem-ec2-compat-0.1.2-r4.ebuild

@@ -10,24 +10,15 @@ SRC_URI=""
 LICENSE="Apache-2.0"
 SLOT="0"
 KEYWORDS="amd64 arm64 x86"
-IUSE="ec2 openstack brightbox"
+IUSE="openstack brightbox"
-REQUIRED_USE="^^ ( ec2 openstack brightbox )"
+REQUIRED_USE="^^ ( openstack brightbox )"
-
-RDEPEND="
-	ec2? ( app-emulation/amazon-ssm-agent )
-	coreos-base/flatcar-eks
-"
 
 # no source directory
 S="${WORKDIR}"
 
 src_prepare() {
 	default
-	if use ec2 ; then
+	if use openstack ; then
-		ID="ami"
-		NAME="Amazon EC2"
-		HOME_URL="http://aws.amazon.com/ec2/"
-	elif use openstack ; then
 		ID="openstack"
 		NAME="Openstack"
 		HOME_URL="https://www.openstack.org/"
@@ -49,9 +40,7 @@ src_prepare() {
 src_install() {
 	insinto "/oem"
 	doins "${T}/oem-release"
-	if use ec2 ; then
+	if use openstack ; then
-		newins "${FILESDIR}/grub-ec2.cfg" grub.cfg
-	elif use openstack ; then
 		newins "${FILESDIR}/grub-openstack.cfg" grub.cfg
 	elif use brightbox ; then
 		newins "${FILESDIR}/grub-brightbox.cfg" grub.cfg
@@ -59,10 +48,6 @@ src_install() {
 
 	insinto "/oem/base"
 	doins "${FILESDIR}/base/README"
-	if use ec2 ; then
-		newins "${FILESDIR}/base/base-ec2.ign" base.ign
-	fi
-
 	if use openstack; then
 		newins "${FILESDIR}/base/openstack.ign" base.ign
 	fi