From f5612a8a951b6220ceb2988eb192188b4338d8c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kai=20L=C3=BCke?= <kai@kinvolk.io>
Date: Wed, 24 Feb 2021 12:13:04 +0100
Subject: [PATCH] app-admin/flannel-wrapper: user docker for the flannel
 service

As rkt is deprecated we need to run the Flannel container with Docker
or Podman. The flannel-wrapper script is based on rkt arguments and
can't be used in a compatible way but we cannot remove it since ct
explicitly uses it in the ExecStart directive when writing out a
drop-in file once flannel settings are given in a Container Linux
Config.
A better way to run the Flannel/etcd container image is Podman because
Flannel depends on etcd but wants to be run before Docker so that it
can set up the Docker networking. Etcd and Flannel are part of the
Container Linux Config specification and thus can't be removed easily.
For now we have to resort to running these services with Docker and try
to restart Docker for the Flannel options to take effect (but that also
terminates the etcd and flannel containers, causing the services to
restart).
---
 .../files/flannel-docker-opts.service         |  13 +-
 .../flannel-wrapper/files/flannel-wrapper     | 114 ++++++------------
 .../flannel-wrapper/files/flanneld.service    |  10 +-
 .../flannel-wrapper-0.12.0.ebuild             |   2 +-
 4 files changed, 50 insertions(+), 89 deletions(-)

diff --git a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-docker-opts.service b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-docker-opts.service
index fe20813d10..9ab7ddce64 100644
--- a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-docker-opts.service
+++ b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-docker-opts.service
@@ -4,18 +4,21 @@ Documentation=https://github.com/coreos/flannel
 PartOf=flanneld.service
 Requires=flanneld.service
 After=flanneld.service
-Before=docker.service
 
 [Service]
 Type=oneshot
+RemainAfterExit=true
+Restart=on-failure
+RestartSec=5s
 
+Environment="FLANNEL_IMAGE_URL=quay.io/coreos/flannel"
 Environment="FLANNEL_IMAGE_TAG=@FLANNEL_IMAGE_TAG@"
-Environment="RKT_RUN_ARGS=--uuid-file-save=/var/lib/flatcar/flannel-wrapper2.uuid"
-Environment="FLANNEL_IMAGE_ARGS=--exec=/opt/bin/mk-docker-opts.sh"
+Environment="FLANNEL_CMD=/opt/bin/mk-docker-opts.sh"
+Environment="FLANNEL_CONTAINER=flannel-docker-opts"
 
-ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/lib/flatcar/flannel-wrapper2.uuid
 ExecStart=/usr/lib/flatcar/flannel-wrapper -d /run/flannel/flannel_docker_opts.env -i
-ExecStop=-/usr/bin/rkt stop --uuid-file=/var/lib/flatcar/flannel-wrapper2.uuid
+ExecStop=/usr/bin/docker stop flannel-docker-opts
+ExecStopPost=/usr/bin/docker rm flannel-docker-opts
 
 [Install]
 WantedBy=multi-user.target
diff --git a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-wrapper b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-wrapper
index 3929a4d97e..c8aa3b05a9 100755
--- a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-wrapper
+++ b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flannel-wrapper
@@ -1,86 +1,42 @@
-#!/bin/bash -e
-# Wrapper for launching flannel via rkt.
-#
-# Make sure to set FLANNEL_IMAGE_TAG to an image tag published here:
-# https://quay.io/repository/coreos/flannel?tab=tags Alternatively,
-# override FLANNEL_IMAGE to a custom image.
-
-function require_ev_all() {
-	for rev in $@ ; do
-		if [[ -z "${!rev}" ]]; then
-			echo "${rev}" is not set
-			exit 1
-		fi
-	done
-}
-
-function require_ev_one() {
-	for rev in $@ ; do
-		if [[ ! -z "${!rev}" ]]; then
-			return
-		fi
-	done
-	echo One of $@ must be set
-	exit 1
-}
-
-if [[ -n "${FLANNEL_VER}" ]]; then
-	echo FLANNEL_VER environment variable is deprecated, please use FLANNEL_IMAGE_TAG instead
-fi
-
-if [[ -n "${FLANNEL_IMG}" ]]; then
-	echo FLANNEL_IMG environment variable is deprecated, please use FLANNEL_IMAGE_URL instead
-fi
-
-FLANNEL_IMAGE_TAG="${FLANNEL_IMAGE_TAG:-${FLANNEL_VER}}"
-
-require_ev_one FLANNEL_IMAGE FLANNEL_IMAGE_TAG
-
-FLANNEL_IMAGE_URL="${FLANNEL_IMAGE_URL:-${FLANNEL_IMG:-docker://quay.io/coreos/flannel}}"
-FLANNEL_IMAGE="${FLANNEL_IMAGE:-${FLANNEL_IMAGE_URL}:${FLANNEL_IMAGE_TAG}}"
-
-if [[ "${FLANNEL_IMAGE%%/*}" == "quay.io" ]] && ! (echo "${RKT_RUN_ARGS}" | grep -q trust-keys-from-https); then
-	RKT_RUN_ARGS="${RKT_RUN_ARGS} --trust-keys-from-https"
-elif [[ "${FLANNEL_IMAGE%%/*}" == "docker:" ]] && ! (echo "${RKT_RUN_ARGS}" | grep -q insecure-options); then
-	RKT_RUN_ARGS="${RKT_RUN_ARGS} --insecure-options=image"
-fi
+#!/bin/bash
+# The "flannel-wrapper" script can't be deleted because ct overwrites
+# the ExecStart directive with flannel-wrapper. Do the new action of
+# ExecStart here.
+set -e
 
 ETCD_SSL_DIR="${ETCD_SSL_DIR:-/etc/ssl/etcd}"
+RUN_ARGS=""
 if [[ -d "${ETCD_SSL_DIR}" ]]; then
-	RKT_RUN_ARGS="${RKT_RUN_ARGS} \
-		--volume coreos-ssl,kind=host,source=${ETCD_SSL_DIR},readOnly=true \
-		--mount volume=coreos-ssl,target=${ETCD_SSL_DIR} \
-	"
-fi
-
-if [[ -S "${NOTIFY_SOCKET}" ]]; then
-	RKT_RUN_ARGS="${RKT_RUN_ARGS} \
-		--mount volume=coreos-notify,target=/run/systemd/notify \
-		--volume coreos-notify,kind=host,source=${NOTIFY_SOCKET} \
-		--set-env=NOTIFY_SOCKET=/run/systemd/notify \
-	"
+	RUN_ARGS="-v ${ETCD_SSL_DIR}:${ETCD_SSL_DIR}:ro"
 fi
 
 mkdir --parents /run/flannel
 
-RKT="${RKT:-/usr/bin/rkt}"
-RKT_STAGE1_ARG="${RKT_STAGE1_ARG:---stage1-from-dir=stage1-fly.aci}"
-set -x
-exec ${RKT} ${RKT_GLOBAL_ARGS} \
-	run ${RKT_RUN_ARGS} \
-	--net=host \
-	--volume coreos-run-flannel,kind=host,source=/run/flannel,readOnly=false \
-	--volume coreos-etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \
-	--volume coreos-usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \
-	--volume coreos-etc-hosts,kind=host,source=/etc/hosts,readOnly=true \
-	--volume coreos-etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \
-	--mount volume=coreos-run-flannel,target=/run/flannel \
-	--mount volume=coreos-etc-ssl-certs,target=/etc/ssl/certs \
-	--mount volume=coreos-usr-share-certs,target=/usr/share/ca-certificates \
-	--mount volume=coreos-etc-hosts,target=/etc/hosts  \
-	--mount volume=coreos-etc-resolv,target=/etc/resolv.conf \
-	--inherit-env \
-	${RKT_STAGE1_ARG} \
-	${FLANNEL_IMAGE} \
-		${FLANNEL_IMAGE_ARGS} \
-		-- "$@"
+
+WRAP=""
+if [[ -S "${NOTIFY_SOCKET}" ]]; then
+  WRAP="/usr/libexec/sdnotify-proxy /run/${FLANNEL_CONTAINER}-notify"
+fi
+
+# A better way to run the Flannel/etcd container image is Podman because
+# Flannel depends on etcd but wants to be run before Docker so that it
+# can set up the Docker networking. Etcd and Flannel are part of the
+# Container Linux Config specification and thus can't be dropped easily.
+# For now we have to resort to running these services with Docker and try
+# to restart Docker for the Flannel options to take effect (but that also
+# terminates the etcd and flannel containers, causing the services to
+# restart).
+RESTART_DOCKER=0
+if [ "${FLANNEL_CMD}" = "/opt/bin/mk-docker-opts.sh" ] && [ "$1" = "-d" ] && [ ! -f "$2" ]; then
+  # only restart Docker only on first run, propagating updates on later runs was not done before in the rkt version, so keep the behavior
+  # (which also helps to break the loop which otherwise exists because "restart docker" mentioned above)
+  RESTART_DOCKER=1
+fi
+/usr/bin/docker stop ${FLANNEL_CONTAINER} || true
+/usr/bin/docker rm -f ${FLANNEL_CONTAINER} || true
+# mapping only /run/${FLANNEL_CONTAINER}-notify does not work and we map the full /run (using /run:/run covers /run/flannel, too), also we must set NOTIFY_SOCKET in the container but use the original for /usr/libexec/sdnotify-proxy
+${WRAP} /usr/bin/docker run --name ${FLANNEL_CONTAINER} --privileged --network=host --ipc=host ${RUN_ARGS} -v /run:/run:rw -v /etc/ssl/certs:/etc/ssl/certs:ro -v /usr/share/ca-certificates:/usr/share/ca-certificates:ro --env-file <(env; echo PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin; echo NOTIFY_SOCKET=/run/${FLANNEL_CONTAINER}-notify) --entrypoint ${FLANNEL_CMD} ${FLANNEL_IMAGE:-${FLANNEL_IMAGE_URL}:${FLANNEL_IMAGE_TAG}} "$@"
+if [ "${RESTART_DOCKER}" = 1 ]; then
+  systemctl restart docker
+  echo "Restarted Docker to apply Flannel options"
+fi
diff --git a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flanneld.service b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flanneld.service
index 5650a4c707..7fa589d6bf 100644
--- a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flanneld.service
+++ b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/files/flanneld.service
@@ -6,22 +6,24 @@ Requires=flannel-docker-opts.service
 
 [Service]
 Type=notify
+NotifyAccess=all
 Restart=always
 RestartSec=10s
 TimeoutStartSec=300
 LimitNOFILE=40000
 LimitNPROC=1048576
 
+Environment="FLANNEL_IMAGE_URL=quay.io/coreos/flannel"
 Environment="FLANNEL_IMAGE_TAG=@FLANNEL_IMAGE_TAG@"
 Environment="FLANNEL_OPTS=--ip-masq=true"
-Environment="RKT_RUN_ARGS=--uuid-file-save=/var/lib/flatcar/flannel-wrapper.uuid"
+Environment="FLANNEL_CMD=/opt/bin/flanneld"
+Environment="FLANNEL_CONTAINER=flannel"
 EnvironmentFile=-/run/flannel/options.env
 
 ExecStartPre=/sbin/modprobe ip_tables
-ExecStartPre=/usr/bin/mkdir --parents /var/lib/flatcar /run/flannel
-ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/lib/flatcar/flannel-wrapper.uuid
 ExecStart=/usr/lib/flatcar/flannel-wrapper $FLANNEL_OPTS
-ExecStop=-/usr/bin/rkt stop --uuid-file=/var/lib/flatcar/flannel-wrapper.uuid
+ExecStop=/usr/bin/docker stop flannel
+ExecStopPost=/usr/bin/docker rm flannel
 
 [Install]
 WantedBy=multi-user.target
diff --git a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/flannel-wrapper-0.12.0.ebuild b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/flannel-wrapper-0.12.0.ebuild
index d9d3adb65c..31fe837498 100644
--- a/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/flannel-wrapper-0.12.0.ebuild
+++ b/sdk_container/src/third_party/coreos-overlay/app-admin/flannel-wrapper/flannel-wrapper-0.12.0.ebuild
@@ -17,7 +17,7 @@ IUSE=""
 
 RDEPEND="
 	!app-admin/flannel
-	>=app-emulation/rkt-1.9.1[rkt_stage1_fly]
+	>=app-admin/sdnotify-proxy-0.1.0
 "
 
 S="$WORKDIR"