From a57a2261c687b82fa820da6e612dd558e923f980 Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Sat, 27 Jan 2024 17:36:25 -0600 Subject: [PATCH] feat(cni): add binaries to host when missing --- .gitignore | 1 + Dockerfile | 9 +++- Makefile | 12 ++++- build/image-assets/cni-install | 54 +++++++++++++++++++ ...erouter-all-features-advertise-routes.yaml | 10 +++- .../generic-kuberouter-all-features.yaml | 10 +++- daemonset/generic-kuberouter.yaml | 10 +++- ...ll-service-daemonset-advertise-routes.yaml | 10 +++- .../kube-router-all-service-daemonset.yaml | 10 +++- daemonset/kube-router-firewall-daemonset.yaml | 10 +++- daemonset/kube-router-proxy-daemonset.yaml | 10 +++- .../kubeadm-kuberouter-all-features-dsr.yaml | 10 +++- ...eadm-kuberouter-all-features-hostport.yaml | 10 +++- .../kubeadm-kuberouter-all-features.yaml | 10 +++- daemonset/kubeadm-kuberouter.yaml | 10 +++- 15 files changed, 172 insertions(+), 14 deletions(-) create mode 100755 build/image-assets/cni-install diff --git a/.gitignore b/.gitignore index e44baf44..12f4f409 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ _output _cache vendor .*.sw? +/cni-download diff --git a/Dockerfile b/Dockerfile index 770eea4f..555c64d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,16 @@ ARG BUILDTIME_BASE=golang:1-alpine ARG RUNTIME_BASE=alpine:latest +ARG TARGETPLATFORM +ARG CNI_VERSION FROM ${BUILDTIME_BASE} as builder ENV BUILD_IN_DOCKER=false WORKDIR /build COPY . /build -RUN apk add --no-cache make git \ +RUN apk add --no-cache make git tar curl \ && make kube-router \ - && make gobgp + && make gobgp \ + && make cni-download FROM ${RUNTIME_BASE} @@ -29,7 +32,9 @@ COPY build/image-assets/bashrc /root/.bashrc COPY build/image-assets/profile /root/.profile COPY build/image-assets/vimrc /root/.vimrc COPY build/image-assets/motd-kube-router.sh /etc/motd-kube-router.sh +COPY build/image-assets/cni-install /usr/local/bin/cni-install COPY --from=builder /build/kube-router /build/gobgp /usr/local/bin/ +COPY --from=builder /build/cni-download /usr/libexec/cni # Use iptables-wrappers so that correct version of iptables-legacy or iptables-nft gets used. Alpine contains both, but # which version is used should be based on the host system as well as where rules that may have been added before diff --git a/Makefile b/Makefile index 0526ed48..a01d6814 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ GOBGP_VERSION=v3.19.0 QEMU_IMAGE?=multiarch/qemu-user-static GORELEASER_VERSION=v1.21.2 MOQ_VERSION=v0.3.2 +CNI_VERSION=v1.4.0 UID?=$(shell id -u) ifeq ($(GOARCH), arm) ARCH_TAG_PREFIX=$(GOARCH) @@ -110,7 +111,7 @@ markdownlint: run: kube-router ## Runs "kube-router --help". ./kube-router --help -container: kube-router gobgp multiarch-binverify ## Builds a Docker container image. +container: kube-router gobgp multiarch-binverify cni-download ## Builds a Docker container image. @echo Starting kube-router container image build for $(GOARCH) on $(shell go env GOHOSTARCH) @if [ "$(GOARCH)" != "$(shell go env GOHOSTARCH)" ]; then \ echo "Using qemu to build non-native container"; \ @@ -182,6 +183,7 @@ release: push-release github-release ## Pushes a release to DockerHub and GitHub clean: ## Removes the kube-router binary and Docker images rm -f kube-router rm -f gobgp + rm -rf cni-download if [ $(shell $(DOCKER) images -q $(REGISTRY_DEV):$(IMG_TAG) 2> /dev/null) ]; then \ $(DOCKER) rmi $(REGISTRY_DEV):$(IMG_TAG); \ fi @@ -228,6 +230,14 @@ multiarch-binverify: @echo 'Verifying kube-router gobgp for ARCH=$(FILE_ARCH) ...' @[ `file kube-router gobgp| cut -d, -f2 |grep -cw "$(FILE_ARCH)"` -eq 2 ] +cni-download: + @echo Downloading CNI Plugins for $(GOARCH) + curl -L -o cni-plugins-$(GOARCH).tgz \ + https://github.com/containernetworking/plugins/releases/download/$(CNI_VERSION)/cni-plugins-linux-$(GOARCH)-$(CNI_VERSION).tgz + mkdir -p cni-download + tar -xf cni-plugins-$(GOARCH).tgz -C cni-download + rm -f cni-plugins-$(GOARCH).tgz + # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html help: @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ diff --git a/build/image-assets/cni-install b/build/image-assets/cni-install new file mode 100755 index 00000000..bfe396ef --- /dev/null +++ b/build/image-assets/cni-install @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -euo pipefail -x + +# CNI binaries that kube-router uses +KUBE_ROUTER_CNI_BINS=("bridge" "portmap" "host-local" "loopback") +# Local path of the CNI binaries within the kube-router container image +LOCAL_BIN_PATH="${LOCAL_BIN_PATH:-/usr/libexec/cni}" +# Path on the host where the CRI will look for the CNI binaries. This should be mounted into the initContainer so that +# the CRI can reference the binaries and this script has the intended effect. +HOST_BIN_PATH="${HOST_BIN_PATH:-/opt/cni/bin}" + +setup_cni() { + local cni_bin cni_dst_path cni_loc_path + + # If the host path for the binaries doesn't exist, create it + if [[ ! -d "${HOST_BIN_PATH}" ]]; then + printf "Host CNI bin path %s doesn't exist on node host, creating it\n" "${HOST_BIN_PATH}" + if mkdir -p "${HOST_BIN_PATH}" >/dev/null; then + printf "Successfully created CNI bin path\n" + else + printf "Failed to create missing CNI bin path, exiting\n" + return 1 + fi + fi + + # Loop over CNI binaries + for cni_bin in "${KUBE_ROUTER_CNI_BINS[@]}"; do + cni_dst_path="${HOST_BIN_PATH}/${cni_bin}" + cni_loc_path="${LOCAL_BIN_PATH}/${cni_bin}" + + # Check to see if the binary already exists on the host node + if [[ -x "${cni_dst_path}" ]]; then + # If it did, then output a message and skip this loop + printf "CNI binary %s already exists and is executable, skipping\n" "${cni_dst_path}" + continue + fi + + # If it didn't then try to install it + printf "CNI binary %s was missing or wasn't executable, installing it\n" "${cni_dst_path}" + if install -m 755 "${cni_loc_path}" "${cni_dst_path}" >/dev/null; then + printf "CNI install successfull\n" + else + printf "Failed to install CNI binary, exiting\n" + return 2 + fi + done + + printf "CNI setup completed successfully!" + return 0 +} + +setup_cni "${@}" +exit $? diff --git a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml index 86f133ae..a0a5ec54 100644 --- a/daemonset/generic-kuberouter-all-features-advertise-routes.yaml +++ b/daemonset/generic-kuberouter-all-features-advertise-routes.yaml @@ -134,7 +134,10 @@ spec: TMP=/var/lib/kube-router/.tmp-kubeconfig; cp /etc/kube-router/kubeconfig ${TMP}; mv ${TMP} /var/lib/kube-router/kubeconfig; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - mountPath: /etc/cni/net.d name: cni-conf-dir @@ -142,6 +145,8 @@ spec: name: kube-router-cfg - name: kubeconfig mountPath: /var/lib/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -168,6 +173,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 diff --git a/daemonset/generic-kuberouter-all-features.yaml b/daemonset/generic-kuberouter-all-features.yaml index d58d10d1..4b936a80 100644 --- a/daemonset/generic-kuberouter-all-features.yaml +++ b/daemonset/generic-kuberouter-all-features.yaml @@ -130,7 +130,10 @@ spec: TMP=/var/lib/kube-router/.tmp-kubeconfig; cp /etc/kube-router/kubeconfig ${TMP}; mv ${TMP} /var/lib/kube-router/kubeconfig; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - mountPath: /etc/cni/net.d name: cni-conf-dir @@ -138,6 +141,8 @@ spec: name: kube-router-cfg - name: kubeconfig mountPath: /var/lib/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -164,6 +169,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 diff --git a/daemonset/generic-kuberouter.yaml b/daemonset/generic-kuberouter.yaml index 8e09054e..0179d35c 100644 --- a/daemonset/generic-kuberouter.yaml +++ b/daemonset/generic-kuberouter.yaml @@ -102,12 +102,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - mountPath: /etc/cni/net.d name: cni-conf-dir - mountPath: /etc/kube-router name: kube-router-cfg + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -131,6 +136,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 diff --git a/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml b/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml index 70624d3b..ce888c49 100644 --- a/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml +++ b/daemonset/kube-router-all-service-daemonset-advertise-routes.yaml @@ -97,12 +97,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -129,3 +134,6 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt diff --git a/daemonset/kube-router-all-service-daemonset.yaml b/daemonset/kube-router-all-service-daemonset.yaml index c8200524..61897311 100644 --- a/daemonset/kube-router-all-service-daemonset.yaml +++ b/daemonset/kube-router-all-service-daemonset.yaml @@ -93,12 +93,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -125,3 +130,6 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt diff --git a/daemonset/kube-router-firewall-daemonset.yaml b/daemonset/kube-router-firewall-daemonset.yaml index 54237eab..2ddfd62d 100644 --- a/daemonset/kube-router-firewall-daemonset.yaml +++ b/daemonset/kube-router-firewall-daemonset.yaml @@ -92,12 +92,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -124,3 +129,6 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt diff --git a/daemonset/kube-router-proxy-daemonset.yaml b/daemonset/kube-router-proxy-daemonset.yaml index bb864efd..a24cee83 100644 --- a/daemonset/kube-router-proxy-daemonset.yaml +++ b/daemonset/kube-router-proxy-daemonset.yaml @@ -92,12 +92,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -124,3 +129,6 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt diff --git a/daemonset/kubeadm-kuberouter-all-features-dsr.yaml b/daemonset/kubeadm-kuberouter-all-features-dsr.yaml index 974f1d81..d5068262 100644 --- a/daemonset/kubeadm-kuberouter-all-features-dsr.yaml +++ b/daemonset/kubeadm-kuberouter-all-features-dsr.yaml @@ -109,12 +109,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostIPC: true hostPID: true @@ -148,6 +153,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 kind: ServiceAccount diff --git a/daemonset/kubeadm-kuberouter-all-features-hostport.yaml b/daemonset/kubeadm-kuberouter-all-features-hostport.yaml index 17eb1ea1..23bc3318 100644 --- a/daemonset/kubeadm-kuberouter-all-features-hostport.yaml +++ b/daemonset/kubeadm-kuberouter-all-features-hostport.yaml @@ -113,12 +113,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -148,6 +153,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 kind: ServiceAccount diff --git a/daemonset/kubeadm-kuberouter-all-features.yaml b/daemonset/kubeadm-kuberouter-all-features.yaml index 67767995..e4c7709e 100644 --- a/daemonset/kubeadm-kuberouter-all-features.yaml +++ b/daemonset/kubeadm-kuberouter-all-features.yaml @@ -106,12 +106,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - name: cni-conf-dir mountPath: /etc/cni/net.d - name: kube-router-cfg mountPath: /etc/kube-router + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -141,6 +146,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 kind: ServiceAccount diff --git a/daemonset/kubeadm-kuberouter.yaml b/daemonset/kubeadm-kuberouter.yaml index c30e1e04..4663ee4c 100644 --- a/daemonset/kubeadm-kuberouter.yaml +++ b/daemonset/kubeadm-kuberouter.yaml @@ -105,12 +105,17 @@ spec: TMP=/etc/cni/net.d/.tmp-kuberouter-cfg; cp /etc/kube-router/cni-conf.json ${TMP}; mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist; - fi + fi; + if [ -x /usr/local/bin/cni-install ]; then + /usr/local/bin/cni-install; + fi; volumeMounts: - mountPath: /etc/cni/net.d name: cni-conf-dir - mountPath: /etc/kube-router name: kube-router-cfg + - name: host-opt + mountPath: /opt hostNetwork: true hostPID: true tolerations: @@ -137,6 +142,9 @@ spec: hostPath: path: /run/xtables.lock type: FileOrCreate + - name: host-opt + hostPath: + path: /opt --- apiVersion: v1 kind: ServiceAccount