fix: reload trusted CA list when client is recreated

Fixes #5652

This reworks and unifies HTTP client/transport management in Talos:

* cleanhttp is used everywhere consistently
* DefaultClient is using pooled client, other clients use regular
  transport
* like before, Proxy vars are inspected on each request (but now
  consistently)
* manifest download functions now recreate the client on each run to
  pick up latest changes
* system CA list is picked up from a fixed locations, and supports
  reloading on changes

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

cmd/talosctl/cmd/mgmt/debug/air-gapped.go

@@ -0,0 +1,291 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package debug
+
+import (
+	"bytes"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"embed"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"math/big"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/talos-systems/talos/pkg/cli"
+	"github.com/talos-systems/talos/pkg/machinery/config/encoder"
+	"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1"
+)
+
+//go:embed httproot/*
+var httpFs embed.FS
+
+var airgappedFlags struct {
+	advertisedAddress net.IP
+	proxyPort         int
+	httpsPort         int
+}
+
+// airgappedCmd represents the `gen ca` command.
+var airgappedCmd = &cobra.Command{
+	Use:   "air-gapped",
+	Short: "Starts a local HTTP proxy and HTTPS server serving a test manifest.",
+	Long:  ``,
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return cli.WithContext(
+			context.Background(), func(ctx context.Context) error {
+				certPEM, keyPEM, err := generateSelfSignedCert()
+				if err != nil {
+					return nil
+				}
+
+				if err = generateConfigPatch(certPEM); err != nil {
+					return err
+				}
+
+				eg, ctx := errgroup.WithContext(ctx)
+
+				eg.Go(func() error { return runHTTPServer(ctx, certPEM, keyPEM) })
+				eg.Go(func() error { return runHTTPProxy(ctx) })
+
+				return eg.Wait()
+			},
+		)
+	},
+}
+
+func generateConfigPatch(caPEM []byte) error {
+	patch := &v1alpha1.Config{
+		MachineConfig: &v1alpha1.MachineConfig{
+			MachineEnv: map[string]string{
+				"http_proxy":  fmt.Sprintf("http://%s", net.JoinHostPort(airgappedFlags.advertisedAddress.String(), strconv.Itoa(airgappedFlags.proxyPort))),
+				"https_proxy": fmt.Sprintf("http://%s", net.JoinHostPort(airgappedFlags.advertisedAddress.String(), strconv.Itoa(airgappedFlags.proxyPort))),
+				"no_proxy":    fmt.Sprintf("%s/24", airgappedFlags.advertisedAddress.String()),
+			},
+			MachineFiles: []*v1alpha1.MachineFile{
+				{
+					FilePath:        "/etc/ssl/certs/ca-certificates",
+					FileContent:     string(caPEM),
+					FilePermissions: 0o644,
+					FileOp:          "append",
+				},
+			},
+		},
+		ClusterConfig: &v1alpha1.ClusterConfig{
+			ExtraManifests: []string{
+				fmt.Sprintf("https://%s/debug.yaml", net.JoinHostPort(airgappedFlags.advertisedAddress.String(), strconv.Itoa(airgappedFlags.httpsPort))),
+			},
+		},
+	}
+
+	patchBytes, err := patch.EncodeBytes(encoder.WithComments(encoder.CommentsDisabled))
+	if err != nil {
+		return err
+	}
+
+	const patchFile = "air-gapped-patch.yaml"
+
+	log.Printf("writing config patch to %s", patchFile)
+
+	return os.WriteFile(patchFile, patchBytes, 0o644)
+}
+
+func generateSelfSignedCert() ([]byte, []byte, error) {
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	template := x509.Certificate{
+		SerialNumber:       big.NewInt(1),
+		SignatureAlgorithm: x509.ECDSAWithSHA256,
+		Subject: pkix.Name{
+			Organization: []string{"Test Only"},
+		},
+		NotBefore: time.Now(),
+		NotAfter:  time.Now().Add(time.Hour * 24),
+
+		KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{
+			x509.ExtKeyUsageServerAuth,
+			x509.ExtKeyUsageClientAuth,
+		},
+		BasicConstraintsValid: true,
+
+		IsCA: true,
+
+		IPAddresses: []net.IP{airgappedFlags.advertisedAddress},
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var crt bytes.Buffer
+
+	if err = pem.Encode(&crt, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return nil, nil, err
+	}
+
+	var key bytes.Buffer
+
+	keyBytes, err := x509.MarshalECPrivateKey(priv)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if err = pem.Encode(&key, &pem.Block{Type: "EC PRIVATE KEY", Bytes: keyBytes}); err != nil {
+		return nil, nil, err
+	}
+
+	return crt.Bytes(), key.Bytes(), nil
+}
+
+func runHTTPServer(ctx context.Context, certPEM, keyPEM []byte) error {
+	certificate, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		return err
+	}
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{certificate},
+	}
+
+	subFs, err := fs.Sub(httpFs, "httproot")
+	if err != nil {
+		return err
+	}
+
+	srv := &http.Server{
+		Addr:      net.JoinHostPort("", strconv.Itoa(airgappedFlags.httpsPort)),
+		Handler:   loggingMiddleware(http.FileServer(http.FS(subFs))),
+		TLSConfig: tlsConfig,
+	}
+
+	log.Printf("starting HTTPS server with self-signed cert on %s", srv.Addr)
+
+	go srv.ListenAndServeTLS("", "") //nolint:errcheck
+
+	<-ctx.Done()
+
+	return srv.Close()
+}
+
+func handleTunneling(w http.ResponseWriter, r *http.Request) {
+	dst, err := net.DialTimeout("tcp", r.Host, 10*time.Second)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "Hijacking not supported", http.StatusInternalServerError)
+
+		return
+	}
+
+	clientConn, _, err := hijacker.Hijack()
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+
+		return
+	}
+
+	go transfer(dst, clientConn)
+	go transfer(clientConn, dst)
+}
+
+func transfer(destination io.WriteCloser, source io.ReadCloser) {
+	defer destination.Close() //nolint:errcheck
+	defer source.Close()      //nolint:errcheck
+
+	io.Copy(destination, source) //nolint:errcheck
+}
+
+func handleHTTP(w http.ResponseWriter, req *http.Request) {
+	resp, err := http.DefaultTransport.RoundTrip(req)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusServiceUnavailable)
+
+		return
+	}
+
+	defer resp.Body.Close() //nolint:errcheck
+
+	copyHeaders(w.Header(), resp.Header)
+
+	w.WriteHeader(resp.StatusCode)
+	io.Copy(w, resp.Body) //nolint:errcheck
+}
+
+func copyHeaders(dst, src http.Header) {
+	for k, vv := range src {
+		for _, v := range vv {
+			dst.Add(k, v)
+		}
+	}
+}
+
+func loggingMiddleware(h http.Handler) http.Handler {
+	logFn := func(rw http.ResponseWriter, r *http.Request) {
+		h.ServeHTTP(rw, r) // serve the original request
+
+		log.Printf("%s %s", r.Method, r.RequestURI)
+	}
+
+	return http.HandlerFunc(logFn)
+}
+
+func runHTTPProxy(ctx context.Context) error {
+	srv := &http.Server{
+		Addr: net.JoinHostPort("", strconv.Itoa(airgappedFlags.proxyPort)),
+		Handler: loggingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if r.Method == http.MethodConnect {
+				handleTunneling(w, r)
+			} else {
+				handleHTTP(w, r)
+			}
+		})),
+		// Disable HTTP/2.
+		TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+	}
+
+	log.Printf("starting HTTP proxy on %s", srv.Addr)
+
+	go srv.ListenAndServe() //nolint:errcheck
+
+	<-ctx.Done()
+
+	return srv.Close()
+}
+
+func init() {
+	airgappedCmd.Flags().IPVar(&airgappedFlags.advertisedAddress, "advertised-address", net.IPv4(10, 5, 0, 2), "The address to advertise to the cluster.")
+	airgappedCmd.Flags().IntVar(&airgappedFlags.httpsPort, "https-port", 8001, "The HTTPS server port.")
+	airgappedCmd.Flags().IntVar(&airgappedFlags.proxyPort, "proxy-port", 8002, "The HTTP proxy port.")
+
+	Cmd.AddCommand(airgappedCmd)
+}

cmd/talosctl/cmd/mgmt/debug/debug.go

@@ -0,0 +1,18 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package debug implements "debug" subcommands.
+package debug
+
+import (
+	"github.com/spf13/cobra"
+)
+
+// Cmd represents the debug command.
+var Cmd = &cobra.Command{
+	Use:    "debug",
+	Short:  "A collection of commands to facilitate debugging of Talos.",
+	Hidden: true,
+	Long:   ``,
+}

cmd/talosctl/cmd/mgmt/debug/httproot/debug.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: debug-container
+  name: debug-container
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: debug-container
+  template:
+    metadata:
+      labels:
+        app: debug-container
+    spec:
+      containers:
+      - args:
+        - "inf"
+        command:
+        - /bin/sleep
+        image: alpine:latest
+        imagePullPolicy: IfNotPresent
+        name: debug-container
+        resources: {}
+      terminationGracePeriodSeconds: 30
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+    type: RollingUpdate

cmd/talosctl/cmd/mgmt/root.go

@@ -8,6 +8,7 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/talos-systems/talos/cmd/talosctl/cmd/mgmt/cluster"
+	"github.com/talos-systems/talos/cmd/talosctl/cmd/mgmt/debug"
 	"github.com/talos-systems/talos/cmd/talosctl/cmd/mgmt/gen"
 )
 
@@ -26,4 +27,5 @@ func addCommand(cmd *cobra.Command) {
 func init() {
 	addCommand(cluster.Cmd)
 	addCommand(gen.Cmd)
+	addCommand(debug.Cmd)
 }

internal/app/machined/main.go

@@ -9,19 +9,17 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/go-cleanhttp"
 	"github.com/talos-systems/go-cmd/pkg/cmd/proc"
 	"github.com/talos-systems/go-cmd/pkg/cmd/proc/reaper"
 	debug "github.com/talos-systems/go-debug"
 	"github.com/talos-systems/go-procfs/procfs"
-	"golang.org/x/net/http/httpproxy"
 	"golang.org/x/sys/unix"
 
 	"github.com/talos-systems/talos/internal/app/apid"
@@ -33,6 +31,7 @@ import (
 	"github.com/talos-systems/talos/internal/app/poweroff"
 	"github.com/talos-systems/talos/internal/app/trustd"
 	"github.com/talos-systems/talos/internal/pkg/mount"
+	"github.com/talos-systems/talos/pkg/httpdefaults"
 	"github.com/talos-systems/talos/pkg/machinery/api/common"
 	"github.com/talos-systems/talos/pkg/machinery/api/machine"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
@@ -40,25 +39,8 @@ import (
 )
 
 func init() {
-	// Explicitly set the default http client transport to work around proxy.Do
-	// once. This is the http.DefaultTransport with the Proxy func overridden so
-	// that the environment variables with be reread/initialized each time the
-	// http call is made.
-	http.DefaultClient.Transport = &http.Transport{
-		Proxy: func(req *http.Request) (*url.URL, error) {
-			return httpproxy.FromEnvironment().ProxyFunc()(req.URL)
-		},
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext,
-		ForceAttemptHTTP2:     true,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
-	}
+	// Patch a default HTTP client with updated transport to handle cases when default client is being used.
+	http.DefaultClient.Transport = httpdefaults.PatchTransport(cleanhttp.DefaultPooledTransport())
 }
 
 func recovery() {

internal/app/machined/pkg/controllers/k8s/extra_manifest.go

@@ -14,12 +14,14 @@ import (
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/resource"
 	"github.com/cosi-project/runtime/pkg/state"
+	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-getter"
 	"github.com/hashicorp/go-multierror"
 	"github.com/siderolabs/go-pointer"
 	"go.uber.org/zap"
 
 	k8sadapter "github.com/talos-systems/talos/internal/app/machined/pkg/adapters/k8s"
+	"github.com/talos-systems/talos/pkg/httpdefaults"
 	"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
 	"github.com/talos-systems/talos/pkg/machinery/resources/network"
 )
@@ -167,8 +169,10 @@ func (ctrl *ExtraManifestController) processURL(ctx context.Context, r controlle
 	// Disable netrc since we don't have getent installed, and most likely
 	// never will.
 	httpGetter := &getter.HttpGetter{
-		Netrc:  false,
-		Client: http.DefaultClient,
+		Netrc: false,
+		Client: &http.Client{
+			Transport: httpdefaults.PatchTransport(cleanhttp.DefaultTransport()),
+		},
 	}
 
 	httpGetter.Header = make(http.Header)

internal/pkg/containers/image/resolver.go

@@ -7,16 +7,15 @@ package image
 import (
 	"encoding/base64"
 	"fmt"
-	"net"
 	"net/http"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
-	"golang.org/x/net/http/httpproxy"
+	"github.com/hashicorp/go-cleanhttp"
 
+	"github.com/talos-systems/talos/pkg/httpdefaults"
 	"github.com/talos-systems/talos/pkg/machinery/config"
 )
 
@@ -159,19 +158,5 @@ func PrepareAuth(auth config.RegistryAuthConfig, host, expectedHost string) (str
 
 // newTransport creates HTTP transport with default settings.
 func newTransport() *http.Transport {
-	return &http.Transport{
-		// work around for  proxy.Do once bug.
-		Proxy: func(req *http.Request) (*url.URL, error) {
-			return httpproxy.FromEnvironment().ProxyFunc()(req.URL)
-		},
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-			DualStack: true,
-		}).DialContext,
-		MaxIdleConns:          10,
-		IdleConnTimeout:       30 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 5 * time.Second,
-	}
+	return httpdefaults.PatchTransport(cleanhttp.DefaultTransport())
 }

internal/pkg/containers/image/resolver_test.go

@@ -145,7 +145,7 @@ func (suite *ResolverSuite) TestRegistryHosts() {
 	suite.Assert().Equal("https", registryHosts[0].Scheme)
 	suite.Assert().Equal("registry-1.docker.io", registryHosts[0].Host)
 	suite.Assert().Equal("/v2", registryHosts[0].Path)
-	suite.Assert().Nil(registryHosts[0].Client.Transport.(*http.Transport).TLSClientConfig)
+	suite.Assert().Nil(registryHosts[0].Client.Transport.(*http.Transport).TLSClientConfig.Certificates)
 
 	cfg := &mockConfig{
 		mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
@@ -161,11 +161,11 @@ func (suite *ResolverSuite) TestRegistryHosts() {
 	suite.Assert().Equal("http", registryHosts[0].Scheme)
 	suite.Assert().Equal("127.0.0.1:5000", registryHosts[0].Host)
 	suite.Assert().Equal("/docker.io", registryHosts[0].Path)
-	suite.Assert().Nil(registryHosts[0].Client.Transport.(*http.Transport).TLSClientConfig)
+	suite.Assert().Nil(registryHosts[0].Client.Transport.(*http.Transport).TLSClientConfig.Certificates)
 	suite.Assert().Equal("https", registryHosts[1].Scheme)
 	suite.Assert().Equal("some.host", registryHosts[1].Host)
 	suite.Assert().Equal("/v2", registryHosts[1].Path)
-	suite.Assert().Nil(registryHosts[1].Client.Transport.(*http.Transport).TLSClientConfig)
+	suite.Assert().Nil(registryHosts[1].Client.Transport.(*http.Transport).TLSClientConfig.Certificates)
 
 	cfg = &mockConfig{
 		mirrors: map[string]*v1alpha1.RegistryMirrorConfig{

pkg/download/download.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/talos-systems/go-retry/retry"
+
+	"github.com/talos-systems/talos/pkg/httpdefaults"
 )
 
 const b64 = "base64"
@@ -167,7 +169,8 @@ func Download(ctx context.Context, endpoint string, opts ...Option) (b []byte, e
 }
 
 func download(req *http.Request, dlOpts *downloadOptions) (data []byte, err error) {
-	client := &http.Client{}
+	transport := httpdefaults.PatchTransport(cleanhttp.DefaultTransport())
+	transport.RegisterProtocol("tftp", NewTFTPTransport())
 
 	if dlOpts.LowSrcPort {
 		port := 100 + rand.Intn(512)
@@ -184,8 +187,11 @@ func download(req *http.Request, dlOpts *downloadOptions) (data []byte, err erro
 			LocalAddr: localTCPAddr,
 		}).DialContext
 
-		client.Transport = cleanhttp.DefaultTransport()
-		client.Transport.(*http.Transport).DialContext = d
+		transport.DialContext = d
+	}
+
+	client := &http.Client{
+		Transport: transport,
 	}
 
 	resp, err := client.Do(req)
@@ -214,8 +220,3 @@ func download(req *http.Request, dlOpts *downloadOptions) (data []byte, err erro
 
 	return data, nil
 }
-
-func init() {
-	transport := (http.DefaultTransport.(*http.Transport))
-	transport.RegisterProtocol("tftp", NewTFTPTransport())
-}

pkg/download/tftp.go

@@ -16,14 +16,12 @@ import (
 // NewTFTPTransport returns an http.RoundTripper capable of handling the TFTP
 // protocol.
 func NewTFTPTransport() http.RoundTripper {
-	return &tftpRoundTripper{}
+	return tftpRoundTripper{}
 }
 
-var _ http.RoundTripper = &tftpRoundTripper{}
-
 type tftpRoundTripper struct{}
 
-func (t *tftpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+func (t tftpRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
 	addr := req.URL.Host
 
 	if req.URL.Port() == "" {

pkg/httpdefaults/httpdefaults.go

@@ -0,0 +1,34 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package httpdefaults provides default HTTP client settings for Talos.
+package httpdefaults
+
+import (
+	"crypto/tls"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/net/http/httpproxy"
+)
+
+// PatchTransport updates *http.Transport with Talos-specific settings.
+//
+// Settings applied here only make sense when running in Talos root filesystem.
+func PatchTransport(transport *http.Transport) *http.Transport {
+	// Explicitly set the Proxy function to work around proxy.Do
+	// once: the environment variables will be reread/initialized each time the
+	// http call is made.
+	transport.Proxy = func(req *http.Request) (*url.URL, error) {
+		return httpproxy.FromEnvironment().ProxyFunc()(req.URL)
+	}
+
+	// Override the TLS config to allow refreshing CA list which might be updated
+	// via the machine config on the fly.
+	transport.TLSClientConfig = &tls.Config{
+		RootCAs: RootCAs(),
+	}
+
+	return transport
+}

pkg/httpdefaults/tls.go

@@ -0,0 +1,56 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package httpdefaults
+
+import (
+	"crypto/x509"
+	"io/fs"
+	"os"
+	"sync"
+
+	"github.com/talos-systems/talos/pkg/machinery/constants"
+)
+
+var (
+	cachedPool *x509.CertPool
+	cachedSt   fs.FileInfo
+	cacheMu    sync.Mutex
+)
+
+// RootCAs provides a cached, but refreshed, list of root CAs.
+//
+// If loading certificates fails for any reason, function returns nil.
+func RootCAs() *x509.CertPool {
+	st, err := os.Stat(constants.DefaultTrustedCAFile)
+	if err != nil {
+		return nil
+	}
+
+	// check if the file hasn't changed
+	cacheMu.Lock()
+	defer cacheMu.Unlock()
+
+	if cachedPool != nil && cachedSt != nil {
+		if cachedSt.ModTime().Equal(st.ModTime()) && cachedSt.Size() == st.Size() {
+			return cachedPool
+		}
+	}
+
+	pool := x509.NewCertPool()
+
+	contents, err := os.ReadFile(constants.DefaultTrustedCAFile)
+	if err == nil {
+		if pool.AppendCertsFromPEM(contents) {
+			cachedPool = pool
+			cachedSt = st
+		}
+	}
+
+	if cachedPool == nil {
+		return nil
+	}
+
+	return cachedPool.Clone()
+}

pkg/machinery/constants/constants.go

@@ -713,6 +713,9 @@ const (
 
 	// ServiceAccountMountPath is the path of the directory in which the Talos service account secrets are mounted.
 	ServiceAccountMountPath = "/var/run/secrets/talos.dev/"
+
+	// DefaultTrustedCAFile is the default path to the trusted CA file.
+	DefaultTrustedCAFile = "/etc/ssl/certs/ca-certificates"
 )
 
 // See https://linux.die.net/man/3/klogctl

website/content/v1.2/advanced/developing-talos.md

@@ -233,3 +233,69 @@ The IP address `172.20.0.2` is the address of the Talos node, and port `:9982` d
 - 9981: `apid`
 - 9982: `machined`
 - 9983: `trustd`
+
+## Testing Air-gapped Environments
+
+There is a hidden `talosctl debug air-gapped` command which launches two components:
+
+- HTTP proxy capable of proxying HTTP and HTTPS requests
+- HTTPS server with a self-signed certificate
+
+The command also writes down Talos machine configuration patch to enable the HTTP proxy and add a self-signed certificate
+to the list of trusted certificates:
+
+```shell
+$ talosctl debug air-gapped --advertised-address 172.20.0.1
+2022/08/04 16:43:14 writing config patch to air-gapped-patch.yaml
+2022/08/04 16:43:14 starting HTTP proxy on :8002
+2022/08/04 16:43:14 starting HTTPS server with self-signed cert on :8001
+```
+
+The `--advertised-address` should match the bridge IP of the Talos node.
+
+Generated machine configuration patch looks like:
+
+```yaml
+machine:
+    files:
+        - content: |
+            -----BEGIN CERTIFICATE-----
+            MIIBijCCAS+gAwIBAgIBATAKBggqhkjOPQQDAjAUMRIwEAYDVQQKEwlUZXN0IE9u
+            bHkwHhcNMjIwODA0MTI0MzE0WhcNMjIwODA1MTI0MzE0WjAUMRIwEAYDVQQKEwlU
+            ZXN0IE9ubHkwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQfOJdaOFSOI1I+EeP1
+            RlMpsDZJaXjFdoo5zYM5VYs3UkLyTAXAmdTi7JodydgLhty0pwLEWG4NUQAEvip6
+            EmzTo3IwcDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG
+            AQUFBwMCMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFCwxL+BjG0pDwaH8QgKW
+            Ex0J2mVXMA8GA1UdEQQIMAaHBKwUAAEwCgYIKoZIzj0EAwIDSQAwRgIhAJoW0z0D
+            JwpjFcgCmj4zT1SbBFhRBUX64PHJpAE8J+LgAiEAvfozZG8Or6hL21+Xuf1x9oh4
+            /4Hx3jozbSjgDyHOLk4=
+            -----END CERTIFICATE-----
+          permissions: 0o644
+          path: /etc/ssl/certs/ca-certificates
+          op: append
+    env:
+        http_proxy: http://172.20.0.1:8002
+        https_proxy: http://172.20.0.1:8002
+        no_proxy: 172.20.0.1/24
+cluster:
+    extraManifests:
+        - https://172.20.0.1:8001/debug.yaml
+```
+
+The first section appends a self-signed certificate of the HTTPS server to the list of trusted certificates,
+followed by the HTTP proxy setup (in-cluster traffic is excluded from the proxy).
+The last section adds an extra Kubernetes manifest hosted on the HTTPS server.
+
+The machine configuration patch can now be used to launch a test Talos cluster:
+
+```shell
+talosctl cluster create ... --config-patch @air-gapped-patch.yaml
+```
+
+The following lines should appear in the output of the `talosctl debug air-gapped` command:
+
+- `CONNECT discovery.talos.dev:443`: the HTTP proxy is used to talk to the discovery service
+- `http: TLS handshake error from 172.20.0.2:53512: remote error: tls: bad certificate`: an expected error on Talos side, as self-signed cert is not written yet to the file
+- `GET /debug.yaml`: Talos successfully fetches the extra manifest successfully
+
+There might be more output depending on the registry caches being used or not.