From dbdd2b237e0aefbba439b90472abf9ec7eea6aa6 Mon Sep 17 00:00:00 2001 From: Mateusz Urbanek Date: Fri, 3 Oct 2025 15:57:27 +0200 Subject: [PATCH] feat: add static registry to talosctl Fixes #11928 Fixes #11929 Signed-off-by: Mateusz Urbanek --- cmd/talosctl/cmd/talos/image.go | 95 +++++- cmd/talosctl/pkg/talos/helpers/flags.go | 14 - hack/release.toml | 15 + .../pkg/system/services/registry/params.go | 3 - .../pkg/system/services/registry/registry.go | 89 +++++- .../system/services/registry/registry_test.go | 2 +- internal/integration/cli/image.go | 28 ++ pkg/imager/cache/cache.go | 297 ++++++++++-------- website/content/v1.12/reference/cli.md | 42 ++- 9 files changed, 418 insertions(+), 167 deletions(-) diff --git a/cmd/talosctl/cmd/talos/image.go b/cmd/talosctl/cmd/talos/image.go index 53ee28069..63677c431 100644 --- a/cmd/talosctl/cmd/talos/image.go +++ b/cmd/talosctl/cmd/talos/image.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "os" + "os/signal" "slices" "strings" "text/tabwriter" @@ -20,9 +21,11 @@ import ( "github.com/dustin/go-humanize" "github.com/spf13/cobra" "github.com/spf13/pflag" + "go.uber.org/zap" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/artifacts" "github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers" + "github.com/siderolabs/talos/internal/app/machined/pkg/system/services/registry" "github.com/siderolabs/talos/pkg/imager/cache" "github.com/siderolabs/talos/pkg/images" "github.com/siderolabs/talos/pkg/machinery/api/common" @@ -168,14 +171,24 @@ var imageDefaultCmd = &cobra.Command{ }, } -var minimumVersion = semver.MustParse("1.11.0-alpha.0") +const ( + provisionerDocker = "docker" + provisionerInstaller = "installer" + provisionerAll = "all" +) + +var imageDefaultCmdFlags = struct { + provisioner pflag.Value +}{ + provisioner: helpers.StringChoice(provisionerInstaller, provisionerDocker, provisionerAll), +} // imageSourceBundleCmd represents the image source-bundle command. var imageSourceBundleCmd = &cobra.Command{ Use: "source-bundle ", Short: "List the source images used for building Talos", Long: ``, - Args: helpers.ChainCobraPositionalArgs( + Args: cobra.MatchAll( cobra.ExactArgs(1), func(cmd *cobra.Command, args []string) error { maximumVersion, err := semver.ParseTolerant(version.Tag) @@ -245,6 +258,8 @@ var imageSourceBundleCmd = &cobra.Command{ }, } +var minimumVersion = semver.MustParse("1.11.0-alpha.0") + // imageIntegrationCmd represents the integration image command. var imageIntegrationCmd = &cobra.Command{ Use: "integration", @@ -374,6 +389,7 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t imageCacheCreateCmdFlags.insecure, imageCacheCreateCmdFlags.imageLayerCachePath, imageCacheCreateCmdFlags.imageCachePath, + imageCacheCreateCmdFlags.layout.String() == layoutFlat, ) if err != nil { return fmt.Errorf("error generating cache: %w", err) @@ -383,27 +399,65 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t }, } -var imageCacheCreateCmdFlags struct { +const ( + layoutOCI = "oci" + layoutFlat = "flat" +) + +var imageCacheCreateCmdFlags = struct { imageCachePath string imageLayerCachePath string - platform string + layout pflag.Value + platform []string images []string insecure bool force bool +}{ + layout: helpers.StringChoice(layoutOCI, layoutFlat), } -const ( - provisionerDocker = "docker" - provisionerInstaller = "installer" - provisionerAll = "all" -) +// imageCacheServeCmd represents the image cache serve command. +var imageCacheServeCmd = &cobra.Command{ + Use: "cache-serve", + Short: "Serve an OCI image cache directory over HTTP(S) as a container registry", + Long: `Serve an OCI image cache directory over HTTP(S) as a container registry`, + Example: ``, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) + defer cancel() -var imageDefaultCmdFlags = struct { - provisioner pflag.Value -}{ - provisioner: helpers.StringChoice(provisionerInstaller, provisionerDocker, provisionerAll), + development, err := zap.NewDevelopment() + if err != nil { + return fmt.Errorf("failed to create development logger: %w", err) + } + + it := func(yield func(string) bool) { + for _, root := range []string{imageCacheServeCmdFlags.imageCachePath} { + if !yield(root) { + return + } + } + } + + return registry.NewService(registry.NewMultiPathFS(it), development).Run( + ctx, + registry.WithTLS( + imageCacheServeCmdFlags.tlsCertFile, + imageCacheServeCmdFlags.tlsKeyFile, + ), + registry.WithAddress(imageCacheServeCmdFlags.address), + ) + }, +} + +var imageCacheServeCmdFlags struct { + imageCachePath string + address string + tlsCertFile string + tlsKeyFile string } func init() { @@ -415,19 +469,28 @@ func init() { imageCmd.AddCommand(imageListCmd) imageCmd.AddCommand(imagePullCmd) - imageCmd.AddCommand(imageCacheCreateCmd) - imageCmd.AddCommand(imageIntegrationCmd) imageCmd.AddCommand(imageSourceBundleCmd) + imageCmd.AddCommand(imageCacheCreateCmd) imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.imageCachePath, "image-cache-path", "", "directory to save the image cache in OCI format") imageCacheCreateCmd.MarkPersistentFlagRequired("image-cache-path") //nolint:errcheck imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.imageLayerCachePath, "image-layer-cache-path", "", "directory to save the image layer cache") - imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.platform, "platform", "linux/amd64", "platform to use for the cache") + imageCacheCreateCmd.PersistentFlags().Var(imageCacheCreateCmdFlags.layout, "layout", + "Specifies the cache layout format: \"oci\" for an OCI image layout directory, or \"flat\" for a registry-like flat file structure") + imageCacheCreateCmd.PersistentFlags().StringSliceVar(&imageCacheCreateCmdFlags.platform, "platform", []string{"linux/amd64"}, "platform to use for the cache") imageCacheCreateCmd.PersistentFlags().StringSliceVar(&imageCacheCreateCmdFlags.images, "images", nil, "images to cache") imageCacheCreateCmd.MarkPersistentFlagRequired("images") //nolint:errcheck imageCacheCreateCmd.PersistentFlags().BoolVar(&imageCacheCreateCmdFlags.insecure, "insecure", false, "allow insecure registries") imageCacheCreateCmd.PersistentFlags().BoolVar(&imageCacheCreateCmdFlags.force, "force", false, "force overwrite of existing image cache") + imageCmd.AddCommand(imageCacheServeCmd) + imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.imageCachePath, "image-cache-path", "", "directory to save the image cache in OCI format") + imageCacheServeCmd.MarkPersistentFlagRequired("image-cache-path") //nolint:errcheck + imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.address, "address", constants.RegistrydListenAddress, "address to serve the registry on") + imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.tlsCertFile, "tls-cert-file", "", "TLS certificate file to use for serving") + imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.tlsKeyFile, "tls-key-file", "", "TLS key file to use for serving") + + imageCmd.AddCommand(imageIntegrationCmd) imageIntegrationCmd.PersistentFlags().StringVar(&imageIntegrationCmdFlags.installerTag, "installer-tag", "", "tag of the installer image to use") imageIntegrationCmd.MarkPersistentFlagRequired("installer-tag") //nolint:errcheck imageIntegrationCmd.PersistentFlags().StringVar(&imageIntegrationCmdFlags.registryAndUser, "registry-and-user", "", "registry and user to use for the images") diff --git a/cmd/talosctl/pkg/talos/helpers/flags.go b/cmd/talosctl/pkg/talos/helpers/flags.go index 2d94cc75c..636b7e0c5 100644 --- a/cmd/talosctl/pkg/talos/helpers/flags.go +++ b/cmd/talosctl/pkg/talos/helpers/flags.go @@ -8,7 +8,6 @@ import ( "fmt" "slices" - "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -52,16 +51,3 @@ func StringChoice(defaultValue string, otherChoices ...string) pflag.Value { }, } } - -// ChainCobraPositionalArgs chains multiple cobra.PositionalArgs validators together. -func ChainCobraPositionalArgs(validators ...cobra.PositionalArgs) cobra.PositionalArgs { - return func(cmd *cobra.Command, args []string) error { - for _, validator := range validators { - if err := validator(cmd, args); err != nil { - return err - } - } - - return nil - } -} diff --git a/hack/release.toml b/hack/release.toml index 338bb3fcb..e9206aa94 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -103,6 +103,21 @@ These fields were removed from the default machine configuration schema in v1.12 etcd container image is now pulled from `registry.k8s.io/etcd` instead of `gcr.io/etcd-development/etcd`. """ + [notes.talosctl] + title = "talosctl image cache-serve" + description = """\ +`talosctl` includes new subcommand `image cache-serve`. +It allows serving the created OCI image registry over HTTP/HTTPS. +It is a read-only registry, meaning images cannot be pushed to it, but the backing storage can be updated by re-running the `cache-create` command; + +Additionally `talosctl image cache-create` has some changes: + * new flag `--layout`: `oci` (_default_), `flat`: + * `oci` preserves current behavior; + * `flat` does not repack artifact layer, but moves it to a destination directory, allowing it to be served by `talosctl image cache-serve`; + * changed flag `--platform`: now can accept multiple os/arch combinations: + * comma separated (`--platform=linux/amd64,linux/arm64`); + * multiple instances (`--platform=linux/amd64 --platform=linux/arm64`); +""" [make_deps] diff --git a/internal/app/machined/pkg/system/services/registry/params.go b/internal/app/machined/pkg/system/services/registry/params.go index de9174f3e..50d588908 100644 --- a/internal/app/machined/pkg/system/services/registry/params.go +++ b/internal/app/machined/pkg/system/services/registry/params.go @@ -15,9 +15,6 @@ import ( func extractParams(req *http.Request) (params, error) { registry := req.URL.Query().Get("ns") - if registry == "" { - return params{}, xerrors.NewTaggedf[badRequestTag]("missing ns") - } value := req.PathValue("args") diff --git a/internal/app/machined/pkg/system/services/registry/registry.go b/internal/app/machined/pkg/system/services/registry/registry.go index e844a7cc9..96d7efb4a 100644 --- a/internal/app/machined/pkg/system/services/registry/registry.go +++ b/internal/app/machined/pkg/system/services/registry/registry.go @@ -44,8 +44,32 @@ type Service struct { root fs.StatFS } +type config struct { + addr string + tlsKeyPath string + tlsCertPath string +} + +// Option is a functional option for configuring the service. +type Option func(*config) + +// WithTLS enables TLS with the given certificate and key paths. +func WithTLS(certPath, keyPath string) Option { + return func(c *config) { + c.tlsCertPath = certPath + c.tlsKeyPath = keyPath + } +} + +// WithAddress sets the address to listen on. +func WithAddress(addr string) Option { + return func(c *config) { + c.addr = addr + } +} + // Run is an entrypoint to the API service. -func (svc *Service) Run(ctx context.Context) error { +func (svc *Service) Run(ctx context.Context, options ...Option) error { mux := http.NewServeMux() mux.HandleFunc("GET /v2/{args...}", svc.serveHTTP) @@ -56,7 +80,15 @@ func (svc *Service) Run(ctx context.Context) error { mux.HandleFunc("GET /"+p+"/{$}", giveOk) } - server := http.Server{Addr: constants.RegistrydListenAddress, Handler: mux} + cfg := &config{ + addr: constants.RegistrydListenAddress, + } + + for _, option := range options { + option(cfg) + } + + server := http.Server{Addr: cfg.addr, Handler: mux} errCh := make(chan error, 1) ctx, cancel := context.WithCancel(ctx) @@ -73,9 +105,18 @@ func (svc *Service) Run(ctx context.Context) error { svc.logger.Info("starting registry server", zap.String("addr", server.Addr)) - err := server.ListenAndServe() - if errors.Is(err, http.ErrServerClosed) { - err = nil + var err error + + if cfg.tlsCertPath != "" && cfg.tlsKeyPath != "" { + err = server.ListenAndServeTLS(cfg.tlsCertPath, cfg.tlsKeyPath) + if errors.Is(err, http.ErrServerClosed) { + err = nil + } + } else { + err = server.ListenAndServe() + if errors.Is(err, http.ErrServerClosed) { + err = nil + } } cancel() @@ -94,6 +135,7 @@ func (svc *Service) serveHTTP(w http.ResponseWriter, req *http.Request) { } } +//nolint:gocyclo func (svc *Service) handler(w http.ResponseWriter, req *http.Request) error { logger := svc.logger.With( zap.String("method", req.Method), @@ -114,6 +156,17 @@ func (svc *Service) handler(w http.ResponseWriter, req *http.Request) error { zap.String("registry", p.registry), ) + if p.registry == "" { + p.registry, err = svc.tryFindRegistry(p) + if err != nil { + return err + } + + if p.registry == "" { + return fmt.Errorf("failed to extract params: %w", xerrors.NewTaggedf[badRequestTag]("missing ns")) + } + } + ref, err := svc.resolveCanonicalRef(p) if err != nil { return err @@ -292,3 +345,29 @@ type ( badRequestTag struct{} internalErrorTag struct{} ) + +func (svc *Service) tryFindRegistry(p params) (string, error) { + entries, err := fs.ReadDir(svc.root, "manifests") + if err != nil { + return "", xerrors.NewTaggedf[internalErrorTag]("failed to read manifests directory: %w", err) + } + + for _, entry := range entries { + p.registry = entry.Name() + + ref, err := reference.ParseDockerRef(p.String()) + if err != nil { + return "", xerrors.NewTaggedf[badRequestTag]("failed to parse docker ref: %w", err) + } + + namedTaggedName := handleRegistryWithPort(ref, p) + + taggedFile := filepath.Join("manifests", namedTaggedName, "reference") + + if _, err := fs.Stat(svc.root, taggedFile); err == nil { + return entry.Name(), nil + } + } + + return "", nil +} diff --git a/internal/app/machined/pkg/system/services/registry/registry_test.go b/internal/app/machined/pkg/system/services/registry/registry_test.go index eb55af646..e4da27875 100644 --- a/internal/app/machined/pkg/system/services/registry/registry_test.go +++ b/internal/app/machined/pkg/system/services/registry/registry_test.go @@ -48,7 +48,7 @@ func TestRegistry(t *testing.T) { platform, err := v1.ParsePlatform("linux/amd64") assert.NoError(t, err) - assert.NoError(t, cache.Generate(images, platform.String(), false, "", cacheDir)) + assert.NoError(t, cache.Generate(images, []string{platform.String()}, false, "", cacheDir, false)) l, err := layout.ImageIndexFromPath(cacheDir) assert.NoError(t, err) diff --git a/internal/integration/cli/image.go b/internal/integration/cli/image.go index b19170929..9c82649f5 100644 --- a/internal/integration/cli/image.go +++ b/internal/integration/cli/image.go @@ -181,6 +181,34 @@ func (suite *ImageSuite) TestCacheCreate() { assert.FileExistsf(suite.T(), cacheDir+"/index.json", "index.json should exist in the image cache directory") } +// TestRegistryCreate verifies creating a registry cache. +func (suite *ImageSuite) TestRegistryCreate() { + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + stdOut, _ := suite.RunCLI([]string{"image", "source-bundle", "v1.11.2"}) + + imagesList := strings.Split(strings.Trim(stdOut, "\n"), "\n") + + imagesArgs := xslices.Map(imagesList[:2], func(image string) string { + return "--images=" + image + }) + + storageDir := suite.T().TempDir() + + args := []string{"image", "registry", "create", storageDir} + + args = append(args, imagesArgs...) + + suite.RunCLI(args, base.StdoutEmpty(), base.StderrNotEmpty()) + + assert.FileExistsf(suite.T(), storageDir+"pause/index.json", "pause/index.json should exist in the image cache directory") + assert.FileExistsf(suite.T(), storageDir+"pause/oci-layout", "pause/oci-layout should exist in the image cache directory") + assert.FileExistsf(suite.T(), storageDir+"siderolabs/kubelet/index.json", "siderolabs/kubelet/index.json should exist in the image cache directory") + assert.FileExistsf(suite.T(), storageDir+"siderolabs/kubelet/oci-layout", "siderolabs/kubelet/oci-layout should exist in the image cache directory") +} + func init() { allSuites = append(allSuites, new(ImageSuite)) } diff --git a/pkg/imager/cache/cache.go b/pkg/imager/cache/cache.go index ce142ae54..3552105f0 100644 --- a/pkg/imager/cache/cache.go +++ b/pkg/imager/cache/cache.go @@ -29,6 +29,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/siderolabs/go-retry/retry" "github.com/siderolabs/talos/pkg/imager/filemap" ) @@ -56,12 +57,7 @@ func rewriteRegistry(registryName, origRef string) string { // Generate generates a cache tarball from the given images. // //nolint:gocyclo,cyclop -func Generate(images []string, platform string, insecure bool, imageLayerCachePath, dest string) error { - v1Platform, err := v1.ParsePlatform(platform) - if err != nil { - return fmt.Errorf("parsing platform: %w", err) - } - +func Generate(images []string, platforms []string, insecure bool, imageLayerCachePath, dest string, flat bool) error { tmpDir, err := os.MkdirTemp("", "talos-image-cache-gen") if err != nil { return fmt.Errorf("creating temporary directory: %w", err) @@ -74,141 +70,71 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa } removeAll := sync.OnceValue(func() error { return os.RemoveAll(tmpDir) }) - defer removeAll() //nolint:errcheck - if err := os.MkdirAll(filepath.Join(tmpDir, blobsDir), 0o755); err != nil { - return err + if len(platforms) < 1 { + return fmt.Errorf("must specify at least one platform") } - var nameOptions []name.Option + for _, platform := range platforms { + v1Platform, err := v1.ParsePlatform(platform) + if err != nil { + return fmt.Errorf("parsing platform: %w", err) + } - craneOpts := []crane.Option{ - crane.WithAuthFromKeychain( - authn.NewMultiKeychain( + if err := os.MkdirAll(filepath.Join(tmpDir, blobsDir), 0o755); err != nil { + return err + } + + var nameOptions []name.Option + + craneOpts := []crane.Option{ + crane.WithAuthFromKeychain( + authn.NewMultiKeychain( + authn.DefaultKeychain, + github.Keychain, + google.Keychain, + ), + ), + } + + remoteOpts := []remote.Option{ + remote.WithAuthFromKeychain(authn.NewMultiKeychain( authn.DefaultKeychain, github.Keychain, google.Keychain, - ), - ), - } - - remoteOpts := []remote.Option{ - remote.WithAuthFromKeychain(authn.NewMultiKeychain( - authn.DefaultKeychain, - github.Keychain, - google.Keychain, - )), - remote.WithPlatform(*v1Platform), - } - - if insecure { - craneOpts = append(craneOpts, crane.Insecure) - nameOptions = append(nameOptions, name.Insecure) - } - - for _, src := range images { - fmt.Fprintf(os.Stderr, "fetching image %q\n", src) - - ref, err := name.ParseReference(src, nameOptions...) - if err != nil { - return fmt.Errorf("parsing reference %q: %w", src, err) + )), + remote.WithPlatform(*v1Platform), } - referenceDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "reference") - digestDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "digest") + if insecure { + craneOpts = append(craneOpts, crane.Insecure) + nameOptions = append(nameOptions, name.Insecure) + } - // if the reference was parsed as a tag, use it - tag, ok := ref.(name.Tag) + for _, src := range images { + r := retry.Exponential( + 30*time.Minute, + retry.WithUnits(time.Second), + retry.WithJitter(time.Second), + retry.WithErrorLogging(true), + ) - if !ok { - if base, _, ok := strings.Cut(src, "@"); ok { - // if the reference was a digest, but contained a tag, re-parse it - tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck + err := r.Retry(func() error { + if err := processImage(src, tmpDir, imageLayerCachePath, nameOptions, craneOpts, remoteOpts); err != nil { + return retry.ExpectedError(err) + } + + return nil + }) + if err != nil { + return fmt.Errorf("failed to prcess image: %w", err) } } + } - if err = os.MkdirAll(referenceDir, 0o755); err != nil { - return err - } - - if err = os.MkdirAll(digestDir, 0o755); err != nil { - return err - } - - manifest, err := crane.Manifest( - ref.String(), - craneOpts..., - ) - if err != nil { - return fmt.Errorf("fetching manifest %q: %w", ref.String(), err) - } - - rmt, err := remote.Get( - ref, - remoteOpts..., - ) - if err != nil { - return fmt.Errorf("fetching image %q: %w", ref.String(), err) - } - - if tag.TagStr() != "" { - if err := os.WriteFile(filepath.Join(referenceDir, tag.TagStr()), manifest, 0o644); err != nil { - return err - } - } - - if err := os.WriteFile(filepath.Join(digestDir, strings.ReplaceAll(rmt.Digest.String(), "sha256:", "sha256-")), manifest, 0o644); err != nil { - return err - } - - img, err := rmt.Image() - if err != nil { - return fmt.Errorf("converting image to index: %w", err) - } - - if imageLayerCachePath != "" { - img = cache.Image(img, cache.NewFilesystemCache(imageLayerCachePath)) - } - - layers, err := img.Layers() - if err != nil { - return fmt.Errorf("getting image layers: %w", err) - } - - config, err := img.RawConfigFile() - if err != nil { - return fmt.Errorf("getting image config: %w", err) - } - - platformManifest, err := img.RawManifest() - if err != nil { - return fmt.Errorf("getting image platform manifest: %w", err) - } - - h := sha256.New() - if _, err := h.Write(platformManifest); err != nil { - return fmt.Errorf("platform manifest hash: %w", err) - } - - if err := os.WriteFile(filepath.Join(digestDir, fmt.Sprintf("sha256-%x", h.Sum(nil))), platformManifest, 0o644); err != nil { - return err - } - - configHash, err := img.ConfigName() - if err != nil { - return fmt.Errorf("getting image config hash: %w", err) - } - - if err := os.WriteFile(filepath.Join(tmpDir, blobsDir, strings.ReplaceAll(configHash.String(), "sha256:", "sha256-")), config, 0o644); err != nil { - return err - } - - for _, layer := range layers { - if err = processLayer(layer, tmpDir); err != nil { - return err - } - } + if flat { + return os.Rename(tmpDir, dest) } newImg := mutate.MediaType(empty.Image, types.OCIManifestSchema1) @@ -240,13 +166,130 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa return fmt.Errorf("creating layout: %w", err) } - if err := ociLayout.AppendImage(newImg, layout.WithPlatform(*v1Platform)); err != nil { + imagePlatform, err := v1.ParsePlatform(platforms[0]) + if err != nil { + return fmt.Errorf("parsing platform: %w", err) + } + + if err := ociLayout.AppendImage(newImg, layout.WithPlatform(*imagePlatform)); err != nil { return fmt.Errorf("appending image: %w", err) } return removeAll() } +//nolint:gocyclo,cyclop +func processImage( + src, tmpDir, imageLayerCachePath string, + nameOptions []name.Option, + craneOpts []crane.Option, + remoteOpts []remote.Option, +) error { + fmt.Fprintf(os.Stderr, "fetching image %q\n", src) + + ref, err := name.ParseReference(src, nameOptions...) + if err != nil { + return fmt.Errorf("parsing reference %q: %w", src, err) + } + + referenceDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "reference") + digestDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "digest") + + // if the reference was parsed as a tag, use it + tag, ok := ref.(name.Tag) + + if !ok { + if base, _, ok := strings.Cut(src, "@"); ok { + // if the reference was a digest, but contained a tag, re-parse it + tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck + } + } + + if err = os.MkdirAll(referenceDir, 0o755); err != nil { + return err + } + + if err = os.MkdirAll(digestDir, 0o755); err != nil { + return err + } + + manifest, err := crane.Manifest( + ref.String(), + craneOpts..., + ) + if err != nil { + return fmt.Errorf("fetching manifest %q: %w", ref.String(), err) + } + + rmt, err := remote.Get( + ref, + remoteOpts..., + ) + if err != nil { + return fmt.Errorf("fetching image %q: %w", ref.String(), err) + } + + if tag.TagStr() != "" { + if err := os.WriteFile(filepath.Join(referenceDir, tag.TagStr()), manifest, 0o644); err != nil { + return err + } + } + + if err := os.WriteFile(filepath.Join(digestDir, strings.ReplaceAll(rmt.Digest.String(), "sha256:", "sha256-")), manifest, 0o644); err != nil { + return err + } + + img, err := rmt.Image() + if err != nil { + return fmt.Errorf("converting image to index: %w", err) + } + + if imageLayerCachePath != "" { + img = cache.Image(img, cache.NewFilesystemCache(imageLayerCachePath)) + } + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("getting image layers: %w", err) + } + + config, err := img.RawConfigFile() + if err != nil { + return fmt.Errorf("getting image config: %w", err) + } + + platformManifest, err := img.RawManifest() + if err != nil { + return fmt.Errorf("getting image platform manifest: %w", err) + } + + h := sha256.New() + if _, err := h.Write(platformManifest); err != nil { + return fmt.Errorf("platform manifest hash: %w", err) + } + + if err := os.WriteFile(filepath.Join(digestDir, fmt.Sprintf("sha256-%x", h.Sum(nil))), platformManifest, 0o644); err != nil { + return err + } + + configHash, err := img.ConfigName() + if err != nil { + return fmt.Errorf("getting image config hash: %w", err) + } + + if err := os.WriteFile(filepath.Join(tmpDir, blobsDir, strings.ReplaceAll(configHash.String(), "sha256:", "sha256-")), config, 0o644); err != nil { + return err + } + + for _, layer := range layers { + if err = processLayer(layer, tmpDir); err != nil { + return err + } + } + + return nil +} + func processLayer(layer v1.Layer, dstDir string) error { digest, err := layer.Digest() if err != nil { diff --git a/website/content/v1.12/reference/cli.md b/website/content/v1.12/reference/cli.md index a4ef296c2..930973f18 100644 --- a/website/content/v1.12/reference/cli.md +++ b/website/content/v1.12/reference/cli.md @@ -1870,7 +1870,46 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t --image-layer-cache-path string directory to save the image layer cache --images strings images to cache --insecure allow insecure registries - --platform string platform to use for the cache (default "linux/amd64") + --layout string Specifies the cache layout format: "oci" for an OCI image layout directory, or "flat" for a registry-like flat file structure (default "oci") + --platform strings platform to use for the cache (default [linux/amd64]) +``` + +### Options inherited from parent commands + +``` + --cluster string Cluster to connect to if a proxy endpoint is used. + --context string Context to be used in command + -e, --endpoints strings override default endpoints in Talos configuration + --namespace system namespace to use: system (etcd and kubelet images) or `cri` for all Kubernetes workloads (default "cri") + -n, --nodes strings target the specified nodes + --siderov1-keys-dir string The path to the SideroV1 auth PGP keys directory. Defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'. Only valid for Contexts that use SideroV1 auth. + --talosconfig string The path to the Talos configuration file. Defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order. +``` + +### SEE ALSO + +* [talosctl image](#talosctl-image) - Manage CRI container images + +## talosctl image cache-serve + +Serve an OCI image cache directory over HTTP(S) as a container registry + +### Synopsis + +Serve an OCI image cache directory over HTTP(S) as a container registry + +``` +talosctl image cache-serve [flags] +``` + +### Options + +``` + --address string address to serve the registry on (default "127.0.0.1:3172") + -h, --help help for cache-serve + --image-cache-path string directory to save the image cache in OCI format + --tls-cert-file string TLS certificate file to use for serving + --tls-key-file string TLS key file to use for serving ``` ### Options inherited from parent commands @@ -2031,6 +2070,7 @@ Manage CRI container images * [talosctl](#talosctl) - A CLI for out-of-band management of Kubernetes nodes created by Talos * [talosctl image cache-create](#talosctl-image-cache-create) - Create a cache of images in OCI format into a directory +* [talosctl image cache-serve](#talosctl-image-cache-serve) - Serve an OCI image cache directory over HTTP(S) as a container registry * [talosctl image default](#talosctl-image-default) - List the default images used by Talos * [talosctl image list](#talosctl-image-list) - List CRI images * [talosctl image pull](#talosctl-image-pull) - Pull an image into CRI