mirror of
https://github.com/siderolabs/talos.git
synced 2026-05-04 20:06:18 +02:00
feat: add static registry to talosctl
Fixes #11928 Fixes #11929 Signed-off-by: Mateusz Urbanek <mateusz.urbanek@siderolabs.com>
This commit is contained in:
parent
77d8cc7c58
commit
dbdd2b237e
@ -11,6 +11,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
"text/tabwriter"
|
||||
@ -20,9 +21,11 @@ import (
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/siderolabs/talos/cmd/talosctl/pkg/talos/artifacts"
|
||||
"github.com/siderolabs/talos/cmd/talosctl/pkg/talos/helpers"
|
||||
"github.com/siderolabs/talos/internal/app/machined/pkg/system/services/registry"
|
||||
"github.com/siderolabs/talos/pkg/imager/cache"
|
||||
"github.com/siderolabs/talos/pkg/images"
|
||||
"github.com/siderolabs/talos/pkg/machinery/api/common"
|
||||
@ -168,14 +171,24 @@ var imageDefaultCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var minimumVersion = semver.MustParse("1.11.0-alpha.0")
|
||||
const (
|
||||
provisionerDocker = "docker"
|
||||
provisionerInstaller = "installer"
|
||||
provisionerAll = "all"
|
||||
)
|
||||
|
||||
var imageDefaultCmdFlags = struct {
|
||||
provisioner pflag.Value
|
||||
}{
|
||||
provisioner: helpers.StringChoice(provisionerInstaller, provisionerDocker, provisionerAll),
|
||||
}
|
||||
|
||||
// imageSourceBundleCmd represents the image source-bundle command.
|
||||
var imageSourceBundleCmd = &cobra.Command{
|
||||
Use: "source-bundle <talos-version>",
|
||||
Short: "List the source images used for building Talos",
|
||||
Long: ``,
|
||||
Args: helpers.ChainCobraPositionalArgs(
|
||||
Args: cobra.MatchAll(
|
||||
cobra.ExactArgs(1),
|
||||
func(cmd *cobra.Command, args []string) error {
|
||||
maximumVersion, err := semver.ParseTolerant(version.Tag)
|
||||
@ -245,6 +258,8 @@ var imageSourceBundleCmd = &cobra.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var minimumVersion = semver.MustParse("1.11.0-alpha.0")
|
||||
|
||||
// imageIntegrationCmd represents the integration image command.
|
||||
var imageIntegrationCmd = &cobra.Command{
|
||||
Use: "integration",
|
||||
@ -374,6 +389,7 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t
|
||||
imageCacheCreateCmdFlags.insecure,
|
||||
imageCacheCreateCmdFlags.imageLayerCachePath,
|
||||
imageCacheCreateCmdFlags.imageCachePath,
|
||||
imageCacheCreateCmdFlags.layout.String() == layoutFlat,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error generating cache: %w", err)
|
||||
@ -383,27 +399,65 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t
|
||||
},
|
||||
}
|
||||
|
||||
var imageCacheCreateCmdFlags struct {
|
||||
const (
|
||||
layoutOCI = "oci"
|
||||
layoutFlat = "flat"
|
||||
)
|
||||
|
||||
var imageCacheCreateCmdFlags = struct {
|
||||
imageCachePath string
|
||||
imageLayerCachePath string
|
||||
platform string
|
||||
layout pflag.Value
|
||||
platform []string
|
||||
|
||||
images []string
|
||||
|
||||
insecure bool
|
||||
force bool
|
||||
}{
|
||||
layout: helpers.StringChoice(layoutOCI, layoutFlat),
|
||||
}
|
||||
|
||||
const (
|
||||
provisionerDocker = "docker"
|
||||
provisionerInstaller = "installer"
|
||||
provisionerAll = "all"
|
||||
)
|
||||
// imageCacheServeCmd represents the image cache serve command.
|
||||
var imageCacheServeCmd = &cobra.Command{
|
||||
Use: "cache-serve",
|
||||
Short: "Serve an OCI image cache directory over HTTP(S) as a container registry",
|
||||
Long: `Serve an OCI image cache directory over HTTP(S) as a container registry`,
|
||||
Example: ``,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt)
|
||||
defer cancel()
|
||||
|
||||
var imageDefaultCmdFlags = struct {
|
||||
provisioner pflag.Value
|
||||
}{
|
||||
provisioner: helpers.StringChoice(provisionerInstaller, provisionerDocker, provisionerAll),
|
||||
development, err := zap.NewDevelopment()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create development logger: %w", err)
|
||||
}
|
||||
|
||||
it := func(yield func(string) bool) {
|
||||
for _, root := range []string{imageCacheServeCmdFlags.imageCachePath} {
|
||||
if !yield(root) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return registry.NewService(registry.NewMultiPathFS(it), development).Run(
|
||||
ctx,
|
||||
registry.WithTLS(
|
||||
imageCacheServeCmdFlags.tlsCertFile,
|
||||
imageCacheServeCmdFlags.tlsKeyFile,
|
||||
),
|
||||
registry.WithAddress(imageCacheServeCmdFlags.address),
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
var imageCacheServeCmdFlags struct {
|
||||
imageCachePath string
|
||||
address string
|
||||
tlsCertFile string
|
||||
tlsKeyFile string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@ -415,19 +469,28 @@ func init() {
|
||||
|
||||
imageCmd.AddCommand(imageListCmd)
|
||||
imageCmd.AddCommand(imagePullCmd)
|
||||
imageCmd.AddCommand(imageCacheCreateCmd)
|
||||
imageCmd.AddCommand(imageIntegrationCmd)
|
||||
imageCmd.AddCommand(imageSourceBundleCmd)
|
||||
|
||||
imageCmd.AddCommand(imageCacheCreateCmd)
|
||||
imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.imageCachePath, "image-cache-path", "", "directory to save the image cache in OCI format")
|
||||
imageCacheCreateCmd.MarkPersistentFlagRequired("image-cache-path") //nolint:errcheck
|
||||
imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.imageLayerCachePath, "image-layer-cache-path", "", "directory to save the image layer cache")
|
||||
imageCacheCreateCmd.PersistentFlags().StringVar(&imageCacheCreateCmdFlags.platform, "platform", "linux/amd64", "platform to use for the cache")
|
||||
imageCacheCreateCmd.PersistentFlags().Var(imageCacheCreateCmdFlags.layout, "layout",
|
||||
"Specifies the cache layout format: \"oci\" for an OCI image layout directory, or \"flat\" for a registry-like flat file structure")
|
||||
imageCacheCreateCmd.PersistentFlags().StringSliceVar(&imageCacheCreateCmdFlags.platform, "platform", []string{"linux/amd64"}, "platform to use for the cache")
|
||||
imageCacheCreateCmd.PersistentFlags().StringSliceVar(&imageCacheCreateCmdFlags.images, "images", nil, "images to cache")
|
||||
imageCacheCreateCmd.MarkPersistentFlagRequired("images") //nolint:errcheck
|
||||
imageCacheCreateCmd.PersistentFlags().BoolVar(&imageCacheCreateCmdFlags.insecure, "insecure", false, "allow insecure registries")
|
||||
imageCacheCreateCmd.PersistentFlags().BoolVar(&imageCacheCreateCmdFlags.force, "force", false, "force overwrite of existing image cache")
|
||||
|
||||
imageCmd.AddCommand(imageCacheServeCmd)
|
||||
imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.imageCachePath, "image-cache-path", "", "directory to save the image cache in OCI format")
|
||||
imageCacheServeCmd.MarkPersistentFlagRequired("image-cache-path") //nolint:errcheck
|
||||
imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.address, "address", constants.RegistrydListenAddress, "address to serve the registry on")
|
||||
imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.tlsCertFile, "tls-cert-file", "", "TLS certificate file to use for serving")
|
||||
imageCacheServeCmd.PersistentFlags().StringVar(&imageCacheServeCmdFlags.tlsKeyFile, "tls-key-file", "", "TLS key file to use for serving")
|
||||
|
||||
imageCmd.AddCommand(imageIntegrationCmd)
|
||||
imageIntegrationCmd.PersistentFlags().StringVar(&imageIntegrationCmdFlags.installerTag, "installer-tag", "", "tag of the installer image to use")
|
||||
imageIntegrationCmd.MarkPersistentFlagRequired("installer-tag") //nolint:errcheck
|
||||
imageIntegrationCmd.PersistentFlags().StringVar(&imageIntegrationCmdFlags.registryAndUser, "registry-and-user", "", "registry and user to use for the images")
|
||||
|
||||
@ -8,7 +8,6 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
@ -52,16 +51,3 @@ func StringChoice(defaultValue string, otherChoices ...string) pflag.Value {
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ChainCobraPositionalArgs chains multiple cobra.PositionalArgs validators together.
|
||||
func ChainCobraPositionalArgs(validators ...cobra.PositionalArgs) cobra.PositionalArgs {
|
||||
return func(cmd *cobra.Command, args []string) error {
|
||||
for _, validator := range validators {
|
||||
if err := validator(cmd, args); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@ -103,6 +103,21 @@ These fields were removed from the default machine configuration schema in v1.12
|
||||
etcd container image is now pulled from `registry.k8s.io/etcd` instead of `gcr.io/etcd-development/etcd`.
|
||||
"""
|
||||
|
||||
[notes.talosctl]
|
||||
title = "talosctl image cache-serve"
|
||||
description = """\
|
||||
`talosctl` includes new subcommand `image cache-serve`.
|
||||
It allows serving the created OCI image registry over HTTP/HTTPS.
|
||||
It is a read-only registry, meaning images cannot be pushed to it, but the backing storage can be updated by re-running the `cache-create` command;
|
||||
|
||||
Additionally `talosctl image cache-create` has some changes:
|
||||
* new flag `--layout`: `oci` (_default_), `flat`:
|
||||
* `oci` preserves current behavior;
|
||||
* `flat` does not repack artifact layer, but moves it to a destination directory, allowing it to be served by `talosctl image cache-serve`;
|
||||
* changed flag `--platform`: now can accept multiple os/arch combinations:
|
||||
* comma separated (`--platform=linux/amd64,linux/arm64`);
|
||||
* multiple instances (`--platform=linux/amd64 --platform=linux/arm64`);
|
||||
"""
|
||||
|
||||
[make_deps]
|
||||
|
||||
|
||||
@ -15,9 +15,6 @@ import (
|
||||
|
||||
func extractParams(req *http.Request) (params, error) {
|
||||
registry := req.URL.Query().Get("ns")
|
||||
if registry == "" {
|
||||
return params{}, xerrors.NewTaggedf[badRequestTag]("missing ns")
|
||||
}
|
||||
|
||||
value := req.PathValue("args")
|
||||
|
||||
|
||||
@ -44,8 +44,32 @@ type Service struct {
|
||||
root fs.StatFS
|
||||
}
|
||||
|
||||
type config struct {
|
||||
addr string
|
||||
tlsKeyPath string
|
||||
tlsCertPath string
|
||||
}
|
||||
|
||||
// Option is a functional option for configuring the service.
|
||||
type Option func(*config)
|
||||
|
||||
// WithTLS enables TLS with the given certificate and key paths.
|
||||
func WithTLS(certPath, keyPath string) Option {
|
||||
return func(c *config) {
|
||||
c.tlsCertPath = certPath
|
||||
c.tlsKeyPath = keyPath
|
||||
}
|
||||
}
|
||||
|
||||
// WithAddress sets the address to listen on.
|
||||
func WithAddress(addr string) Option {
|
||||
return func(c *config) {
|
||||
c.addr = addr
|
||||
}
|
||||
}
|
||||
|
||||
// Run is an entrypoint to the API service.
|
||||
func (svc *Service) Run(ctx context.Context) error {
|
||||
func (svc *Service) Run(ctx context.Context, options ...Option) error {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("GET /v2/{args...}", svc.serveHTTP)
|
||||
@ -56,7 +80,15 @@ func (svc *Service) Run(ctx context.Context) error {
|
||||
mux.HandleFunc("GET /"+p+"/{$}", giveOk)
|
||||
}
|
||||
|
||||
server := http.Server{Addr: constants.RegistrydListenAddress, Handler: mux}
|
||||
cfg := &config{
|
||||
addr: constants.RegistrydListenAddress,
|
||||
}
|
||||
|
||||
for _, option := range options {
|
||||
option(cfg)
|
||||
}
|
||||
|
||||
server := http.Server{Addr: cfg.addr, Handler: mux}
|
||||
errCh := make(chan error, 1)
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
@ -73,9 +105,18 @@ func (svc *Service) Run(ctx context.Context) error {
|
||||
|
||||
svc.logger.Info("starting registry server", zap.String("addr", server.Addr))
|
||||
|
||||
err := server.ListenAndServe()
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
err = nil
|
||||
var err error
|
||||
|
||||
if cfg.tlsCertPath != "" && cfg.tlsKeyPath != "" {
|
||||
err = server.ListenAndServeTLS(cfg.tlsCertPath, cfg.tlsKeyPath)
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
err = nil
|
||||
}
|
||||
} else {
|
||||
err = server.ListenAndServe()
|
||||
if errors.Is(err, http.ErrServerClosed) {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
cancel()
|
||||
@ -94,6 +135,7 @@ func (svc *Service) serveHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:gocyclo
|
||||
func (svc *Service) handler(w http.ResponseWriter, req *http.Request) error {
|
||||
logger := svc.logger.With(
|
||||
zap.String("method", req.Method),
|
||||
@ -114,6 +156,17 @@ func (svc *Service) handler(w http.ResponseWriter, req *http.Request) error {
|
||||
zap.String("registry", p.registry),
|
||||
)
|
||||
|
||||
if p.registry == "" {
|
||||
p.registry, err = svc.tryFindRegistry(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if p.registry == "" {
|
||||
return fmt.Errorf("failed to extract params: %w", xerrors.NewTaggedf[badRequestTag]("missing ns"))
|
||||
}
|
||||
}
|
||||
|
||||
ref, err := svc.resolveCanonicalRef(p)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -292,3 +345,29 @@ type (
|
||||
badRequestTag struct{}
|
||||
internalErrorTag struct{}
|
||||
)
|
||||
|
||||
func (svc *Service) tryFindRegistry(p params) (string, error) {
|
||||
entries, err := fs.ReadDir(svc.root, "manifests")
|
||||
if err != nil {
|
||||
return "", xerrors.NewTaggedf[internalErrorTag]("failed to read manifests directory: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
p.registry = entry.Name()
|
||||
|
||||
ref, err := reference.ParseDockerRef(p.String())
|
||||
if err != nil {
|
||||
return "", xerrors.NewTaggedf[badRequestTag]("failed to parse docker ref: %w", err)
|
||||
}
|
||||
|
||||
namedTaggedName := handleRegistryWithPort(ref, p)
|
||||
|
||||
taggedFile := filepath.Join("manifests", namedTaggedName, "reference")
|
||||
|
||||
if _, err := fs.Stat(svc.root, taggedFile); err == nil {
|
||||
return entry.Name(), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", nil
|
||||
}
|
||||
|
||||
@ -48,7 +48,7 @@ func TestRegistry(t *testing.T) {
|
||||
platform, err := v1.ParsePlatform("linux/amd64")
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.NoError(t, cache.Generate(images, platform.String(), false, "", cacheDir))
|
||||
assert.NoError(t, cache.Generate(images, []string{platform.String()}, false, "", cacheDir, false))
|
||||
|
||||
l, err := layout.ImageIndexFromPath(cacheDir)
|
||||
assert.NoError(t, err)
|
||||
|
||||
@ -181,6 +181,34 @@ func (suite *ImageSuite) TestCacheCreate() {
|
||||
assert.FileExistsf(suite.T(), cacheDir+"/index.json", "index.json should exist in the image cache directory")
|
||||
}
|
||||
|
||||
// TestRegistryCreate verifies creating a registry cache.
|
||||
func (suite *ImageSuite) TestRegistryCreate() {
|
||||
if testing.Short() {
|
||||
suite.T().Skip("skipping in short mode")
|
||||
}
|
||||
|
||||
stdOut, _ := suite.RunCLI([]string{"image", "source-bundle", "v1.11.2"})
|
||||
|
||||
imagesList := strings.Split(strings.Trim(stdOut, "\n"), "\n")
|
||||
|
||||
imagesArgs := xslices.Map(imagesList[:2], func(image string) string {
|
||||
return "--images=" + image
|
||||
})
|
||||
|
||||
storageDir := suite.T().TempDir()
|
||||
|
||||
args := []string{"image", "registry", "create", storageDir}
|
||||
|
||||
args = append(args, imagesArgs...)
|
||||
|
||||
suite.RunCLI(args, base.StdoutEmpty(), base.StderrNotEmpty())
|
||||
|
||||
assert.FileExistsf(suite.T(), storageDir+"pause/index.json", "pause/index.json should exist in the image cache directory")
|
||||
assert.FileExistsf(suite.T(), storageDir+"pause/oci-layout", "pause/oci-layout should exist in the image cache directory")
|
||||
assert.FileExistsf(suite.T(), storageDir+"siderolabs/kubelet/index.json", "siderolabs/kubelet/index.json should exist in the image cache directory")
|
||||
assert.FileExistsf(suite.T(), storageDir+"siderolabs/kubelet/oci-layout", "siderolabs/kubelet/oci-layout should exist in the image cache directory")
|
||||
}
|
||||
|
||||
func init() {
|
||||
allSuites = append(allSuites, new(ImageSuite))
|
||||
}
|
||||
|
||||
297
pkg/imager/cache/cache.go
vendored
297
pkg/imager/cache/cache.go
vendored
@ -29,6 +29,7 @@ import (
|
||||
"github.com/google/go-containerregistry/pkg/v1/mutate"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
"github.com/google/go-containerregistry/pkg/v1/types"
|
||||
"github.com/siderolabs/go-retry/retry"
|
||||
|
||||
"github.com/siderolabs/talos/pkg/imager/filemap"
|
||||
)
|
||||
@ -56,12 +57,7 @@ func rewriteRegistry(registryName, origRef string) string {
|
||||
// Generate generates a cache tarball from the given images.
|
||||
//
|
||||
//nolint:gocyclo,cyclop
|
||||
func Generate(images []string, platform string, insecure bool, imageLayerCachePath, dest string) error {
|
||||
v1Platform, err := v1.ParsePlatform(platform)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing platform: %w", err)
|
||||
}
|
||||
|
||||
func Generate(images []string, platforms []string, insecure bool, imageLayerCachePath, dest string, flat bool) error {
|
||||
tmpDir, err := os.MkdirTemp("", "talos-image-cache-gen")
|
||||
if err != nil {
|
||||
return fmt.Errorf("creating temporary directory: %w", err)
|
||||
@ -74,141 +70,71 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa
|
||||
}
|
||||
|
||||
removeAll := sync.OnceValue(func() error { return os.RemoveAll(tmpDir) })
|
||||
|
||||
defer removeAll() //nolint:errcheck
|
||||
|
||||
if err := os.MkdirAll(filepath.Join(tmpDir, blobsDir), 0o755); err != nil {
|
||||
return err
|
||||
if len(platforms) < 1 {
|
||||
return fmt.Errorf("must specify at least one platform")
|
||||
}
|
||||
|
||||
var nameOptions []name.Option
|
||||
for _, platform := range platforms {
|
||||
v1Platform, err := v1.ParsePlatform(platform)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing platform: %w", err)
|
||||
}
|
||||
|
||||
craneOpts := []crane.Option{
|
||||
crane.WithAuthFromKeychain(
|
||||
authn.NewMultiKeychain(
|
||||
if err := os.MkdirAll(filepath.Join(tmpDir, blobsDir), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var nameOptions []name.Option
|
||||
|
||||
craneOpts := []crane.Option{
|
||||
crane.WithAuthFromKeychain(
|
||||
authn.NewMultiKeychain(
|
||||
authn.DefaultKeychain,
|
||||
github.Keychain,
|
||||
google.Keychain,
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
remoteOpts := []remote.Option{
|
||||
remote.WithAuthFromKeychain(authn.NewMultiKeychain(
|
||||
authn.DefaultKeychain,
|
||||
github.Keychain,
|
||||
google.Keychain,
|
||||
),
|
||||
),
|
||||
}
|
||||
|
||||
remoteOpts := []remote.Option{
|
||||
remote.WithAuthFromKeychain(authn.NewMultiKeychain(
|
||||
authn.DefaultKeychain,
|
||||
github.Keychain,
|
||||
google.Keychain,
|
||||
)),
|
||||
remote.WithPlatform(*v1Platform),
|
||||
}
|
||||
|
||||
if insecure {
|
||||
craneOpts = append(craneOpts, crane.Insecure)
|
||||
nameOptions = append(nameOptions, name.Insecure)
|
||||
}
|
||||
|
||||
for _, src := range images {
|
||||
fmt.Fprintf(os.Stderr, "fetching image %q\n", src)
|
||||
|
||||
ref, err := name.ParseReference(src, nameOptions...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing reference %q: %w", src, err)
|
||||
)),
|
||||
remote.WithPlatform(*v1Platform),
|
||||
}
|
||||
|
||||
referenceDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "reference")
|
||||
digestDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "digest")
|
||||
if insecure {
|
||||
craneOpts = append(craneOpts, crane.Insecure)
|
||||
nameOptions = append(nameOptions, name.Insecure)
|
||||
}
|
||||
|
||||
// if the reference was parsed as a tag, use it
|
||||
tag, ok := ref.(name.Tag)
|
||||
for _, src := range images {
|
||||
r := retry.Exponential(
|
||||
30*time.Minute,
|
||||
retry.WithUnits(time.Second),
|
||||
retry.WithJitter(time.Second),
|
||||
retry.WithErrorLogging(true),
|
||||
)
|
||||
|
||||
if !ok {
|
||||
if base, _, ok := strings.Cut(src, "@"); ok {
|
||||
// if the reference was a digest, but contained a tag, re-parse it
|
||||
tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck
|
||||
err := r.Retry(func() error {
|
||||
if err := processImage(src, tmpDir, imageLayerCachePath, nameOptions, craneOpts, remoteOpts); err != nil {
|
||||
return retry.ExpectedError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prcess image: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(referenceDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(digestDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := crane.Manifest(
|
||||
ref.String(),
|
||||
craneOpts...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching manifest %q: %w", ref.String(), err)
|
||||
}
|
||||
|
||||
rmt, err := remote.Get(
|
||||
ref,
|
||||
remoteOpts...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching image %q: %w", ref.String(), err)
|
||||
}
|
||||
|
||||
if tag.TagStr() != "" {
|
||||
if err := os.WriteFile(filepath.Join(referenceDir, tag.TagStr()), manifest, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(digestDir, strings.ReplaceAll(rmt.Digest.String(), "sha256:", "sha256-")), manifest, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
img, err := rmt.Image()
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting image to index: %w", err)
|
||||
}
|
||||
|
||||
if imageLayerCachePath != "" {
|
||||
img = cache.Image(img, cache.NewFilesystemCache(imageLayerCachePath))
|
||||
}
|
||||
|
||||
layers, err := img.Layers()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image layers: %w", err)
|
||||
}
|
||||
|
||||
config, err := img.RawConfigFile()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image config: %w", err)
|
||||
}
|
||||
|
||||
platformManifest, err := img.RawManifest()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image platform manifest: %w", err)
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := h.Write(platformManifest); err != nil {
|
||||
return fmt.Errorf("platform manifest hash: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(digestDir, fmt.Sprintf("sha256-%x", h.Sum(nil))), platformManifest, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configHash, err := img.ConfigName()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image config hash: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(tmpDir, blobsDir, strings.ReplaceAll(configHash.String(), "sha256:", "sha256-")), config, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, layer := range layers {
|
||||
if err = processLayer(layer, tmpDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if flat {
|
||||
return os.Rename(tmpDir, dest)
|
||||
}
|
||||
|
||||
newImg := mutate.MediaType(empty.Image, types.OCIManifestSchema1)
|
||||
@ -240,13 +166,130 @@ func Generate(images []string, platform string, insecure bool, imageLayerCachePa
|
||||
return fmt.Errorf("creating layout: %w", err)
|
||||
}
|
||||
|
||||
if err := ociLayout.AppendImage(newImg, layout.WithPlatform(*v1Platform)); err != nil {
|
||||
imagePlatform, err := v1.ParsePlatform(platforms[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing platform: %w", err)
|
||||
}
|
||||
|
||||
if err := ociLayout.AppendImage(newImg, layout.WithPlatform(*imagePlatform)); err != nil {
|
||||
return fmt.Errorf("appending image: %w", err)
|
||||
}
|
||||
|
||||
return removeAll()
|
||||
}
|
||||
|
||||
//nolint:gocyclo,cyclop
|
||||
func processImage(
|
||||
src, tmpDir, imageLayerCachePath string,
|
||||
nameOptions []name.Option,
|
||||
craneOpts []crane.Option,
|
||||
remoteOpts []remote.Option,
|
||||
) error {
|
||||
fmt.Fprintf(os.Stderr, "fetching image %q\n", src)
|
||||
|
||||
ref, err := name.ParseReference(src, nameOptions...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parsing reference %q: %w", src, err)
|
||||
}
|
||||
|
||||
referenceDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "reference")
|
||||
digestDir := filepath.Join(tmpDir, manifestsDir, rewriteRegistry(ref.Context().RegistryStr(), src), filepath.FromSlash(ref.Context().RepositoryStr()), "digest")
|
||||
|
||||
// if the reference was parsed as a tag, use it
|
||||
tag, ok := ref.(name.Tag)
|
||||
|
||||
if !ok {
|
||||
if base, _, ok := strings.Cut(src, "@"); ok {
|
||||
// if the reference was a digest, but contained a tag, re-parse it
|
||||
tag, _ = name.NewTag(base, nameOptions...) //nolint:errcheck
|
||||
}
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(referenceDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(digestDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifest, err := crane.Manifest(
|
||||
ref.String(),
|
||||
craneOpts...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching manifest %q: %w", ref.String(), err)
|
||||
}
|
||||
|
||||
rmt, err := remote.Get(
|
||||
ref,
|
||||
remoteOpts...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching image %q: %w", ref.String(), err)
|
||||
}
|
||||
|
||||
if tag.TagStr() != "" {
|
||||
if err := os.WriteFile(filepath.Join(referenceDir, tag.TagStr()), manifest, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(digestDir, strings.ReplaceAll(rmt.Digest.String(), "sha256:", "sha256-")), manifest, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
img, err := rmt.Image()
|
||||
if err != nil {
|
||||
return fmt.Errorf("converting image to index: %w", err)
|
||||
}
|
||||
|
||||
if imageLayerCachePath != "" {
|
||||
img = cache.Image(img, cache.NewFilesystemCache(imageLayerCachePath))
|
||||
}
|
||||
|
||||
layers, err := img.Layers()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image layers: %w", err)
|
||||
}
|
||||
|
||||
config, err := img.RawConfigFile()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image config: %w", err)
|
||||
}
|
||||
|
||||
platformManifest, err := img.RawManifest()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image platform manifest: %w", err)
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := h.Write(platformManifest); err != nil {
|
||||
return fmt.Errorf("platform manifest hash: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(digestDir, fmt.Sprintf("sha256-%x", h.Sum(nil))), platformManifest, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configHash, err := img.ConfigName()
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting image config hash: %w", err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(filepath.Join(tmpDir, blobsDir, strings.ReplaceAll(configHash.String(), "sha256:", "sha256-")), config, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, layer := range layers {
|
||||
if err = processLayer(layer, tmpDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processLayer(layer v1.Layer, dstDir string) error {
|
||||
digest, err := layer.Digest()
|
||||
if err != nil {
|
||||
|
||||
@ -1870,7 +1870,46 @@ talosctl images default | talosctl images cache-create --image-cache-path=/tmp/t
|
||||
--image-layer-cache-path string directory to save the image layer cache
|
||||
--images strings images to cache
|
||||
--insecure allow insecure registries
|
||||
--platform string platform to use for the cache (default "linux/amd64")
|
||||
--layout string Specifies the cache layout format: "oci" for an OCI image layout directory, or "flat" for a registry-like flat file structure (default "oci")
|
||||
--platform strings platform to use for the cache (default [linux/amd64])
|
||||
```
|
||||
|
||||
### Options inherited from parent commands
|
||||
|
||||
```
|
||||
--cluster string Cluster to connect to if a proxy endpoint is used.
|
||||
--context string Context to be used in command
|
||||
-e, --endpoints strings override default endpoints in Talos configuration
|
||||
--namespace system namespace to use: system (etcd and kubelet images) or `cri` for all Kubernetes workloads (default "cri")
|
||||
-n, --nodes strings target the specified nodes
|
||||
--siderov1-keys-dir string The path to the SideroV1 auth PGP keys directory. Defaults to 'SIDEROV1_KEYS_DIR' env variable if set, otherwise '$HOME/.talos/keys'. Only valid for Contexts that use SideroV1 auth.
|
||||
--talosconfig string The path to the Talos configuration file. Defaults to 'TALOSCONFIG' env variable if set, otherwise '$HOME/.talos/config' and '/var/run/secrets/talos.dev/config' in order.
|
||||
```
|
||||
|
||||
### SEE ALSO
|
||||
|
||||
* [talosctl image](#talosctl-image) - Manage CRI container images
|
||||
|
||||
## talosctl image cache-serve
|
||||
|
||||
Serve an OCI image cache directory over HTTP(S) as a container registry
|
||||
|
||||
### Synopsis
|
||||
|
||||
Serve an OCI image cache directory over HTTP(S) as a container registry
|
||||
|
||||
```
|
||||
talosctl image cache-serve [flags]
|
||||
```
|
||||
|
||||
### Options
|
||||
|
||||
```
|
||||
--address string address to serve the registry on (default "127.0.0.1:3172")
|
||||
-h, --help help for cache-serve
|
||||
--image-cache-path string directory to save the image cache in OCI format
|
||||
--tls-cert-file string TLS certificate file to use for serving
|
||||
--tls-key-file string TLS key file to use for serving
|
||||
```
|
||||
|
||||
### Options inherited from parent commands
|
||||
@ -2031,6 +2070,7 @@ Manage CRI container images
|
||||
|
||||
* [talosctl](#talosctl) - A CLI for out-of-band management of Kubernetes nodes created by Talos
|
||||
* [talosctl image cache-create](#talosctl-image-cache-create) - Create a cache of images in OCI format into a directory
|
||||
* [talosctl image cache-serve](#talosctl-image-cache-serve) - Serve an OCI image cache directory over HTTP(S) as a container registry
|
||||
* [talosctl image default](#talosctl-image-default) - List the default images used by Talos
|
||||
* [talosctl image list](#talosctl-image-list) - List CRI images
|
||||
* [talosctl image pull](#talosctl-image-pull) - Pull an image into CRI
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user