chore: support http downloads for assets in talosctl cluster create

This allows to pass direct URLs to Image Factory assets for disk
image/ISO/vmlinuz/initramfs, so that we can test Image Factory with
Talos.

Also add an integration test for Image Factory.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

.drone.jsonnet

@@ -465,6 +465,71 @@ local integration_qemu_trusted_boot = Step('e2e-qemu-trusted-boot', target='e2e-
   EXTRA_TEST_ARGS: '-talos.trustedboot',
 });
 
+local integration_factory_16_iso = Step('factory-1.6-iso', target='e2e-image-factory', privileged=true, depends_on=[load_artifacts], environment={
+  FACTORY_BOOT_METHOD: 'iso',
+  FACTORY_VERSION: 'v1.6.0',
+  FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  KUBERNETES_VERSION: '1.29.0',
+  FACTORY_UPGRADE: 'true',
+  FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f',
+  FACTORY_UPGRADE_VERSION: 'v1.6.1',
+});
+
+local integration_factory_16_image = Step('factory-1.6-image', depends_on=[integration_factory_16_iso], target='e2e-image-factory', privileged=true, environment={
+  FACTORY_BOOT_METHOD: 'disk-image',
+  FACTORY_VERSION: 'v1.6.0',
+  FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  KUBERNETES_VERSION: '1.29.0',
+  FACTORY_UPGRADE: 'true',
+  FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f',
+  FACTORY_UPGRADE_VERSION: 'v1.6.1',
+});
+
+local integration_factory_16_pxe = Step('factory-1.6-pxe', depends_on=[integration_factory_16_image], target='e2e-image-factory', privileged=true, environment={
+  FACTORY_BOOT_METHOD: 'pxe',
+  FACTORY_VERSION: 'v1.6.1',
+  FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  KUBERNETES_VERSION: '1.29.0',
+});
+
+local integration_factory_16_secureboot = Step('factory-1.6-secureboot', depends_on=[integration_factory_16_pxe], target='e2e-image-factory', privileged=true, environment={
+  FACTORY_BOOT_METHOD: 'secureboot-iso',
+  FACTORY_VERSION: 'v1.6.0',
+  FACTORY_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f',
+  KUBERNETES_VERSION: '1.29.0',
+  FACTORY_UPGRADE: 'true',
+  FACTORY_UPGRADE_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  FACTORY_UPGRADE_VERSION: 'v1.6.1',
+});
+
+local integration_factory_15_iso = Step('factory-1.5-iso', depends_on=[integration_factory_16_secureboot], target='e2e-image-factory', privileged=true, environment={
+  FACTORY_BOOT_METHOD: 'iso',
+  FACTORY_VERSION: 'v1.5.5',
+  FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  KUBERNETES_VERSION: '1.28.5',
+  FACTORY_UPGRADE: 'true',
+  FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f',
+  FACTORY_UPGRADE_VERSION: 'v1.5.5',
+});
+
+local integration_factory_13_iso = Step('factory-1.3-iso', depends_on=[integration_factory_15_iso], target='e2e-image-factory', privileged=true, environment={
+  FACTORY_BOOT_METHOD: 'iso',
+  FACTORY_VERSION: 'v1.3.7',
+  FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  KUBERNETES_VERSION: '1.26.5',
+  FACTORY_UPGRADE: 'true',
+  FACTORY_UPGRADE_SCHEMATIC: 'cf9b7aab9ed7c365d5384509b4d31c02fdaa06d2b3ac6cc0bc806f28130eff1f',
+  FACTORY_UPGRADE_VERSION: 'v1.3.7',
+});
+
+local integration_factory_13_image = Step('factory-1.3-image', depends_on=[integration_factory_13_iso], target='e2e-image-factory', privileged=true, environment={
+  FACTORY_BOOT_METHOD: 'disk-image',
+  FACTORY_VERSION: 'v1.3.7',
+  FACTORY_SCHEMATIC: '376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba',
+  KUBERNETES_VERSION: '1.26.5',
+});
+
+
 local build_race = Step('build-race', target='initramfs installer', depends_on=[load_artifacts], environment={ IMAGE_REGISTRY: local_registry, PUSH: true, TAG_SUFFIX: '-race', WITH_RACE: '1', PLATFORM: 'linux/amd64' });
 local integration_qemu_race = Step('e2e-qemu-race', target='e2e-qemu', privileged=true, depends_on=[build_race], environment={ IMAGE_REGISTRY: local_registry, TAG_SUFFIX: '-race' });
 
@@ -642,6 +707,15 @@ local integration_pipelines = [
   Pipeline('integration-images', default_pipeline_steps + [integration_images, integration_sbcs]) + integration_trigger(['integration-images']),
   Pipeline('integration-reproducibility-test', default_pipeline_steps + [integration_reproducibility_test]) + integration_trigger(['integration-reproducibility']),
   Pipeline('integration-cloud-images', default_pipeline_steps + [integration_images, integration_cloud_images]) + literal_trigger(['integration-cloud-images']),
+  Pipeline('image-factory', default_pipeline_steps + [
+    integration_factory_16_iso,
+    integration_factory_16_image,
+    integration_factory_16_pxe,
+    integration_factory_16_secureboot,
+    integration_factory_15_iso,
+    integration_factory_13_iso,
+    integration_factory_13_image,
+  ]) + literal_trigger(['image-factory']),
 
   // cron pipelines, triggered on schedule events
   Pipeline('cron-integration-qemu', default_pipeline_steps + [integration_qemu, push_edge], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
@@ -666,6 +740,16 @@ local integration_pipelines = [
   Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']),
   Pipeline('cron-integration-images', default_pipeline_steps + [integration_images, integration_sbcs], [default_cron_pipeline]) + cron_trigger(['nightly']),
   Pipeline('cron-integration-reproducibility-test', default_pipeline_steps + [integration_reproducibility_test], [default_cron_pipeline]) + cron_trigger(['nightly']),
+  Pipeline('cron-image-factory', default_pipeline_steps + [
+    integration_factory_16_iso,
+    integration_factory_16_image,
+    integration_factory_16_pxe,
+    integration_factory_16_secureboot,
+    integration_factory_15_iso,
+    integration_factory_13_iso,
+    integration_factory_13_image,
+  ],
+    [default_cron_pipeline]) + cron_trigger(['nightly']),
 ];
 
 

cmd/talosctl/cmd/mgmt/cluster/create.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"math/big"
 	"net/netip"
+	"net/url"
 	"os"
 	"path/filepath"
 	stdruntime "runtime"
@@ -18,6 +19,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/hashicorp/go-getter/v2"
 	"github.com/siderolabs/go-blockdevice/blockdevice/encryption"
 	"github.com/siderolabs/go-kubeconfig"
 	"github.com/siderolabs/go-procfs/procfs"
@@ -180,8 +182,82 @@ var createCmd = &cobra.Command{
 	},
 }
 
+func downloadBootAssets(ctx context.Context) error {
+	// download & cache images if provides as URLs
+	for _, downloadableImage := range []*string{
+		&nodeVmlinuzPath,
+		&nodeInitramfsPath,
+		&nodeISOPath,
+		&nodeDiskImagePath,
+	} {
+		if *downloadableImage == "" {
+			continue
+		}
+
+		u, err := url.Parse(*downloadableImage)
+		if err != nil || !(u.Scheme == "http" || u.Scheme == "https") {
+			// not a URL
+			continue
+		}
+
+		defaultStateDir, err := clientconfig.GetTalosDirectory()
+		if err != nil {
+			return err
+		}
+
+		cacheDir := filepath.Join(defaultStateDir, "cache")
+
+		if os.MkdirAll(cacheDir, 0o755) != nil {
+			return err
+		}
+
+		destPath := strings.ReplaceAll(
+			strings.ReplaceAll(u.String(), "/", "-"),
+			":", "-")
+
+		_, err = os.Stat(filepath.Join(cacheDir, destPath))
+		if err == nil {
+			*downloadableImage = filepath.Join(cacheDir, destPath)
+
+			// already cached
+			continue
+		}
+
+		fmt.Fprintf(os.Stderr, "downloading asset from %q to %q\n", u.String(), filepath.Join(cacheDir, destPath))
+
+		client := getter.Client{
+			Getters: []getter.Getter{
+				&getter.HttpGetter{
+					HeadFirstTimeout: 30 * time.Minute,
+					ReadTimeout:      30 * time.Minute,
+				},
+			},
+		}
+
+		_, err = client.Get(ctx, &getter.Request{
+			Src:     u.String(),
+			Dst:     filepath.Join(cacheDir, destPath),
+			GetMode: getter.ModeFile,
+		})
+		if err != nil {
+			// clean up the destination on failure
+			os.Remove(filepath.Join(cacheDir, destPath)) //nolint:errcheck
+
+			return err
+		}
+
+		*downloadableImage = filepath.Join(cacheDir, destPath)
+	}
+
+	return nil
+}
+
 //nolint:gocyclo,cyclop
-func create(ctx context.Context, flags *pflag.FlagSet) (err error) {
+func create(ctx context.Context, flags *pflag.FlagSet) error {
+	if err := downloadBootAssets(ctx); err != nil {
+		return err
+	}
+
 	if controlplanes < 1 {
 		return fmt.Errorf("number of controlplanes can't be less than 1")
 	}

hack/test/e2e-image-factory.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+set -eou pipefail
+
+# shellcheck source=/dev/null
+source ./hack/test/e2e.sh
+
+PROVISIONER=qemu
+CLUSTER_NAME="e2e-${PROVISIONER}"
+
+FACTORY_HOSTNAME=${FACTORY_HOSTNAME:-factory.talos.dev}
+PXE_FACTORY_HOSTNAME=${PXE_FACTORY_HOSTNAME:-pxe.factory.talos.dev}
+FACTORY_SCHEME=${FACTORY_SCHEME:-https}
+INSTALLER_IMAGE_NAME=${INSTALLER_IMAGE_NAME:-installer}
+
+case "${FACTORY_BOOT_METHOD:-iso}" in
+  iso)
+    QEMU_FLAGS+=("--iso-path=${FACTORY_SCHEME}://${FACTORY_HOSTNAME}/image/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64.iso")
+    ;;
+  disk-image)
+    QEMU_FLAGS+=("--disk-image-path=${FACTORY_SCHEME}://${FACTORY_HOSTNAME}/image/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64.raw.xz")
+    ;;
+  ipxe)
+    QEMU_FLAGS+=("--ipxe-boot-script=${FACTORY_SCHEME}://${PXE_FACTORY_HOSTNAME}/pxe/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64")
+    ;;
+  secureboot-iso)
+    QEMU_FLAGS+=("--iso-path=${FACTORY_SCHEME}://${FACTORY_HOSTNAME}/image/${FACTORY_SCHEMATIC}/${FACTORY_VERSION}/metal-amd64-secureboot.iso" "--with-tpm2" "--encrypt-ephemeral" "--encrypt-state" "--disk-encryption-key-types=tpm")
+    INSTALLER_IMAGE_NAME=installer-secureboot
+    ;;
+esac
+
+function assert_secureboot {
+  if [[ "${FACTORY_BOOT_METHOD:-iso}" != "secureboot-iso" ]]; then
+    return
+  fi
+
+  ${TALOSCTL} get securitystate -o json
+  ${TALOSCTL} get securitystate -o json | jq -e '.spec.secureBoot == true'
+}
+
+function create_cluster {
+  build_registry_mirrors
+
+  "${TALOSCTL}" cluster create \
+    --provisioner="${PROVISIONER}" \
+    --name="${CLUSTER_NAME}" \
+    --kubernetes-version="${KUBERNETES_VERSION}" \
+    --controlplanes=3 \
+    --workers="${QEMU_WORKERS:-1}" \
+    --disk=15360 \
+    --mtu=1450 \
+    --memory=2048 \
+    --memory-workers="${QEMU_MEMORY_WORKERS:-2048}" \
+    --cpus="${QEMU_CPUS:-2}" \
+    --cpus-workers="${QEMU_CPUS_WORKERS:-2}" \
+    --cidr=172.20.1.0/24 \
+    --cni-bundle-url="${ARTIFACTS}/talosctl-cni-bundle-\${ARCH}.tar.gz" \
+    --skip-injecting-config \
+    --with-apply-config \
+    --talos-version="${FACTORY_VERSION}" \
+    --install-image="${FACTORY_HOSTNAME}/${INSTALLER_IMAGE_NAME}/${FACTORY_SCHEMATIC}:${FACTORY_VERSION}" \
+    --crashdump \
+    "${REGISTRY_MIRROR_FLAGS[@]}" \
+    "${QEMU_FLAGS[@]}"
+
+    ${TALOSCTL} config node 172.20.1.2
+}
+
+function destroy_cluster() {
+  "${TALOSCTL}" cluster destroy --name "${CLUSTER_NAME}" --provisioner "${PROVISIONER}"
+}
+
+create_cluster
+
+${TALOSCTL} health --run-e2e
+${TALOSCTL} version | grep "${FACTORY_VERSION}"
+${TALOSCTL} get extensions | grep "${FACTORY_SCHEMATIC}"
+assert_secureboot
+
+if [[ "${FACTORY_UPGRADE:-false}" == "true" ]]; then
+    ${TALOSCTL} upgrade -i "${FACTORY_HOSTNAME}/${INSTALLER_IMAGE_NAME}/${FACTORY_UPGRADE_SCHEMATIC:-$FACTORY_SCHEMATIC}:${FACTORY_UPGRADE_VERSION:-$FACTORY_VERSION}"
+    ${TALOSCTL} version | grep "${FACTORY_UPGRADE_VERSION:-$FACTORY_VERSION}"
+    ${TALOSCTL} get extensions | grep "${FACTORY_UPGRADE_SCHEMATIC:-$FACTORY_SCHEMATIC}"
+    assert_secureboot
+fi
+
+destroy_cluster

hack/test/e2e.sh

@@ -217,7 +217,7 @@ function build_registry_mirrors {
     done
   else
     # use the value from the environment, if present
-    REGISTRY_MIRROR_FLAGS=("${REGISTRY_MIRROR_FLAGS:-}")
+    REGISTRY_MIRROR_FLAGS=(${REGISTRY_MIRROR_FLAGS:-})
   fi
 }