feat: implement extension services

Fixes #4694

User services run alongside with Talos system services.
Every user service container root filesystem should be already present
in the Talos root filesystem.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

.drone.jsonnet

@@ -358,13 +358,13 @@ local integration_provision_tests_track_0 = Step("provision-tests-track-0", priv
 local integration_provision_tests_track_1 = Step("provision-tests-track-1", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"IMAGE_REGISTRY": local_registry});
 local integration_provision_tests_track_2 = Step("provision-tests-track-2", privileged=true, depends_on=[integration_provision_tests_prepare], environment={"IMAGE_REGISTRY": local_registry});
 
-local integration_gvisor = Step("e2e-gvisor", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
+local integration_extensions = Step("e2e-extensions", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
         "SHORT_INTEGRATION_TEST": "yes",
-        "WITH_CONFIG_PATCH": '[{"op":"add","path":"/machine/install/extensions","value":[{"image":"ghcr.io/talos-systems/gvisor:54b831d"},{"image":"ghcr.io/talos-systems/intel-ucode:54b831d"}]},{"op":"add","path":"/machine/sysctls","value":{"user.max_user_namespaces": "11255"}}]',
-        "WITH_TEST": "run_gvisor_test",
+        "WITH_CONFIG_PATCH": '[{"op":"add","path":"/machine/install/extensions","value":[{"image":"ghcr.io/talos-systems/gvisor:54b831d"},{"image":"ghcr.io/talos-systems/intel-ucode:54b831d"},{"image":"ghcr.io/talos-systems/hello-world-service:a05f558"}]},{"op":"add","path":"/machine/sysctls","value":{"user.max_user_namespaces": "11255"}}]',
+        "WITH_TEST": "run_extensions_test",
         "IMAGE_REGISTRY": local_registry,
 });
-local integration_cilium = Step("e2e-cilium-1.9.10", target="e2e-qemu", privileged=true, depends_on=[integration_gvisor], environment={
+local integration_cilium = Step("e2e-cilium-1.9.10", target="e2e-qemu", privileged=true, depends_on=[integration_extensions], environment={
         "SHORT_INTEGRATION_TEST": "yes",
         "CUSTOM_CNI_URL": "https://raw.githubusercontent.com/cilium/cilium/v1.9.10/install/kubernetes/quick-install.yaml",
         "WITH_CONFIG_PATCH": '[{"op": "replace", "path": "/cluster/network/podSubnets", "value": ["10.0.0.0/8"]}]', # use Pod CIDRs as hardcoded in Cilium's quick-install
@@ -451,7 +451,8 @@ local integration_pipelines = [
   Pipeline('integration-provision-0', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_0]) + integration_trigger(['integration-provision', 'integration-provision-0']),
   Pipeline('integration-provision-1', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_1]) + integration_trigger(['integration-provision', 'integration-provision-1']),
   Pipeline('integration-provision-2', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_2]) + integration_trigger(['integration-provision', 'integration-provision-2']),
-  Pipeline('integration-misc', default_pipeline_steps + [integration_gvisor, integration_cilium, integration_uefi, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan]) + integration_trigger(['integration-misc']),
+  Pipeline('integration-misc', default_pipeline_steps + [integration_extensions
+, integration_cilium, integration_uefi, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan]) + integration_trigger(['integration-misc']),
   Pipeline('integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip]) + integration_trigger(['integration-qemu-encrypted-vip']),
   Pipeline('integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race]) + integration_trigger(['integration-qemu-race']),
   Pipeline('integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi]) + integration_trigger(['integration-qemu-csi']),
@@ -462,7 +463,8 @@ local integration_pipelines = [
   Pipeline('cron-integration-provision-0', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_0], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-provision-1', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_1], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-provision-2', default_pipeline_steps + [integration_provision_tests_prepare, integration_provision_tests_track_2], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
-  Pipeline('cron-integration-misc', default_pipeline_steps + [integration_gvisor, integration_cilium, integration_uefi, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
+  Pipeline('cron-integration-misc', default_pipeline_steps + [integration_extensions
+, integration_cilium, integration_uefi, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race], [default_cron_pipeline]) + cron_trigger(['nightly']),
   Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']),

Dockerfile

@@ -224,6 +224,7 @@ COPY --from=go-generate /src/pkg/machinery/resources/kubespan/ /pkg/machinery/re
 COPY --from=go-generate /src/pkg/machinery/resources/network/ /pkg/machinery/resources/network/
 COPY --from=go-generate /src/pkg/machinery/config/types/v1alpha1/ /pkg/machinery/config/types/v1alpha1/
 COPY --from=go-generate /src/pkg/machinery/nethelpers/ /pkg/machinery/nethelpers/
+COPY --from=go-generate /src/pkg/machinery/extensions/ /pkg/machinery/extensions/
 
 # The base target provides a container that can be used to build all Talos
 # assets.

hack/release.toml

@@ -129,6 +129,13 @@ Old behavior can be achieved by specifiying empty flag value: `--kubernetes-vers
         description="""\
 Pod Security Policy Kubernetes feature is deprecated and is going to be removed in Kubernetes 1.25.
 Talos by default skips setting up PSP now (see machine configuration `.cluster.apiServer.disablePodSecurityPolicy`).
+"""
+
+    [notes.extservices]
+        title = "Extension Services"
+        description = """\
+Talos now provides a way to extend set of system services Talos runs with extension services.
+Extension services should be included in the Talos root filesystem (e.g. via system extensions).
 """
 
 [make_deps]

hack/test/e2e.sh

@@ -212,10 +212,17 @@ function build_registry_mirrors {
   fi
 }
 
-function run_gvisor_test {
+function run_extensions_test {
+  echo "Testing gVsisor..."
   ${KUBECTL} apply -f ${PWD}/hack/test/gvisor/manifest.yaml
   sleep 10
   ${KUBECTL} wait --for=condition=ready pod nginx-gvisor --timeout=1m
+
+  echo "Testing firmware extension..."
+  ${TALOSCTL} ls -lr /lib/firmware | grep intel-ucode
+
+  echo "Testing extension service..."
+  curl http://172.20.1.2/ | grep Hello
 }
 
 function run_csi_tests {

internal/app/machined/pkg/controllers/k8s/kubelet_service.go

@@ -29,7 +29,6 @@ import (
 	"github.com/talos-systems/talos/pkg/machinery/resources/files"
 	"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
 	"github.com/talos-systems/talos/pkg/machinery/resources/secrets"
-	"github.com/talos-systems/talos/pkg/machinery/resources/v1alpha1"
 )
 
 // ServiceManager is the interface to the v1alpha1 services subsystems.
@@ -64,14 +63,8 @@ func (ctrl *KubeletServiceController) Outputs() []controller.Output {
 //
 //nolint:gocyclo,cyclop
 func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
-	// initially, wait for the cri to be up and for machine-id to be generated
+	// initially, wait for the machine-id to be generated
 	if err := r.UpdateInputs([]controller.Input{
-		{
-			Namespace: v1alpha1.NamespaceName,
-			Type:      v1alpha1.ServiceType,
-			ID:        pointer.ToString("cri"),
-			Kind:      controller.InputWeak,
-		},
 		{
 			Namespace: files.NamespaceName,
 			Type:      files.EtcFileStatusType,
@@ -98,18 +91,7 @@ func (ctrl *KubeletServiceController) Run(ctx context.Context, r controller.Runt
 			return fmt.Errorf("error getting etc file status: %w", err)
 		}
 
-		svc, err := r.Get(ctx, resource.NewMetadata(v1alpha1.NamespaceName, v1alpha1.ServiceType, "cri", resource.VersionUndefined))
-		if err != nil {
-			if state.IsNotFoundError(err) {
-				continue
-			}
-
-			return fmt.Errorf("error getting service: %w", err)
-		}
-
-		if svc.(*v1alpha1.Service).Healthy() && svc.(*v1alpha1.Service).Running() {
-			break
-		}
+		break
 	}
 
 	// normal reconcile loop, ignore cri state

internal/app/machined/pkg/controllers/runtime/extension_service.go

@@ -0,0 +1,131 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package runtime
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/cosi-project/runtime/pkg/controller"
+	"go.uber.org/zap"
+	"gopkg.in/yaml.v3"
+
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/services"
+	extservices "github.com/talos-systems/talos/pkg/machinery/extensions/services"
+)
+
+// ServiceManager is the interface to the v1alpha1 services subsystems.
+type ServiceManager interface {
+	Load(services ...system.Service) []string
+	Start(serviceIDs ...string) error
+}
+
+// ExtensionServiceController creates extension services based on the extension service configuration found in the rootfs.
+type ExtensionServiceController struct {
+	V1Alpha1Services ServiceManager
+	ConfigPath       string
+}
+
+// Name implements controller.Controller interface.
+func (ctrl *ExtensionServiceController) Name() string {
+	return "runtime.ExtensionServiceController"
+}
+
+// Inputs implements controller.Controller interface.
+func (ctrl *ExtensionServiceController) Inputs() []controller.Input {
+	return nil
+}
+
+// Outputs implements controller.Controller interface.
+func (ctrl *ExtensionServiceController) Outputs() []controller.Output {
+	return nil
+}
+
+// Run implements controller.Controller interface.
+//
+//nolint:gocyclo
+func (ctrl *ExtensionServiceController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error {
+	select {
+	case <-ctx.Done():
+		return nil
+	case <-r.EventCh():
+	}
+
+	// controller runs only once, as services are static
+	serviceFiles, err := os.ReadDir(ctrl.ConfigPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// directory not present, skip completely
+			logger.Debug("extension service directory is not found")
+
+			return nil
+		}
+
+		return err
+	}
+
+	extServices := map[string]struct{}{}
+
+	for _, serviceFile := range serviceFiles {
+		if filepath.Ext(serviceFile.Name()) != ".yaml" {
+			logger.Debug("skipping config file", zap.String("filename", serviceFile.Name()))
+
+			continue
+		}
+
+		spec, err := ctrl.loadSpec(filepath.Join(ctrl.ConfigPath, serviceFile.Name()))
+		if err != nil {
+			logger.Error("error loading extension service spec", zap.String("filename", serviceFile.Name()), zap.Error(err))
+
+			continue
+		}
+
+		if err = spec.Validate(); err != nil {
+			logger.Error("error validating extension service spec", zap.String("filename", serviceFile.Name()), zap.Error(err))
+
+			continue
+		}
+
+		if _, exists := extServices[spec.Name]; exists {
+			logger.Error("duplicate service spec", zap.String("filename", serviceFile.Name()), zap.String("name", spec.Name))
+
+			continue
+		}
+
+		extServices[spec.Name] = struct{}{}
+
+		svc := &services.Extension{
+			Spec: spec,
+		}
+
+		ctrl.V1Alpha1Services.Load(svc)
+
+		if err = ctrl.V1Alpha1Services.Start(svc.ID(nil)); err != nil {
+			return fmt.Errorf("error starting %q service: %w", spec.Name, err)
+		}
+	}
+
+	return nil
+}
+
+func (ctrl *ExtensionServiceController) loadSpec(path string) (*extservices.Spec, error) {
+	var spec extservices.Spec
+
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	defer f.Close() //nolint:errcheck
+
+	if err = yaml.NewDecoder(f).Decode(&spec); err != nil {
+		return nil, fmt.Errorf("error unmarshalling extension service config: %w", err)
+	}
+
+	return &spec, nil
+}

internal/app/machined/pkg/controllers/runtime/extension_service_test.go

@@ -0,0 +1,103 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+package runtime_test
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+	"github.com/talos-systems/go-retry/retry"
+
+	runtimecontrollers "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/runtime"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/services"
+)
+
+type ExtensionServiceSuite struct {
+	RuntimeSuite
+}
+
+type serviceMock struct {
+	mu       sync.Mutex
+	services map[string]system.Service
+}
+
+func (mock *serviceMock) Load(services ...system.Service) []string {
+	mock.mu.Lock()
+	defer mock.mu.Unlock()
+
+	ids := []string{}
+
+	for _, svc := range services {
+		mock.services[svc.ID(nil)] = svc
+		ids = append(ids, svc.ID(nil))
+	}
+
+	return ids
+}
+
+func (mock *serviceMock) Start(serviceIDs ...string) error {
+	return nil
+}
+
+func (mock *serviceMock) getIDs() []string {
+	mock.mu.Lock()
+	defer mock.mu.Unlock()
+
+	ids := []string{}
+
+	for id := range mock.services {
+		ids = append(ids, id)
+	}
+
+	sort.Strings(ids)
+
+	return ids
+}
+
+func (mock *serviceMock) get(id string) system.Service {
+	mock.mu.Lock()
+	defer mock.mu.Unlock()
+
+	return mock.services[id]
+}
+
+func (suite *ExtensionServiceSuite) TestReconcile() {
+	svcMock := &serviceMock{
+		services: map[string]system.Service{},
+	}
+
+	suite.Require().NoError(suite.runtime.RegisterController(&runtimecontrollers.ExtensionServiceController{
+		V1Alpha1Services: svcMock,
+		ConfigPath:       "testdata/extservices/",
+	}))
+
+	suite.startRuntime()
+
+	suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
+		func() error {
+			ids := svcMock.getIDs()
+
+			if !reflect.DeepEqual(ids, []string{"ext-hello-world"}) {
+				return retry.ExpectedError(fmt.Errorf("services registered: %q", ids))
+			}
+
+			return nil
+		},
+	))
+
+	helloSvc := svcMock.get("ext-hello-world")
+	suite.Require().IsType(&services.Extension{}, helloSvc)
+
+	suite.Assert().Equal("./hello-world", helloSvc.(*services.Extension).Spec.Container.Entrypoint)
+}
+
+func TestExtensionServiceSuite(t *testing.T) {
+	suite.Run(t, new(ExtensionServiceSuite))
+}

internal/app/machined/pkg/controllers/runtime/testdata/extservices/hello.yaml

@@ -0,0 +1,10 @@
+name: hello-world
+container:
+  entrypoint: ./hello-world
+  args:
+    - --msg
+    - Talos Linux Extension Service
+depends:
+  - network:
+    - addresses
+restart: always

internal/app/machined/pkg/controllers/runtime/testdata/extservices/invalid.yaml

@@ -0,0 +1,9 @@
+name: invalid
+container:
+  entrypoint: ./hello-world
+  args:
+    - --msg
+    - Talos Linux Extension Service
+depends:
+  - nothing: true
+restart: random

internal/app/machined/pkg/controllers/runtime/testdata/extservices/zduplicate.yaml

@@ -0,0 +1,9 @@
+name: hello-world
+container:
+  entrypoint: ./duplicate
+  args:
+    - should not get registered
+depends:
+  - network:
+    - addresses
+restart: always

internal/app/machined/pkg/runtime/v1alpha2/v1alpha2_controller.go

@@ -190,6 +190,10 @@ func (ctrl *Controller) Run(ctx context.Context, drainer *runtime.Drainer) error
 			Cmdline:        procfs.ProcCmdline(),
 			Drainer:        drainer,
 		},
+		&runtimecontrollers.ExtensionServiceController{
+			V1Alpha1Services: system.Services(ctrl.v1alpha1Runtime),
+			ConfigPath:       constants.ExtensionServicesConfigPath,
+		},
 		&runtimecontrollers.ExtensionStatusController{},
 		&runtimecontrollers.KernelModuleConfigController{},
 		&runtimecontrollers.KernelModuleSpecController{

internal/app/machined/pkg/system/service_events.go

@@ -7,6 +7,8 @@ package system
 import (
 	"context"
 	"fmt"
+	"sync"
+	"time"
 
 	"github.com/talos-systems/talos/pkg/conditions"
 )
@@ -22,6 +24,9 @@ const (
 )
 
 type serviceCondition struct {
+	mu              sync.Mutex
+	waitingRegister bool
+
 	event   StateEvent
 	service string
 }
@@ -32,9 +37,13 @@ func (sc *serviceCondition) Wait(ctx context.Context) error {
 	instance.mu.Unlock()
 
 	if svcrunner == nil {
-		return fmt.Errorf("service %q is not registered", sc.service)
+		return sc.waitRegister(ctx)
 	}
 
+	return sc.waitEvent(ctx, svcrunner)
+}
+
+func (sc *serviceCondition) waitEvent(ctx context.Context, svcrunner *ServiceRunner) error {
 	notifyCh := make(chan struct{}, 1)
 
 	svcrunner.Subscribe(sc.event, notifyCh)
@@ -48,11 +57,54 @@ func (sc *serviceCondition) Wait(ctx context.Context) error {
 	}
 }
 
+func (sc *serviceCondition) waitRegister(ctx context.Context) error {
+	sc.mu.Lock()
+	sc.waitingRegister = true
+	sc.mu.Unlock()
+
+	ticker := time.NewTicker(100 * time.Millisecond)
+	defer ticker.Stop()
+
+	var svcrunner *ServiceRunner
+
+	for {
+		instance.mu.Lock()
+		svcrunner = instance.state[sc.service]
+		instance.mu.Unlock()
+
+		if svcrunner != nil {
+			break
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+	}
+	sc.mu.Lock()
+	sc.waitingRegister = false
+	sc.mu.Unlock()
+
+	return sc.waitEvent(ctx, svcrunner)
+}
+
 func (sc *serviceCondition) String() string {
+	sc.mu.Lock()
+	waitingRegister := sc.waitingRegister
+	sc.mu.Unlock()
+
+	if waitingRegister {
+		return fmt.Sprintf("service %q to be registered", sc.service)
+	}
+
 	return fmt.Sprintf("service %q to be %q", sc.service, string(sc.event))
 }
 
 // WaitForService waits for service to reach some state event.
 func WaitForService(event StateEvent, service string) conditions.Condition {
-	return &serviceCondition{event, service}
+	return &serviceCondition{
+		event:   event,
+		service: service,
+	}
 }

internal/app/machined/pkg/system/services/extension.go

@@ -0,0 +1,158 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package services
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/oci"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+
+	"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/events"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/containerd"
+	"github.com/talos-systems/talos/internal/app/machined/pkg/system/runner/restart"
+	"github.com/talos-systems/talos/internal/pkg/capability"
+	"github.com/talos-systems/talos/internal/pkg/mount"
+	"github.com/talos-systems/talos/pkg/conditions"
+	"github.com/talos-systems/talos/pkg/machinery/constants"
+	extservices "github.com/talos-systems/talos/pkg/machinery/extensions/services"
+	"github.com/talos-systems/talos/pkg/machinery/resources/network"
+	"github.com/talos-systems/talos/pkg/machinery/resources/time"
+)
+
+// Extension service is a generic wrapper around extension services spec.
+type Extension struct {
+	Spec *extservices.Spec
+
+	overlay *mount.Point
+}
+
+// ID implements the Service interface.
+func (svc *Extension) ID(r runtime.Runtime) string {
+	return "ext-" + svc.Spec.Name
+}
+
+// PreFunc implements the Service interface.
+func (svc *Extension) PreFunc(ctx context.Context, r runtime.Runtime) error {
+	// re-mount service rootfs as overlay rw mount to allow containerd to mount there /dev, /proc, etc.
+	svc.overlay = mount.NewMountPoint(
+		"",
+		filepath.Join(constants.ExtensionServicesRootfsPath, svc.Spec.Name),
+		"",
+		0,
+		"",
+		mount.WithFlags(mount.Overlay|mount.SystemOverlay),
+	)
+
+	return svc.overlay.Mount()
+}
+
+// PostFunc implements the Service interface.
+func (svc *Extension) PostFunc(r runtime.Runtime, state events.ServiceState) (err error) {
+	return svc.overlay.Unmount()
+}
+
+// Condition implements the Service interface.
+func (svc *Extension) Condition(r runtime.Runtime) conditions.Condition {
+	conds := []conditions.Condition{}
+
+	for _, dep := range svc.Spec.Depends {
+		switch {
+		case dep.Service != "":
+			conds = append(conds, system.WaitForService(system.StateEventUp, dep.Service))
+		case dep.Path != "":
+			conds = append(conds, conditions.WaitForFileToExist(dep.Path))
+		case len(dep.Network) > 0:
+			conds = append(conds, network.NewReadyCondition(r.State().V1Alpha2().Resources(), network.StatusChecksFromStatuses(dep.Network...)...))
+		case dep.Time:
+			conds = append(conds, time.NewSyncCondition(r.State().V1Alpha2().Resources()))
+		}
+	}
+
+	if len(conds) == 0 {
+		return nil
+	}
+
+	return conditions.WaitForAll(conds...)
+}
+
+// DependsOn implements the Service interface.
+func (svc *Extension) DependsOn(r runtime.Runtime) []string {
+	return []string{"containerd"}
+}
+
+// Runner implements the Service interface.
+func (svc *Extension) Runner(r runtime.Runtime) (runner.Runner, error) {
+	args := runner.Args{
+		ID:          svc.ID(r),
+		ProcessArgs: append([]string{svc.Spec.Container.Entrypoint}, svc.Spec.Container.Args...),
+	}
+
+	for _, mount := range svc.Spec.Container.Mounts {
+		if err := os.MkdirAll(mount.Source, 0o700); err != nil {
+			return nil, err
+		}
+	}
+
+	env := []string{}
+	for key, val := range r.Config().Machine().Env() {
+		env = append(env, fmt.Sprintf("%s=%s", key, val))
+	}
+
+	var restartType restart.Type
+
+	switch svc.Spec.Restart {
+	case extservices.RestartAlways:
+		restartType = restart.Forever
+	case extservices.RestartNever:
+		restartType = restart.Once
+	case extservices.RestartUntilSuccess:
+		restartType = restart.UntilSuccess
+	}
+
+	return restart.New(containerd.NewRunner(
+		r.Config().Debug(),
+		&args,
+		runner.WithLoggingManager(r.Logging()),
+		runner.WithNamespace(constants.SystemContainerdNamespace),
+		runner.WithContainerdAddress(constants.SystemContainerdAddress),
+		runner.WithEnv(env),
+		runner.WithOCISpecOpts(
+			oci.WithRootFSPath(filepath.Join(constants.ExtensionServicesRootfsPath, svc.Spec.Name)),
+			oci.WithRootFSReadonly(),
+			oci.WithCgroup(constants.CgroupExtensions),
+			oci.WithMounts(svc.Spec.Container.Mounts),
+			oci.WithHostNamespace(specs.NetworkNamespace),
+			oci.WithSelinuxLabel(""),
+			oci.WithApparmorProfile(""),
+			oci.WithCapabilities(capability.AllGrantableCapabilities()),
+			oci.WithAllDevicesAllowed,
+		),
+		runner.WithOOMScoreAdj(-600),
+	),
+		restart.WithType(restartType),
+	), nil
+}
+
+// APIRestartAllowed implements APIRestartableService.
+func (svc *Extension) APIRestartAllowed(runtime.Runtime) bool {
+	return true
+}
+
+// APIStartAllowed implements APIStartableService.
+func (svc *Extension) APIStartAllowed(runtime.Runtime) bool {
+	return true
+}
+
+// APIStopAllowed implements APIStoppableService.
+func (svc *Extension) APIStopAllowed(runtime.Runtime) bool {
+	return true
+}

internal/pkg/mount/mount.go

@@ -405,8 +405,15 @@ func share(p *Point) error {
 func overlay(p *Point) error {
 	parts := strings.Split(p.target, "/")
 	prefix := strings.Join(parts[1:], "-")
-	diff := fmt.Sprintf(filepath.Join(constants.SystemOverlaysPath, "%s-diff"), prefix)
-	workdir := fmt.Sprintf(filepath.Join(constants.SystemOverlaysPath, "%s-workdir"), prefix)
+
+	basePath := constants.VarSystemOverlaysPath
+
+	if p.MountFlags.Check(SystemOverlay) {
+		basePath = constants.SystemOverlaysPath
+	}
+
+	diff := fmt.Sprintf(filepath.Join(basePath, "%s-diff"), prefix)
+	workdir := fmt.Sprintf(filepath.Join(basePath, "%s-workdir"), prefix)
 
 	for _, target := range []string{diff, workdir} {
 		if err := ensureDirectory(target); err != nil {

internal/pkg/mount/options.go

@@ -17,6 +17,10 @@ const (
 	// Overlay indicates that a the partition for a given mount point should be
 	// mounted using overlayfs.
 	Overlay
+	// SystemOverlay indicates that overlay directory should be created under tmpfs.
+	//
+	// SystemOverlay should be combined with Overlay option.
+	SystemOverlay
 	// ReadonlyOverlay indicates that a the partition for a given mount point should be
 	// mounted using multi-layer readonly overlay from multiple partitions given as sources.
 	ReadonlyOverlay

pkg/machinery/constants/constants.go

@@ -425,8 +425,8 @@ const (
 	// and directories.
 	SystemPath = "/system"
 
-	// SystemOverlaysPath is the path where overlay mounts are created.
-	SystemOverlaysPath = "/var/system/overlays"
+	// VarSystemOverlaysPath is the path where overlay mounts are created.
+	VarSystemOverlaysPath = "/var/system/overlays"
 
 	// SystemRunPath is the path to the system run directory.
 	SystemRunPath = SystemPath + "/run"
@@ -443,6 +443,9 @@ const (
 	// SystemExtensionsPath is the path to the system extensions directory.
 	SystemExtensionsPath = SystemPath + "/extensions"
 
+	// SystemOverlaysPath is the path to the system overlay directory.
+	SystemOverlaysPath = SystemPath + "/overlays"
+
 	// CgroupMountPath is the default mount path for unified cgroupsv2 setup.
 	CgroupMountPath = "/sys/fs/cgroup"
 
@@ -455,6 +458,9 @@ const (
 	// CgroupRuntime is the cgroup name for containerd runtime processes.
 	CgroupRuntime = CgroupSystem + "/runtime"
 
+	// CgroupExtensions is the cgroup name for system extension processes.
+	CgroupExtensions = CgroupSystem + "/extensions"
+
 	// CgroupPodRuntime is the cgroup name for kubernetes containerd runtime processes.
 	CgroupPodRuntime = "/podruntime/runtime"
 
@@ -598,6 +604,14 @@ const (
 
 	// FirmwarePath is the path to the standard Linux firmware location.
 	FirmwarePath = "/lib/firmware"
+
+	// ExtensionServicesConfigPath is the directory path which contains  configuration files of extension services.
+	//
+	// See pkg/machinery/extensions/services for the file format.
+	ExtensionServicesConfigPath = "/usr/local/etc/containers"
+
+	// ExtensionServicesRootfsPath is the path to the extracted rootfs files of extension services.
+	ExtensionServicesRootfsPath = "/usr/local/lib/containers"
 )
 
 // See https://linux.die.net/man/3/klogctl

pkg/machinery/extensions/services/restart_kind.go

@@ -0,0 +1,17 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package services
+
+//go:generate enumer -type=RestartKind -linecomment -text
+
+// RestartKind specifies how the service should be restarted.
+type RestartKind int
+
+// RestartKind constants.
+const (
+	RestartAlways       RestartKind = 1 // always
+	RestartNever        RestartKind = 2 // never
+	RestartUntilSuccess RestartKind = 3 // untilSuccess
+)

pkg/machinery/extensions/services/restartkind_enumer.go

@@ -0,0 +1,64 @@
+// Code generated by "enumer -type=RestartKind -linecomment -text"; DO NOT EDIT.
+
+//
+package services
+
+import (
+	"fmt"
+)
+
+const _RestartKindName = "alwaysneveruntilSuccess"
+
+var _RestartKindIndex = [...]uint8{0, 6, 11, 23}
+
+func (i RestartKind) String() string {
+	i -= 1
+	if i < 0 || i >= RestartKind(len(_RestartKindIndex)-1) {
+		return fmt.Sprintf("RestartKind(%d)", i+1)
+	}
+	return _RestartKindName[_RestartKindIndex[i]:_RestartKindIndex[i+1]]
+}
+
+var _RestartKindValues = []RestartKind{1, 2, 3}
+
+var _RestartKindNameToValueMap = map[string]RestartKind{
+	_RestartKindName[0:6]:   1,
+	_RestartKindName[6:11]:  2,
+	_RestartKindName[11:23]: 3,
+}
+
+// RestartKindString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func RestartKindString(s string) (RestartKind, error) {
+	if val, ok := _RestartKindNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to RestartKind values", s)
+}
+
+// RestartKindValues returns all values of the enum
+func RestartKindValues() []RestartKind {
+	return _RestartKindValues
+}
+
+// IsARestartKind returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i RestartKind) IsARestartKind() bool {
+	for _, v := range _RestartKindValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for RestartKind
+func (i RestartKind) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for RestartKind
+func (i *RestartKind) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = RestartKindString(string(text))
+	return err
+}

pkg/machinery/extensions/services/services.go

@@ -0,0 +1,136 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// Package services contains definitions for non-system services.
+package services
+
+import (
+	"fmt"
+	"path/filepath"
+	"regexp"
+
+	"github.com/hashicorp/go-multierror"
+	"github.com/opencontainers/runtime-spec/specs-go"
+
+	"github.com/talos-systems/talos/pkg/machinery/nethelpers"
+)
+
+// Spec is represents non-system service definition.
+type Spec struct {
+	// Name of the service to run, will be prefixed with `ext-` when registered as Talos service.
+	//
+	// Valid: [-_a-z0-9]+
+	Name string `yaml:"name"`
+	// Container to run.
+	//
+	// Container rootfs should be extracted to the /usr/local/lib/containers/<name>.
+	Container Container `yaml:"container"`
+	// Service dependencies.
+	Depends []Dependency `yaml:"depends"`
+	// Restart configuration.
+	Restart RestartKind `yaml:"restart"`
+}
+
+// Container specifies service container to run.
+type Container struct {
+	// Entrypoint for the service, relative to the container rootfs.
+	Entrypoint string `yaml:"entrypoint"`
+	// Args to pass to the entrypoint.
+	Args []string `yaml:"args"`
+	// Volume mounts.
+	Mounts []specs.Mount `yaml:"mounts"`
+}
+
+// Dependency describes a service Dependency.
+//
+// Only a single dependency out of the list might be specified.
+type Dependency struct {
+	// Depends on a service being running and healthy (if health checks are available).
+	Service string `yaml:"service,omitempty"`
+	// Depends on file/directory existence.
+	Path string `yaml:"path,omitempty"`
+	// Network readiness checks.
+	//
+	// Valid options are nethelpers.Status string values.
+	Network []nethelpers.Status `yaml:"network,omitempty"`
+	// Time sync check.
+	Time bool `yaml:"time,omitempty"`
+}
+
+var nameRe = regexp.MustCompile(`^[-_a-z0-9]{1,}$`)
+
+// Validate the service spec.
+func (spec *Spec) Validate() error {
+	var multiErr *multierror.Error
+
+	if !nameRe.MatchString(spec.Name) {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("name %q is invalid", spec.Name))
+	}
+
+	if !spec.Restart.IsARestartKind() {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("restart kind is invalid: %s", spec.Restart))
+	}
+
+	multiErr = multierror.Append(multiErr, spec.Container.Validate())
+
+	for _, dep := range spec.Depends {
+		multiErr = multierror.Append(multiErr, dep.Validate())
+	}
+
+	return multiErr.ErrorOrNil()
+}
+
+// Validate the container spec.
+func (ctr *Container) Validate() error {
+	var multiErr *multierror.Error
+
+	if ctr.Entrypoint == "" {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("container endpoint can't be empty"))
+	}
+
+	return multiErr.ErrorOrNil()
+}
+
+// Validate the dependency spec.
+func (dep *Dependency) Validate() error {
+	var multiErr *multierror.Error
+
+	nonZeroDeps := 0
+
+	if dep.Service != "" {
+		nonZeroDeps++
+	}
+
+	if dep.Path != "" {
+		nonZeroDeps++
+
+		if !filepath.IsAbs(dep.Path) {
+			multiErr = multierror.Append(multiErr, fmt.Errorf("path is not absolute: %q", dep.Path))
+		}
+	}
+
+	if len(dep.Network) > 0 {
+		nonZeroDeps++
+
+		for _, st := range dep.Network {
+			if !st.IsAStatus() {
+				multiErr = multierror.Append(multiErr, fmt.Errorf("invalid network dependency: %s", st))
+			}
+		}
+	}
+
+	if dep.Time {
+		nonZeroDeps++
+	}
+
+	if nonZeroDeps == 0 {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("no dependency specified"))
+	}
+
+	if nonZeroDeps > 1 {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("more than a single dependency is set"))
+	}
+
+	return multiErr.ErrorOrNil()
+}

pkg/machinery/extensions/services/services_test.go

@@ -0,0 +1,117 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package services_test
+
+import (
+	_ "embed"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	"github.com/talos-systems/talos/pkg/machinery/extensions/services"
+	"github.com/talos-systems/talos/pkg/machinery/nethelpers"
+)
+
+//go:embed "testdata/hello.yaml"
+var helloYAML []byte
+
+func TestUnmarshal(t *testing.T) {
+	var spec services.Spec
+
+	require.NoError(t, yaml.Unmarshal(helloYAML, &spec))
+
+	assert.Equal(t, services.Spec{
+		Name: "hello",
+		Container: services.Container{
+			Entrypoint: "hello-world",
+			Args:       []string{"--development", "--log=debug"},
+			Mounts: []specs.Mount{
+				{
+					Destination: "/var/lib/example",
+					Type:        "bind",
+					Source:      "/var/lib/example",
+					Options:     []string{"rbind", "ro"},
+				},
+			},
+		},
+		Depends: []services.Dependency{
+			{
+				Service: "cri",
+			},
+			{
+				Path: "/system/run/machined/machined.sock",
+			},
+			{
+				Network: []nethelpers.Status{nethelpers.StatusAddresses},
+			},
+		},
+		Restart: services.RestartNever,
+	}, spec)
+
+	assert.NoError(t, spec.Validate())
+}
+
+func TestValidate(t *testing.T) {
+	for _, tt := range []struct {
+		name          string
+		spec          services.Spec
+		expectedError string
+	}{
+		{
+			name:          "empty",
+			spec:          services.Spec{},
+			expectedError: "3 errors occurred:\n\t* name \"\" is invalid\n\t* restart kind is invalid: RestartKind(0)\n\t* container endpoint can't be empty\n\n",
+		},
+		{
+			name: "invalid name",
+			spec: services.Spec{
+				Name: "FOO",
+				Container: services.Container{
+					Entrypoint: "foo",
+				},
+				Restart: services.RestartAlways,
+			},
+			expectedError: "1 error occurred:\n\t* name \"FOO\" is invalid\n\n",
+		},
+		{
+			name: "invalid deps",
+			spec: services.Spec{
+				Name: "foo",
+				Container: services.Container{
+					Entrypoint: "foo",
+				},
+				Depends: []services.Dependency{
+					{},
+					{
+						Path: "./somefile",
+					},
+					{
+						Network: []nethelpers.Status{
+							0,
+						},
+					},
+					{
+						Network: []nethelpers.Status{
+							nethelpers.StatusAddresses,
+						},
+						Path: "/foo",
+					},
+				},
+				Restart: services.RestartAlways,
+			},
+			expectedError: "4 errors occurred:\n\t* no dependency specified\n\t* path is not absolute: \"./somefile\"\n\t* invalid network dependency: Status(0)\n\t* more than a single dependency is set\n\n",
+		},
+	} {
+		tt := tt
+
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.spec.Validate()
+			assert.EqualError(t, err, tt.expectedError)
+		})
+	}
+}

pkg/machinery/extensions/services/testdata/hello.yaml

@@ -0,0 +1,19 @@
+name: hello
+container:
+  entrypoint: hello-world
+  args:
+    - --development
+    - --log=debug
+  mounts:
+    - source: /var/lib/example
+      destination: /var/lib/example
+      type: bind
+      options:
+        - rbind
+        - ro
+depends:
+  - service: cri
+  - path: /system/run/machined/machined.sock
+  - network:
+    - addresses
+restart: never

pkg/machinery/nethelpers/status.go

@@ -0,0 +1,20 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package nethelpers
+
+//go:generate enumer -type=Status -linecomment -text
+
+// Status is a network status.
+//
+// Please see resources/network/status.go.
+type Status int
+
+// Status constants.
+const (
+	StatusAddresses    Status = 1 // addresses
+	StatusConnectivity Status = 2 // connectivity
+	StatusHostname     Status = 3 // hostname
+	StatusEtcFiles     Status = 4 // etcfiles
+)

pkg/machinery/nethelpers/status_enumer.go

@@ -0,0 +1,65 @@
+// Code generated by "enumer -type=Status -linecomment -text"; DO NOT EDIT.
+
+//
+package nethelpers
+
+import (
+	"fmt"
+)
+
+const _StatusName = "addressesconnectivityhostnameetcfiles"
+
+var _StatusIndex = [...]uint8{0, 9, 21, 29, 37}
+
+func (i Status) String() string {
+	i -= 1
+	if i < 0 || i >= Status(len(_StatusIndex)-1) {
+		return fmt.Sprintf("Status(%d)", i+1)
+	}
+	return _StatusName[_StatusIndex[i]:_StatusIndex[i+1]]
+}
+
+var _StatusValues = []Status{1, 2, 3, 4}
+
+var _StatusNameToValueMap = map[string]Status{
+	_StatusName[0:9]:   1,
+	_StatusName[9:21]:  2,
+	_StatusName[21:29]: 3,
+	_StatusName[29:37]: 4,
+}
+
+// StatusString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StatusString(s string) (Status, error) {
+	if val, ok := _StatusNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to Status values", s)
+}
+
+// StatusValues returns all values of the enum
+func StatusValues() []Status {
+	return _StatusValues
+}
+
+// IsAStatus returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i Status) IsAStatus() bool {
+	for _, v := range _StatusValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for Status
+func (i Status) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for Status
+func (i *Status) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = StatusString(string(text))
+	return err
+}

pkg/machinery/resources/network/condition.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/cosi-project/runtime/pkg/resource"
 	"github.com/cosi-project/runtime/pkg/state"
+
+	"github.com/talos-systems/talos/pkg/machinery/nethelpers"
 )
 
 // ReadyCondition implements condition which waits for the network to be ready.
@@ -76,3 +78,23 @@ func HostnameReady(spec *StatusSpec) bool {
 func EtcFilesReady(spec *StatusSpec) bool {
 	return spec.EtcFilesReady
 }
+
+// StatusChecksFromStatuses converts nethelpers.Status list into list of checks.
+func StatusChecksFromStatuses(statuses ...nethelpers.Status) []StatusCheck {
+	checks := make([]StatusCheck, 0, len(statuses))
+
+	for _, st := range statuses {
+		switch st {
+		case nethelpers.StatusAddresses:
+			checks = append(checks, AddressReady)
+		case nethelpers.StatusConnectivity:
+			checks = append(checks, ConnectivityReady)
+		case nethelpers.StatusHostname:
+			checks = append(checks, HostnameReady)
+		case nethelpers.StatusEtcFiles:
+			checks = append(checks, EtcFilesReady)
+		}
+	}
+
+	return checks
+}

website/content/docs/v0.15/Learn More/architecture.md

@@ -1,6 +1,6 @@
 ---
 title: "Architecture"
-weight: 3
+weight: 30
 ---
 
 Talos is designed to be **atomic** in _deployment_ and **modular** in _composition_.

website/content/docs/v0.15/Learn More/components.md

@@ -1,6 +1,6 @@
 ---
 title: "Components"
-weight: 4
+weight: 40
 ---
 
 In this section, we discuss the various components that underpin Talos.

website/content/docs/v0.15/Learn More/concepts.md

@@ -1,6 +1,6 @@
 ---
 title: "Concepts"
-weight: 2
+weight: 20
 ---
 
 When people come across Talos, they frequently want a nice, bite-sized summary

website/content/docs/v0.15/Learn More/control-plane.md

@@ -1,6 +1,6 @@
 ---
 title: "Control Plane"
-weight: 8
+weight: 80
 ---
 
 This guide provides details on how Talos runs and bootstraps the Kubernetes control plane.

website/content/docs/v0.15/Learn More/controllers-resources.md

@@ -1,6 +1,6 @@
 ---
 title: "Controllers and Resources"
-weight: 9
+weight: 90
 ---
 
 <!-- markdownlint-disable MD038 -->

website/content/docs/v0.15/Learn More/developing-talos.md

@@ -1,6 +1,6 @@
 ---
 title: "Developing Talos"
-weight: 13
+weight: 130
 ---
 
 This guide outlines steps and tricks to develop Talos operating systems and related components.

website/content/docs/v0.15/Learn More/discovery.md

@@ -1,6 +1,6 @@
 ---
 title: "Discovery"
-weight: 11
+weight: 110
 ---
 
 We maintain a public discovery service whereby members of your cluster can use a common and unique key to coordinate the most basic connection information (i.e. the set of possible "endpoints", or IP:port pairs).

website/content/docs/v0.15/Learn More/extension-services.md

@@ -0,0 +1,151 @@
+---
+title: "Extension Services"
+weight: 105
+---
+
+Talos provides a way to run additional system services early in the Talos boot process.
+Extension services should be included into the Talos root filesystem (e.g. using [system extensions](../../guides/system-extensions/)).
+Extension services run as privileged containers with ephemeral root filesystem located in the Talos root filesystem.
+
+Extension services can be used to use extend core features of Talos in a way that is not possible via [static pods](../../guides/static-pods) or
+Kubernetes DaemonSets.
+
+Potential extension services use-cases:
+
+* storage: Open iSCSI, software RAID, etc.
+* networking: BGP FRR, etc.
+* platform integration: VMWare open VM tools, etc.
+
+## Configuration
+
+Talos on boot scans directory `/usr/local/etc/containers` for `*.yaml` files describing the extension services to run.
+Format of the extension service config:
+
+```yaml
+name: hello-world
+container:
+  entrypoint: ./hello-world
+  args:
+     - -f
+  mounts:
+     - # OCI Mount Spec
+depends:
+   - service: cri
+   - file: /run/machined/machined.sock
+   - network:
+       - address
+       - connectivity
+       - hostname
+       - etcfiles
+   - time: true
+restart: never|always|untilSuccess
+```
+
+### `name`
+
+Field `name` sets the service name, valid names are `[a-z0-9-_]+`.
+The service container root filesystem path is derived from the `name`: `/usr/local/lib/containers/<name>`.
+The extension service will be registered as a Talos service under an `ext-<name>` identifier.
+
+### `container`
+
+* `entrypoint` defines the container entrypoint relative to the container root filesystem (`/usr/local/lib/containers/<name>`)
+* `args` defines the additional arguments to pass to the entrypoint
+* `mounts` defines the volumes to be mounted into the container root
+
+#### `container.mounts`
+
+The section `mounts` uses the standard OCI spec:
+
+```yaml
+- source: /var/log/audit
+  destination: /var/log/audit
+  type: bind
+  options:
+    - rshared
+    - bind
+    - ro
+```
+
+All requested directories will be mounted into the extension service container mount namespace.
+If the `source` directory doesn't exist in the host filesystem, it will be created (only for writable paths in the Talos root filesystem).
+
+### `depends`
+
+The `depends` section describes extension service start dependencies: the service will not be started until all dependencies are met.
+
+Available dependencies:
+
+* `service: <name>`: wait for the service `<name>` to be running and healthy
+* `file: <path>`: wait for the `<path>` to exist
+* `network: [address, connectivity, hostname, etcfiles]`: wait for the specified network readiness checks to succeed
+* `time: true`: wait for the NTP time sync
+
+### `restart`
+
+Field `restart` defines the service restart policy, it allows to either configure an always running service or a one-shot service:
+
+* `always`: restart service always
+* `never`: start service only once and never restart
+* `untilSuccess`: restart failing service, stop restarting on successful run
+
+## Example
+
+Example layout of the Talos root filesystem contents for the extension service:
+
+```text
+/
+└── usr
+    └── local
+        ├── etc
+        │   └── containers
+        │       └── hello-world.yaml
+        └── lib
+            └── containers
+                └── hello-world
+                    ├── hello
+                    └── config.ini
+```
+
+Talos discovers the extension service configuration in `/usr/local/etc/containers/hello-world.yaml`:
+
+```yaml
+name: hello-world
+container:
+  entrypoint: ./hello
+  args:
+    - --config
+    - config.ini
+depends:
+  - network:
+    - addresses
+restart: always
+```
+
+Talos starts the container for the extension service with container root filesystem at `/usr/local/lib/containers/hello-world`:
+
+```text
+/
+├── hello
+└── config.ini
+```
+
+Extension service is registered as `ext-hello-world` in `talosctl services`:
+
+```shell
+$ talosctl service ext-hello-world
+NODE     172.20.0.5
+ID       ext-hello-world
+STATE    Running
+HEALTH   ?
+EVENTS   [Running]: Started task ext-hello-world (PID 1100) for container ext-hello-world (2m47s ago)
+         [Preparing]: Creating service runner (2m47s ago)
+         [Preparing]: Running pre state (2m47s ago)
+         [Waiting]: Waiting for service "containerd" to be "up" (2m48s ago)
+         [Waiting]: Waiting for service "containerd" to be "up", network (2m49s ago)
+```
+
+An extension service can be started, restarted and stopped using `talosctl service ext-hello-world start|restart|stop`.
+Use `talosctl logs ext-hello-world` to get the logs of the service.
+
+Complete example of the extension service can be found in the [extensions repository](https://github.com/talos-systems/extensions/tree/main/examples/hello-world-service).

website/content/docs/v0.15/Learn More/faqs.md

@@ -1,6 +1,6 @@
 ---
 title: "FAQs"
-weight: 6
+weight: 60
 ---
 
 <!-- markdownlint-disable MD026 -->

website/content/docs/v0.15/Learn More/kubespan.md

@@ -1,6 +1,6 @@
 ---
 title: "KubeSpan"
-weight: 12
+weight: 120
 ---
 
 ## WireGuard Peer Discovery

website/content/docs/v0.15/Learn More/networking-resources.md

@@ -1,6 +1,6 @@
 ---
 title: "Networking Resources"
-weight: 10
+weight: 100
 ---
 
 Starting with version 0.11, a new implementation of the network configuration subsystem is powered by [COSI](../controllers-resources/).

website/content/docs/v0.15/Learn More/philosophy.md

@@ -1,6 +1,6 @@
 ---
 title: Philosophy
-weight: 1
+weight: 10
 ---
 
 ## Distributed

website/content/docs/v0.15/Learn More/talosctl.md

@@ -1,6 +1,6 @@
 ---
 title: "talosctl"
-weight: 7
+weight: 70
 ---
 
 The `talosctl` tool packs a lot of power into a small package.