chore(ci): add tests with longhorn v2 engine

Add tests with longhorn v2 engine. Fixes: #9669 Signed-off-by: Noel Georgi <git@frezbo.dev>
2025-10-01 18:51:13 +02:00 · 2025-01-22 19:20:31 +05:30 · 2025-01-22 19:20:31 +05:30 · 82c9ec158e
commit 82c9ec158e
parent 689ea1dbfe
16 changed files with 252 additions and 135 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-22T14:28:18Z by kres 3075de9.
+# Generated on 2025-01-22T17:37:55Z by kres 3075de9.

 name: default
 concurrency:
@ -2957,10 +2957,14 @@ jobs:
          EXTRA_TEST_ARGS: -talos.csi=longhorn
          GITHUB_STEP_NAME: ${{ github.job}}-e2e-qemu-csi-longhorn
          IMAGE_REGISTRY: registry.dev.siderolabs.io
-          QEMU_MEMORY_WORKERS: "3072"
+          QEMU_EXTRA_DISKS: "1"
+          QEMU_EXTRA_DISKS_DRIVERS: nvme
+          QEMU_EXTRA_DISKS_SIZE: "12288"
+          QEMU_MEMORY_WORKERS: "4096"
          QEMU_WORKERS: "3"
          SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/longhorn-cp.yaml'
+          WITH_CONFIG_PATCH_WORKER: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
        run: |
          sudo -E make e2e-qemu
      - name: save artifacts
@ -3067,7 +3071,7 @@ jobs:
          QEMU_MEMORY_WORKERS: "4096"
          QEMU_WORKERS: "3"
          SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/openebs-cp.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/openebs-cp.yaml'
          WITH_CONFIG_PATCH_WORKER: '@hack/test/patches/openebs.yaml'
        run: |
          sudo -E make e2e-qemu
@ -3175,7 +3179,7 @@ jobs:
          QEMU_MEMORY_WORKERS: "8192"
          QEMU_WORKERS: "3"
          SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/rook-ceph.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/rook-ceph.yaml'
        run: |
          sudo -E make e2e-qemu
      - name: save artifacts
--- a/.github/workflows/integration-qemu-csi-longhorn-cron.yaml
+++ b/.github/workflows/integration-qemu-csi-longhorn-cron.yaml
@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-21T10:29:14Z by kres 3075de9.
+# Generated on 2025-01-22T17:37:55Z by kres 3075de9.

 name: integration-qemu-csi-longhorn-cron
 concurrency:
@ -112,10 +112,14 @@ jobs:
          EXTRA_TEST_ARGS: -talos.csi=longhorn
          GITHUB_STEP_NAME: ${{ github.job}}-e2e-qemu-csi-longhorn
          IMAGE_REGISTRY: registry.dev.siderolabs.io
-          QEMU_MEMORY_WORKERS: "3072"
+          QEMU_EXTRA_DISKS: "1"
+          QEMU_EXTRA_DISKS_DRIVERS: nvme
+          QEMU_EXTRA_DISKS_SIZE: "12288"
+          QEMU_MEMORY_WORKERS: "4096"
          QEMU_WORKERS: "3"
          SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/longhorn-cp.yaml'
+          WITH_CONFIG_PATCH_WORKER: '@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml'
        run: |
          sudo -E make e2e-qemu
      - name: save artifacts
--- a/.github/workflows/integration-qemu-csi-openebs-cron.yaml
+++ b/.github/workflows/integration-qemu-csi-openebs-cron.yaml
@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-21T10:29:14Z by kres 3075de9.
+# Generated on 2025-01-22T14:23:24Z by kres 3075de9.

 name: integration-qemu-csi-openebs-cron
 concurrency:
@ -89,7 +89,7 @@ jobs:
          QEMU_MEMORY_WORKERS: "4096"
          QEMU_WORKERS: "3"
          SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/openebs-cp.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/openebs-cp.yaml'
          WITH_CONFIG_PATCH_WORKER: '@hack/test/patches/openebs.yaml'
        run: |
          sudo -E make e2e-qemu
--- a/.github/workflows/integration-qemu-csi-rook-ceph-cron.yaml
+++ b/.github/workflows/integration-qemu-csi-rook-ceph-cron.yaml
@ -1,6 +1,6 @@
 # THIS FILE WAS AUTOMATICALLY GENERATED, PLEASE DO NOT EDIT.
 #
-# Generated on 2025-01-21T10:29:14Z by kres 3075de9.
+# Generated on 2025-01-22T14:23:24Z by kres 3075de9.

 name: integration-qemu-csi-rook-ceph-cron
 concurrency:
@ -89,7 +89,7 @@ jobs:
          QEMU_MEMORY_WORKERS: "8192"
          QEMU_WORKERS: "3"
          SHORT_INTEGRATION_TEST: "yes"
-          WITH_CONFIG_PATCH: '@hack/test/patches/rook-ceph.yaml'
+          WITH_CONFIG_PATCH_CONTROLPLANE: '@hack/test/patches/rook-ceph.yaml'
        run: |
          sudo -E make e2e-qemu
      - name: save artifacts
--- a/.kres.yaml
+++ b/.kres.yaml
@ -1352,7 +1352,7 @@ spec:
            QEMU_MEMORY_WORKERS: 8192
            QEMU_EXTRA_DISKS: 1
            QEMU_EXTRA_DISKS_SIZE: 12288
-            WITH_CONFIG_PATCH: "@hack/test/patches/rook-ceph.yaml"
+            WITH_CONFIG_PATCH_CONTROLPLANE: "@hack/test/patches/rook-ceph.yaml"
            EXTRA_TEST_ARGS: -talos.csi=rook-ceph
            IMAGE_REGISTRY: registry.dev.siderolabs.io
        - name: save-fio-benchmark
@ -1441,8 +1441,12 @@ spec:
            GITHUB_STEP_NAME: ${{ github.job}}-e2e-qemu-csi-longhorn
            SHORT_INTEGRATION_TEST: yes
            QEMU_WORKERS: 3
-            QEMU_MEMORY_WORKERS: 3072
-            WITH_CONFIG_PATCH: "@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml"
+            QEMU_MEMORY_WORKERS: 4096
+            QEMU_EXTRA_DISKS: 1
+            QEMU_EXTRA_DISKS_SIZE: 12288
+            QEMU_EXTRA_DISKS_DRIVERS: nvme
+            WITH_CONFIG_PATCH_CONTROLPLANE: "@hack/test/patches/longhorn-cp.yaml"
+            WITH_CONFIG_PATCH_WORKER: "@_out/installer-extensions-patch.yaml:@_out/kubelet-fat-patch.yaml:@hack/test/patches/longhorn.yaml"
            EXTRA_TEST_ARGS: -talos.csi=longhorn
            IMAGE_REGISTRY: registry.dev.siderolabs.io
        - name: save-fio-benchmark
@ -1508,7 +1512,7 @@ spec:
            QEMU_MEMORY_WORKERS: 4096
            QEMU_EXTRA_DISKS: 1
            QEMU_EXTRA_DISKS_SIZE: 12288
-            WITH_CONFIG_PATCH: "@hack/test/patches/openebs-cp.yaml"
+            WITH_CONFIG_PATCH_CONTROLPLANE: "@hack/test/patches/openebs-cp.yaml"
            WITH_CONFIG_PATCH_WORKER: "@hack/test/patches/openebs.yaml"
            EXTRA_TEST_ARGS: -talos.csi=openebs
            IMAGE_REGISTRY: registry.dev.siderolabs.io
--- a/hack/test/e2e-qemu.sh
+++ b/hack/test/e2e-qemu.sh
@ -139,6 +139,18 @@ case "${WITH_ISO:-false}" in
    ;;
 esac

+case "${WITH_CONFIG_PATCH_CONTROLPLANE:-false}" in
+  false)
+    ;;
+  *)
+    [[ ! ${WITH_CONFIG_PATCH_CONTROLPLANE} =~ ^@ ]] && echo "WITH_CONFIG_PATCH_CONTROLPLANE variable should start with @" && exit 1
+
+    for i in ${WITH_CONFIG_PATCH_CONTROLPLANE//:/ }; do
+      QEMU_FLAGS+=("--config-patch-control-plane=${i}")
+    done
+    ;;
+esac
+
 case "${WITH_CONFIG_PATCH_WORKER:-false}" in
  false)
    ;;
--- a/hack/test/patches/longhorn-cp.yaml
+++ b/hack/test/patches/longhorn-cp.yaml
@ -0,0 +1,9 @@
+---
+cluster:
+  apiServer:
+    admissionControl:
+      - name: PodSecurity
+        configuration:
+          exemptions:
+            namespaces:
+              - longhorn-system
--- a/hack/test/patches/longhorn.yaml
+++ b/hack/test/patches/longhorn.yaml
@ -1,4 +1,12 @@
+---
 machine:
+  sysctls:
+    vm.nr_hugepages: "1024"
+  kernel:
+    modules:
+      - name: nvme_tcp
+      - name: vfio_pci
+      - name: uio_pci_generic
  kubelet:
    extraMounts:
      - destination: /var/lib/longhorn
@ -8,11 +16,3 @@ machine:
          - bind
          - rshared
          - rw
-cluster:
-  apiServer:
-    admissionControl:
-      - name: PodSecurity
-        configuration:
-          exemptions:
-            namespaces:
-              - longhorn-system
--- a/hack/test/patches/openebs-cp.yaml
+++ b/hack/test/patches/openebs-cp.yaml
@ -1,10 +1,9 @@
+---
 cluster:
  apiServer:
    admissionControl:
      - name: PodSecurity
        configuration:
-          apiVersion: pod-security.admission.config.k8s.io/v1beta1
-          kind: PodSecurityConfiguration
          exemptions:
            namespaces:
              - openebs
--- a/hack/test/patches/openebs.yaml
+++ b/hack/test/patches/openebs.yaml
@ -1,3 +1,4 @@
+---
 machine:
  sysctls:
    vm.nr_hugepages: "1024"
--- a/internal/integration/base/k8s.go
+++ b/internal/integration/base/k8s.go
@ -33,6 +33,7 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/discovery"
@ -588,7 +589,7 @@ func (k8sSuite *K8sSuite) WaitForResource(ctx context.Context, namespace, group,
 		}

 		if !exists {
-			return true, fmt.Errorf("resource %s/%s/%s/%s not found", group, version, kind, resourceName)
+			return true, errors.NewNotFound(mapping.Resource.GroupResource(), resourceName)
 		}

 		return false, nil
@ -657,15 +658,15 @@ func (k8sSuite *K8sSuite) GetUnstructuredResource(ctx context.Context, namespace
 }

 // RunFIOTest runs the FIO test with the given storage class and size using kubestr.
-func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClasss, size string) error {
+func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClass, size string) error {
 	args := []string{
 		"--outfile",
-		fmt.Sprintf("/tmp/fio-%s.json", storageClasss),
+		fmt.Sprintf("/tmp/fio-%s.json", storageClass),
 		"--output",
 		"json",
 		"fio",
 		"--storageclass",
-		storageClasss,
+		storageClass,
 		"--size",
 		size,
 	}
@ -821,6 +822,24 @@ func (k8sSuite *K8sSuite) DeleteManifests(ctx context.Context, manifests []unstr
 	}
 }

+// PatchK8sObject patches the kubernetes object with the given namespace, group, kind, version and name.
+func (k8sSuite *K8sSuite) PatchK8sObject(ctx context.Context, namespace, group, kind, version, resourceName string, patchBytes []byte) {
+	patchBytes, err := yaml.ToJSON(patchBytes)
+	k8sSuite.Require().NoError(err, "error converting patch to JSON")
+
+	mapping, err := k8sSuite.Mapper.RESTMapping(schema.GroupKind{
+		Group: group,
+		Kind:  kind,
+	}, version)
+
+	k8sSuite.Require().NoError(err, "error creating mapping for resource %s/%s/%s", group, kind, version)
+
+	dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(namespace)
+
+	_, err = dr.Patch(ctx, resourceName, types.MergePatchType, patchBytes, metav1.PatchOptions{})
+	k8sSuite.Require().NoError(err, "error patching resource %s/%s/%s/%s", group, version, kind, resourceName)
+}
+
 // ToUnstructured converts the given runtime.Object to unstructured.Unstructured.
 func (k8sSuite *K8sSuite) ToUnstructured(obj runtime.Object) unstructured.Unstructured {
 	unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
--- a/internal/integration/k8s/constants.go
+++ b/internal/integration/k8s/constants.go
@ -13,7 +13,7 @@ const (
 	RookCephHelmChartVersion = "v1.16.0"
 	// LongHornHelmChartVersion is the version of the Longhorn Helm chart to use.
 	// renovate: datasource=helm versioning=helm depName=longhorn registryUrl=https://charts.longhorn.io
-	LongHornHelmChartVersion = "v1.7.2"
+	LongHornHelmChartVersion = "v1.8.0"
 	// OpenEBSChartVersion is the version of the OpenEBS Helm chart to use.
 	// renovate: datasource=helm versioning=helm depName=openebs registryUrl=https://openebs.github.io/openebs
 	OpenEBSChartVersion = "v4.1.2"
--- a/internal/integration/k8s/longhorn.go
+++ b/internal/integration/k8s/longhorn.go
@ -28,6 +28,15 @@ var (

 	//go:embed testdata/pod-iscsi-volume.yaml
 	podWithISCSIVolumeTemplate []byte
+
+	//go:embed testdata/longhorn-v2-engine-values.yaml
+	longhornEngineV2Values []byte
+
+	//go:embed testdata/longhorn-v2-storageclass.yaml
+	longhornV2StorageClassManifest []byte
+
+	//go:embed testdata/longhorn-v2-disk-patch.yaml
+	longhornNodeDiskPatch []byte
 )

 // LongHornSuite tests deploying Longhorn.
@ -41,8 +50,6 @@ func (suite *LongHornSuite) SuiteName() string {
 }

 // TestDeploy tests deploying Longhorn and running a simple test.
-//
-//nolint:gocyclo
 func (suite *LongHornSuite) TestDeploy() {
 	suite.T().Parallel()

@ -69,20 +76,54 @@ func (suite *LongHornSuite) TestDeploy() {
 		LongHornHelmChartVersion,
 		"longhorn",
 		"longhorn",
-		nil,
+		longhornEngineV2Values,
 	); err != nil {
 		suite.T().Fatalf("failed to install Longhorn chart: %v", err)
 	}

+	longhornV2StorageClassunstructured := suite.ParseManifests(longhornV2StorageClassManifest)
+
+	suite.T().Cleanup(func() {
+		suite.DeleteManifests(ctx, longhornV2StorageClassunstructured)
+	})
+
+	suite.ApplyManifests(ctx, longhornV2StorageClassunstructured)
+
+	nodes := suite.DiscoverNodeInternalIPsByType(ctx, machine.TypeWorker)
+
+	suite.Require().Equal(3, len(nodes), "expected 3 worker nodes")
+
+	for _, node := range nodes {
+		k8sNode, err := suite.GetK8sNodeByInternalIP(ctx, node)
+		suite.Require().NoError(err)
+
+		suite.Require().NoError(suite.WaitForResource(ctx, "longhorn-system", "longhorn.io", "Node", "v1beta2", k8sNode.Name, "{.status.diskStatus.*.conditions[?(@.type==\"Ready\")].status}", "True"))
+		suite.Require().NoError(suite.WaitForResource(ctx, "longhorn-system", "longhorn.io", "Node", "v1beta2", k8sNode.Name, "{.status.diskStatus.*.conditions[?(@.type==\"Schedulable\")].status}", "True"))
+
+		suite.PatchK8sObject(ctx, "longhorn-system", "longhorn.io", "Node", "v1beta2", k8sNode.Name, longhornNodeDiskPatch)
+	}
+
 	suite.T().Run("fio", func(t *testing.T) {
 		t.Parallel()

 		suite.Require().NoError(suite.RunFIOTest(ctx, "longhorn", "10G"))
 	})

+	suite.T().Run("fio-v2", func(t *testing.T) {
+		t.Parallel()
+
+		suite.Require().NoError(suite.RunFIOTest(ctx, "longhorn-v2", "10G"))
+	})
+
 	suite.T().Run("iscsi", func(t *testing.T) {
 		t.Parallel()

+		suite.testDeployISCSI(ctx)
+	})
+}
+
+//nolint:gocyclo
+func (suite *LongHornSuite) testDeployISCSI(ctx context.Context) {
 	longHornISCSIVolumeManifestUnstructured := suite.ParseManifests(longHornISCSIVolumeManifest)

 	defer func() {
@ -184,7 +225,6 @@ func (suite *LongHornSuite) TestDeploy() {
 	suite.ApplyManifests(ctx, podWithISCSIVolumeUnstructured)

 	suite.Require().NoError(suite.WaitForPodToBeRunning(ctx, 3*time.Minute, "default", "iscsipd"))
-	})
 }

 func init() {
--- a/internal/integration/k8s/testdata/longhorn-v2-disk-patch.yaml
+++ b/internal/integration/k8s/testdata/longhorn-v2-disk-patch.yaml
@ -0,0 +1,10 @@
+---
+spec:
+  disks:
+    nvme:
+      allowScheduling: true
+      evictionRequested: false
+      path: /dev/nvme0n1
+      storageReserved: 0
+      tags: []
+      diskType: block
--- a/internal/integration/k8s/testdata/longhorn-v2-engine-values.yaml
+++ b/internal/integration/k8s/testdata/longhorn-v2-engine-values.yaml
@ -0,0 +1,2 @@
+defaultSettings:
+  v2DataEngine: true
--- a/internal/integration/k8s/testdata/longhorn-v2-storageclass.yaml
+++ b/internal/integration/k8s/testdata/longhorn-v2-storageclass.yaml
@ -0,0 +1,13 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: longhorn-v2
+provisioner: driver.longhorn.io
+allowVolumeExpansion: true
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+parameters:
+  numberOfReplicas: "3"
+  staleReplicaTimeout: "2880"
+  fsType: "ext4"
+  dataEngine: "v2"