diff --git a/.drone.jsonnet b/.drone.jsonnet index c7ea63112..769a2c55f 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -403,7 +403,7 @@ local integration_qemu_encrypted_vip = Step("e2e-encrypted-vip", target="e2e-qem "IMAGE_REGISTRY": local_registry, }); -local integration_qemu_csi = Step("e2e-csi", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={ +local integration_qemu_day_two = Step("e2e-day-two", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={ "IMAGE_REGISTRY": local_registry, "SHORT_INTEGRATION_TEST": "yes", "QEMU_WORKERS": "3", @@ -411,7 +411,7 @@ local integration_qemu_csi = Step("e2e-csi", target="e2e-qemu", privileged=true, "QEMU_MEMORY_WORKERS": "5120", "QEMU_EXTRA_DISKS": "1", "QEMU_EXTRA_DISKS_SIZE": "12288", - "WITH_TEST": "run_csi_tests", + "WITH_TEST": "run_day_two_tests", }); local integration_images = Step("images", target="images", depends_on=[load_artifacts], environment={"IMAGE_REGISTRY": local_registry}); @@ -455,7 +455,7 @@ local integration_pipelines = [ , integration_cilium, integration_bios, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan]) + integration_trigger(['integration-misc']), Pipeline('integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip]) + integration_trigger(['integration-qemu-encrypted-vip']), Pipeline('integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race]) + integration_trigger(['integration-qemu-race']), - Pipeline('integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi]) + integration_trigger(['integration-qemu-csi']), + Pipeline('integration-qemu-day-two', default_pipeline_steps + [integration_qemu_day_two]) + integration_trigger(['integration-qemu-day-two']), Pipeline('integration-images', default_pipeline_steps + [integration_images, integration_sbcs]) + integration_trigger(['integration-images']), // cron pipelines, triggered on schedule events @@ -467,7 +467,7 @@ local integration_pipelines = [ , integration_cilium, integration_bios, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']), Pipeline('cron-integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']), Pipeline('cron-integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race], [default_cron_pipeline]) + cron_trigger(['nightly']), - Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']), + Pipeline('cron-integration-qemu-day-two', default_pipeline_steps + [integration_qemu_day_two], [default_cron_pipeline]) + cron_trigger(['nightly']), Pipeline('cron-integration-images', default_pipeline_steps + [integration_images, integration_sbcs], [default_cron_pipeline]) + cron_trigger(['nightly']), ]; diff --git a/Makefile b/Makefile index a08cf9c82..e79bcf679 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,8 @@ KUBECTL_URL ?= https://storage.googleapis.com/kubernetes-release/release/v1.23.5 KUBESTR_URL ?= https://github.com/kastenhq/kubestr/releases/download/v0.4.31/kubestr_0.4.31_Linux_amd64.tar.gz CLUSTERCTL_VERSION ?= 1.0.4 CLUSTERCTL_URL ?= https://github.com/kubernetes-sigs/cluster-api/releases/download/v$(CLUSTERCTL_VERSION)/clusterctl-$(OPERATING_SYSTEM)-amd64 +D2CTL_URL ?= https://github.com/talos-systems/day-two/releases/download/v0.1.0-alpha.1/d2ctl-$(OPERATING_SYSTEM)-amd64 +PULUMI_URL ?= https://get.pulumi.com/releases/sdk/pulumi-v3.26.1-$(OPERATING_SYSTEM)-x64.tar.gz TESTPKGS ?= github.com/talos-systems/talos/... RELEASES ?= v0.13.4 v0.14.1 SHORT_INTEGRATION_TEST ?= @@ -173,7 +175,7 @@ docker-%: ## Builds the specified target defined in the Dockerfile using the doc registry-%: ## Builds the specified target defined in the Dockerfile using the image/registry output type. The build result will be pushed to the registry if PUSH=true. @$(MAKE) target-$* TARGET_ARGS="--output type=image,name=$(REGISTRY_AND_USERNAME)/$*:$(IMAGE_TAG) $(TARGET_ARGS)" -hack-test-%: ## Runs the specied script in ./hack/test with well known environment variables. +hack-test-%: ## Runs the specified script in ./hack/test with well known environment variables. @./hack/test/$*.sh # Generators @@ -318,7 +320,16 @@ $(ARTIFACTS)/clusterctl: @curl -L -o $(ARTIFACTS)/clusterctl "$(CLUSTERCTL_URL)" @chmod +x $(ARTIFACTS)/clusterctl -e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubectl $(ARTIFACTS)/clusterctl $(ARTIFACTS)/kubestr ## Runs the E2E test for the specified platform (e.g. e2e-docker). +$(ARTIFACTS)/d2ctl: + @mkdir -p $(ARTIFACTS) + @curl -L -o $(ARTIFACTS)/d2ctl "$(D2CTL_URL)" + @chmod +x $(ARTIFACTS)/d2ctl + +$(ARTIFACTS)/pulumi: + @mkdir -p $(ARTIFACTS) + @curl -L "$(PULUMI_URL)" | tar xzf - -C $(ARTIFACTS) --strip-components 1 pulumi/pulumi + +e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubectl $(ARTIFACTS)/clusterctl $(ARTIFACTS)/kubestr $(ARTIFACTS)/d2ctl $(ARTIFACTS)/pulumi ## Runs the E2E test for the specified platform (e.g. e2e-docker). @$(MAKE) hack-test-$@ \ PLATFORM=$* \ TAG=$(TAG) \ @@ -333,7 +344,9 @@ e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubect CUSTOM_CNI_URL=$(CUSTOM_CNI_URL) \ KUBECTL=$(PWD)/$(ARTIFACTS)/kubectl \ KUBESTR=$(PWD)/$(ARTIFACTS)/kubestr \ - CLUSTERCTL=$(PWD)/$(ARTIFACTS)/clusterctl + CLUSTERCTL=$(PWD)/$(ARTIFACTS)/clusterctl \ + D2CTL=$(PWD)/$(ARTIFACTS)/d2ctl \ + PULUMI=$(PWD)/$(ARTIFACTS)/pulumi provision-tests-prepare: release-artifacts $(ARTIFACTS)/$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET)-amd64 diff --git a/hack/test/day-two/config.yaml b/hack/test/day-two/config.yaml new file mode 100644 index 000000000..8ff0c5937 --- /dev/null +++ b/hack/test/day-two/config.yaml @@ -0,0 +1,31 @@ +charts: + loki: + namespace: loki + repo: https://grafana.github.io/helm-charts + chart: loki-stack + valuesPath: hack/test/day-two/loki-values.yaml + + metallb: + namespace: metallb + repo: https://metallb.github.io/metallb + chart: metallb + valuesPath: hack/test/day-two/metallb-values.yaml + + ingress-nginx: + namespace: ingress + repo: https://kubernetes.github.io/ingress-nginx + chart: ingress-nginx + depends: + - metallb + + rook: + namespace: rook-ceph + repo: https://charts.rook.io/release + chart: rook-ceph + + ceph: + namespace: rook-ceph + repo: https://charts.rook.io/release + chart: rook-ceph-cluster + depends: + - rook diff --git a/hack/test/day-two/loki-values.yaml b/hack/test/day-two/loki-values.yaml new file mode 100644 index 000000000..97fc20a7f --- /dev/null +++ b/hack/test/day-two/loki-values.yaml @@ -0,0 +1,12 @@ +grafana: + enabled: true + +prometheus: + enabled: true + alertmanager: + persistentVolume: + enabled: false + + server: + persistentVolume: + enabled: false diff --git a/hack/test/day-two/metallb-values.yaml b/hack/test/day-two/metallb-values.yaml new file mode 100644 index 000000000..072086be1 --- /dev/null +++ b/hack/test/day-two/metallb-values.yaml @@ -0,0 +1,6 @@ +configInline: + address-pools: + - name: default + protocol: layer2 + addresses: + - 10.5.0.240-10.5.0.242 diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh index 60e56c925..adb5b58c5 100755 --- a/hack/test/e2e.sh +++ b/hack/test/e2e.sh @@ -225,20 +225,26 @@ function run_extensions_test { curl http://172.20.1.2/ | grep Hello } -function run_csi_tests { - rm -rf "${TMP}/rook" - git clone --depth=1 --single-branch --branch v1.8.2 https://github.com/rook/rook.git "${TMP}/rook" - pushd "${TMP}/rook/deploy/examples" - ${KUBECTL} apply -f crds.yaml -f common.yaml -f operator.yaml - ${KUBECTL} apply -f cluster.yaml +function run_day_two_tests { + rm -rf "${TMP}/day-two-state" + # pulumi needs to be in $PATH temporarily + PATH="${PATH}:$(dirname ${PULUMI})" ${D2CTL} up --state-path "${TMP}/day-two-state" --config-path "${PWD}/hack/test/day-two/config.yaml" + + ${KUBECTL} --namespace loki wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/loki-kube-state-metrics + ${KUBECTL} --namespace loki wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/loki-prometheus-server + ${KUBECTL} --namespace loki wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/loki-grafana + + ${KUBECTL} --namespace metallb wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/metallb-controller + + ${KUBECTL} --namespace ingress wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/ingress-nginx-controller + # wait for the controller to populate the status field - sleep 30 - ${KUBECTL} --namespace rook-ceph wait --timeout=900s --for=jsonpath='{.status.phase}=Ready' cephclusters.ceph.rook.io/rook-ceph - ${KUBECTL} --namespace rook-ceph wait --timeout=900s --for=jsonpath='{.status.state}=Created' cephclusters.ceph.rook.io/rook-ceph - # .status.ceph is populated later only - sleep 60 - ${KUBECTL} --namespace rook-ceph wait --timeout=900s --for=jsonpath='{.status.ceph.health}=HEALTH_OK' cephclusters.ceph.rook.io/rook-ceph - ${KUBECTL} create -f csi/rbd/storageclass.yaml + sleep 10 + ${KUBECTL} --namespace rook-ceph wait --timeout=1800s --for=jsonpath='{.status.phase}=Ready' cephclusters.ceph.rook.io/rook-ceph + ${KUBECTL} --namespace rook-ceph wait --timeout=1800s --for=jsonpath='{.status.state}=Created' cephclusters.ceph.rook.io/rook-ceph + # .status.ceph is populated only after the cluster comes up + sleep 20 + ${KUBECTL} --namespace rook-ceph wait --timeout=1800s --for=jsonpath='{.status.ceph.health}=HEALTH_OK' cephclusters.ceph.rook.io/rook-ceph # hack until https://github.com/kastenhq/kubestr/issues/101 is addressed - KUBERNETES_SERVICE_HOST= KUBECONFIG="${TMP}/kubeconfig" ${KUBESTR} fio --storageclass rook-ceph-block --size 10G + KUBERNETES_SERVICE_HOST= KUBECONFIG="${TMP}/kubeconfig" ${KUBESTR} fio --storageclass ceph-block --size 10G }