Scripts and readme (#258)

* Avoid race condition when deploying quickstart example

The namespace and CRD creation must happen before any dependent objects
are created.  So we can put these in a separate directory (manifest/setup)
so they can be created before the other objects.

Some minor updates to the README and added a couple of scripts
for the quickstarts

Update travis script to avoid race condition

Signed-off-by: Paul Gier <pgier@redhat.com>

* simplify the example quickstart script and improve readme

Signed-off-by: Paul Gier <pgier@redhat.com>

* increase minikube memory to 6g for quickstart example

README.md

@@ -71,13 +71,13 @@ This adapter is an Extension API Server and Kubernetes needs to be have this fea
 
 ### minikube
 
-In order to just try out this stack, start [minikube](https://github.com/kubernetes/minikube) with the following command:
+To try out this stack, start [minikube](https://github.com/kubernetes/minikube) with the following command:
 
 ```shell
-$ minikube delete && minikube start --kubernetes-version=v1.14.4 --memory=4096 --bootstrapper=kubeadm --extra-config=kubelet.authentication-token-webhook=true --extra-config=kubelet.authorization-mode=Webhook --extra-config=scheduler.address=0.0.0.0 --extra-config=controller-manager.address=0.0.0.0
+$ minikube delete && minikube start --kubernetes-version=v1.16.0 --memory=6g --bootstrapper=kubeadm --extra-config=kubelet.authentication-token-webhook=true --extra-config=kubelet.authorization-mode=Webhook --extra-config=scheduler.address=0.0.0.0 --extra-config=controller-manager.address=0.0.0.0
 ```
 
-The kube-prometheus stack includes a resource metrics API server, like the metrics-server does. So ensure the metrics-server plugin is disabled on minikube:
+The kube-prometheus stack includes a resource metrics API server, so the metrics-server addon is not necessary. Ensure the metrics-server addon is disabled on minikube:
 
 ```shell
 $ minikube addons disable metrics-server
@@ -90,20 +90,23 @@ $ minikube addons disable metrics-server
 This project is intended to be used as a library (i.e. the intent is not for you to create your own modified copy of this repository).
 
 Though for a quickstart a compiled version of the Kubernetes [manifests](manifests) generated with this library (specifically with `example.jsonnet`) is checked into this repository in order to try the content out quickly. To try out the stack un-customized run:
- * Simply create the stack:
+ * Create the monitoring stack using the config in the `manifests` directory:
+
 ```shell
-$ kubectl create -f manifests/
+# Create the namespace and CRDs, and then wait for them to be availble before creating the remaining resources
-
+kubectl create -f manifests/setup
-# It can take a few seconds for the above 'create manifests' command to fully create the following resources, so verify the resources are ready before proceeding.
+until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
-$ until kubectl get customresourcedefinitions servicemonitors.monitoring.coreos.com ; do date; sleep 1; echo ""; done
+kubectl create -f manifests/
-$ until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
-
-$ kubectl apply -f manifests/ # This command sometimes may need to be done twice (to workaround a race condition).
 ```
 
+We create the namespace and CustomResourceDefinitions first to avoid race conditions when deploying the monitoring components.
+Alternatively, the resources in both folders can be applied with a single command 
+`kubectl create -f manifests/setup -f manifests`, but it may be necessary to run the command multiple times for all components to
+be created successfullly.
+
  * And to teardown the stack:
 ```shell
-$ kubectl delete -f manifests/
+kubectl delete --ignore-not-found=true -f manifests/ -f manifests/setup
 ```
 
 ### Access the dashboards
@@ -187,8 +190,13 @@ local kp =
     },
   };
 
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor is separated so that it can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +
@@ -212,7 +220,7 @@ set -o pipefail
 
 # Make sure to start with a clean 'manifests' dir
 rm -rf manifests
-mkdir manifests
+mkdir -p manifests/setup
 
                                                # optional, but we would like to generate yaml, not json
 jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml; rm -f {}' -- {}

build.sh

@@ -9,7 +9,7 @@ set -o pipefail
 
 # Make sure to start with a clean 'manifests' dir
 rm -rf manifests
-mkdir manifests
+mkdir -p manifests/setup
 
                                                # optional, but we would like to generate yaml, not json
 jsonnet -J vendor -m manifests "${1-example.jsonnet}" | xargs -I{} sh -c 'cat {} | gojsontoyaml > {}.yaml; rm -f {}' -- {}

docs/developing-prometheus-rules-and-grafana-dashboards.md

@@ -24,8 +24,13 @@ local kp =
     },
   };
 
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor is separated so that it can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +

example.jsonnet

@@ -12,8 +12,13 @@ local kp =
     },
   };
 
-{ ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+{ ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-{ ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+{
+  ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+  for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+} +
+// serviceMonitor is separated so that it can be created after the CRDs are ready
+{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
 { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
 { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
 { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +

examples/kustomize.jsonnet

@@ -8,8 +8,13 @@ local kp =
 local manifests =
   // Uncomment line below to enable vertical auto scaling of kube-state-metrics
   //{ ['ksm-autoscaler-' + name]: kp.ksmAutoscaler[name] for name in std.objectFields(kp.ksmAutoscaler) } +
-  { ['00namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
+  { ['setup/0namespace-' + name]: kp.kubePrometheus[name] for name in std.objectFields(kp.kubePrometheus) } +
-  { ['0prometheus-operator-' + name]: kp.prometheusOperator[name] for name in std.objectFields(kp.prometheusOperator) } +
+  {
+    ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name]
+    for name in std.filter((function(name) name != 'serviceMonitor'), std.objectFields(kp.prometheusOperator))
+  } +
+  // serviceMonitor is separated so that it can be created after the CRDs are ready
+  { 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } +
   { ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } +
   { ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } +
   { ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } +

kustomization.yaml

@@ -1,18 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- ./manifests/00namespace-namespace.yaml
-- ./manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0podmonitorCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
-- ./manifests/0prometheus-operator-clusterRole.yaml
-- ./manifests/0prometheus-operator-clusterRoleBinding.yaml
-- ./manifests/0prometheus-operator-deployment.yaml
-- ./manifests/0prometheus-operator-service.yaml
-- ./manifests/0prometheus-operator-serviceAccount.yaml
-- ./manifests/0prometheus-operator-serviceMonitor.yaml
 - ./manifests/alertmanager-alertmanager.yaml
 - ./manifests/alertmanager-secret.yaml
 - ./manifests/alertmanager-service.yaml
@@ -52,6 +40,7 @@ resources:
 - ./manifests/prometheus-adapter-serviceAccount.yaml
 - ./manifests/prometheus-clusterRole.yaml
 - ./manifests/prometheus-clusterRoleBinding.yaml
+- ./manifests/prometheus-operator-serviceMonitor.yaml
 - ./manifests/prometheus-prometheus.yaml
 - ./manifests/prometheus-roleBindingConfig.yaml
 - ./manifests/prometheus-roleBindingSpecificNamespaces.yaml
@@ -66,3 +55,14 @@ resources:
 - ./manifests/prometheus-serviceMonitorKubeControllerManager.yaml
 - ./manifests/prometheus-serviceMonitorKubeScheduler.yaml
 - ./manifests/prometheus-serviceMonitorKubelet.yaml
+- ./manifests/setup/0namespace-namespace.yaml
+- ./manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
+- ./manifests/setup/prometheus-operator-clusterRole.yaml
+- ./manifests/setup/prometheus-operator-clusterRoleBinding.yaml
+- ./manifests/setup/prometheus-operator-deployment.yaml
+- ./manifests/setup/prometheus-operator-service.yaml
+- ./manifests/setup/prometheus-operator-serviceAccount.yaml

scripts/minikube-start-kvm.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+minikube delete
+minikube addons disable metrics-server
+minikube start \
+    --vm-driver=kvm2 \
+    --kubernetes-version=v1.16.0 \
+    --memory=6g \
+    --bootstrapper=kubeadm \
+    --extra-config=kubelet.authentication-token-webhook=true \
+    --extra-config=kubelet.authorization-mode=Webhook \
+    --extra-config=scheduler.address=0.0.0.0 \
+    --extra-config=controller-manager.address=0.0.0.0

scripts/minikube-start.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+minikube delete
+minikube addons disable metrics-server
+minikube start \
+    --kubernetes-version=v1.16.0 \
+    --memory=6g \
+    --bootstrapper=kubeadm \
+    --extra-config=kubelet.authentication-token-webhook=true \
+    --extra-config=kubelet.authorization-mode=Webhook \
+    --extra-config=scheduler.address=0.0.0.0 \
+    --extra-config=controller-manager.address=0.0.0.0

scripts/monitoring-deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# create namespace and CRDs
+kubectl create -f manifests/setup
+
+# wait for CRD creation to complete
+until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+
+# create monitoring components
+kubectl create -f manifests/
+

tests/e2e/travis-e2e.sh

@@ -16,13 +16,13 @@ chmod +x kind
 ./kind create cluster
 export KUBECONFIG="$(./kind get kubeconfig-path)"
 
-./kubectl apply -f manifests/0prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
+# create namespace, permissions, and CRDs
-./kubectl apply -f manifests/0prometheus-operator-0prometheusCustomResourceDefinition.yaml
+./kubectl create -f manifests/setup
-./kubectl apply -f manifests/0prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
-./kubectl apply -f manifests/0prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
 
-# Wait for CRDs to be successfully registered
+# wait for CRD creation to complete
-sleep 10
+until ./kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
+
+# create monitoring components
+./kubectl create -f manifests/
 
-./kubectl apply -f manifests
 make test-e2e