From 1237843e62bb46395ae7b4eca760a87f2d172dc4 Mon Sep 17 00:00:00 2001 From: ArthurSens Date: Fri, 5 Mar 2021 20:14:19 +0000 Subject: [PATCH 1/5] Adds an addon for podSecurityPolicies Signed-off-by: ArthurSens --- examples/pod-security-policies.jsonnet | 23 ++ .../addons/podsecuritypolicies.libsonnet | 242 ++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 examples/pod-security-policies.jsonnet create mode 100644 jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet diff --git a/examples/pod-security-policies.jsonnet b/examples/pod-security-policies.jsonnet new file mode 100644 index 00000000..3274c937 --- /dev/null +++ b/examples/pod-security-policies.jsonnet @@ -0,0 +1,23 @@ +local kp = + (import 'kube-prometheus/main.libsonnet') + + (import 'kube-prometheus/addons/podsecuritypolicies.libsonnet'); + +{ 'setup/0namespace-namespace': kp.kubePrometheus.namespace } + +// Add the restricted psp to setup +{ 'setup/0podsecuritypolicy-restricted': kp.restrictedPodSecurityPolicy } + +{ + ['setup/prometheus-operator-' + name]: kp.prometheusOperator[name] + for name in std.filter((function(name) name != 'serviceMonitor' && name != 'prometheusRule'), std.objectFields(kp.prometheusOperator)) +} + +// serviceMonitor and prometheusRule are separated so that they can be created after the CRDs are ready +{ 'prometheus-operator-serviceMonitor': kp.prometheusOperator.serviceMonitor } + +{ 'prometheus-operator-prometheusRule': kp.prometheusOperator.prometheusRule } + +{ 'kube-prometheus-prometheusRule': kp.kubePrometheus.prometheusRule } + +{ ['alertmanager-' + name]: kp.alertmanager[name] for name in std.objectFields(kp.alertmanager) } + +{ ['blackbox-exporter-' + name]: kp.blackboxExporter[name] for name in std.objectFields(kp.blackboxExporter) } + +{ ['grafana-' + name]: kp.grafana[name] for name in std.objectFields(kp.grafana) } + +{ ['kube-state-metrics-' + name]: kp.kubeStateMetrics[name] for name in std.objectFields(kp.kubeStateMetrics) } + +{ ['kubernetes-' + name]: kp.kubernetesControlPlane[name] for name in std.objectFields(kp.kubernetesControlPlane) } +{ ['node-exporter-' + name]: kp.nodeExporter[name] for name in std.objectFields(kp.nodeExporter) } + +{ ['prometheus-' + name]: kp.prometheus[name] for name in std.objectFields(kp.prometheus) } + +{ ['prometheus-adapter-' + name]: kp.prometheusAdapter[name] for name in std.objectFields(kp.prometheusAdapter) } diff --git a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet new file mode 100644 index 00000000..32ef6176 --- /dev/null +++ b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet @@ -0,0 +1,242 @@ +local restrictedPodSecurityPolicy = { + apiVersion: 'policy/v1beta1', + kind: 'PodSecurityPolicy', + metadata: { + name: 'restricted', + }, + spec: { + privileged: false, + // Required to prevent escalations to root. + allowPrivilegeEscalation: false, + // This is redundant with non-root + disallow privilege escalation, + // but we can provide it for defense in depth. + requiredDropCapabilities: ['ALL'], + // Allow core volume types. + volumes: [ + 'configMap', + 'emptyDir', + 'secret', + // Assume that persistentVolumes set up by the cluster admin are safe to use. + 'persistentVolumeClaim', + ], + hostNetwork: false, + hostIPC: false, + hostPID: false, + runAsUser: { + // Require the container to run without root privileges. + rule: 'MustRunAsNonRoot', + }, + seLinux: { + // This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny', + }, + supplementalGroups: { + rule: 'MustRunAs', + ranges: [{ + // Forbid adding the root group. + min: 1, + max: 65535, + }], + }, + fsGroup: { + rule: 'MustRunAs', + ranges: [{ + // Forbid adding the root group. + min: 1, + max: 65535, + }], + }, + readOnlyRootFilesystem: false, + }, +}; + +{ + restrictedPodSecurityPolicy: restrictedPodSecurityPolicy, + + alertmanager+: { + role: { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'Role', + metadata: { + name: 'alertmanager-' + $.values.alertmanager.name, + }, + rules: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: [restrictedPodSecurityPolicy.metadata.name], + }], + }, + + roleBinding: { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'RoleBinding', + metadata: { + name: 'alertmanager-' + $.values.alertmanager.name, + }, + roleRef: { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'Role', + name: 'alertmanager-' + $.values.alertmanager.name, + }, + subjects: [{ + kind: 'ServiceAccount', + name: 'alertmanager-' + $.values.alertmanager.name, + namespace: $.values.alertmanager.namespace, + }], + }, + }, + + blackboxExporter+: { + clusterRole+: { + rules+: [ + { + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: ['blackbox-exporter-psp'], + }, + ], + }, + + podSecurityPolicy: + local blackboxExporterPspPrivileged = + if $.blackboxExporter.config.privileged then + { + metadata+: { + name: 'blackbox-exporter-psp', + }, + spec+: { + privileged: true, + allowedCapabilities: ['NET_RAW'], + runAsUser: { + rule: 'RunAsAny', + }, + }, + } + else + {}; + + restrictedPodSecurityPolicy + blackboxExporterPspPrivileged, + }, + + grafana+: { + role: { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'Role', + metadata: { + name: 'grafana', + }, + rules: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: [restrictedPodSecurityPolicy.metadata.name], + }], + }, + + roleBinding: { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'RoleBinding', + metadata: { + name: 'grafana', + }, + roleRef: { + apiGroup: 'rbac.authorization.k8s.io', + kind: 'Role', + name: 'grafana', + }, + subjects: [{ + kind: 'ServiceAccount', + name: $.grafana.serviceAccount.metadata.name, + namespace: $.grafana.serviceAccount.metadata.namespace, + }], + }, + }, + + kubeStateMetrics+: { + clusterRole+: { + rules+: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: [restrictedPodSecurityPolicy.metadata.name], + }], + }, + }, + + nodeExporter+: { + clusterRole+: { + rules+: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: ['node-exporter-psp'], + }], + }, + + podSecurityPolicy: restrictedPodSecurityPolicy { + metadata+: { + name: 'node-exporter-psp', + }, + spec+: { + allowedHostPaths+: [ + { + pathPrefix: '/proc', + readOnly: true, + }, + { + pathPrefix: '/sys', + readOnly: true, + }, + { + pathPrefix: '/', + readOnly: true, + }, + ], + hostNetwork: true, + hostPID: true, + hostPorts: [ + { + max: 9100, + min: 9100, + }, + ], + readOnlyRootFilesystem: true, + }, + }, + }, + + prometheusAdapter+: { + clusterRole+: { + rules+: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: [restrictedPodSecurityPolicy.metadata.name], + }], + }, + }, + + prometheusOperator+: { + clusterRole+: { + rules+: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: [restrictedPodSecurityPolicy.metadata.name], + }], + }, + }, + + prometheus+: { + clusterRole+: { + rules+: [{ + apiGroups: ['policy'], + resources: ['podsecuritypolicies'], + verbs: ['use'], + resourceNames: [restrictedPodSecurityPolicy.metadata.name], + }], + }, + }, +} From 98559a0f42dde38980f6493dca974524034da110 Mon Sep 17 00:00:00 2001 From: ArthurSens Date: Fri, 12 Mar 2021 20:41:00 +0000 Subject: [PATCH 2/5] Allow kube-state-metrics to run as any user Signed-off-by: ArthurSens --- .../addons/podsecuritypolicies.libsonnet | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet index 32ef6176..38dc736c 100644 --- a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet +++ b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet @@ -160,9 +160,20 @@ local restrictedPodSecurityPolicy = { apiGroups: ['policy'], resources: ['podsecuritypolicies'], verbs: ['use'], - resourceNames: [restrictedPodSecurityPolicy.metadata.name], + resourceNames: ['kube-state-metrics-psp'], }], }, + + podSecurityPolicy: restrictedPodSecurityPolicy { + metadata+: { + name: 'kube-state-metrics-psp', + }, + spec+: { + runAsUser: { + rule: 'RunAsAny', + }, + }, + }, }, nodeExporter+: { From e8abcd3dc9d022ffb2030bea660607176dcdb8de Mon Sep 17 00:00:00 2001 From: ArthurSens Date: Fri, 12 Mar 2021 20:51:15 +0000 Subject: [PATCH 3/5] Allow node-exporter to use hostPath volumes Signed-off-by: ArthurSens --- jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet index 38dc736c..46493c55 100644 --- a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet +++ b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet @@ -214,6 +214,9 @@ local restrictedPodSecurityPolicy = { }, ], readOnlyRootFilesystem: true, + volumes+: [ + 'hostPath', + ], }, }, }, From 478a18a6a7353bdad6c0b24f2eb0ee9e51efa778 Mon Sep 17 00:00:00 2001 From: ArthurSens Date: Fri, 12 Mar 2021 21:14:39 +0000 Subject: [PATCH 4/5] Turn alertmanager's and grafana's roles into clusterRoles Signed-off-by: ArthurSens --- .../addons/podsecuritypolicies.libsonnet | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet index 46493c55..888d553e 100644 --- a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet +++ b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet @@ -54,9 +54,9 @@ local restrictedPodSecurityPolicy = { restrictedPodSecurityPolicy: restrictedPodSecurityPolicy, alertmanager+: { - role: { + clusterRole: { apiVersion: 'rbac.authorization.k8s.io/v1', - kind: 'Role', + kind: 'ClusterRole', metadata: { name: 'alertmanager-' + $.values.alertmanager.name, }, @@ -68,15 +68,15 @@ local restrictedPodSecurityPolicy = { }], }, - roleBinding: { + clusterRoleBinding: { apiVersion: 'rbac.authorization.k8s.io/v1', - kind: 'RoleBinding', + kind: 'ClusterRoleBinding', metadata: { name: 'alertmanager-' + $.values.alertmanager.name, }, roleRef: { apiGroup: 'rbac.authorization.k8s.io', - kind: 'Role', + kind: 'ClusterRole', name: 'alertmanager-' + $.values.alertmanager.name, }, subjects: [{ @@ -121,9 +121,9 @@ local restrictedPodSecurityPolicy = { }, grafana+: { - role: { + clusterRole: { apiVersion: 'rbac.authorization.k8s.io/v1', - kind: 'Role', + kind: 'ClusterRole', metadata: { name: 'grafana', }, @@ -135,15 +135,15 @@ local restrictedPodSecurityPolicy = { }], }, - roleBinding: { + clusterRoleBinding: { apiVersion: 'rbac.authorization.k8s.io/v1', - kind: 'RoleBinding', + kind: 'ClusterRoleBinding', metadata: { name: 'grafana', }, roleRef: { apiGroup: 'rbac.authorization.k8s.io', - kind: 'Role', + kind: 'ClusterRole', name: 'grafana', }, subjects: [{ From b7d7ba02710ac56df23a0de1894ab192a781d8f1 Mon Sep 17 00:00:00 2001 From: ArthurSens Date: Thu, 18 Mar 2021 20:56:47 +0000 Subject: [PATCH 5/5] Use node-exporter defaults.port on psp Signed-off-by: ArthurSens --- jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet index 888d553e..65908f8c 100644 --- a/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet +++ b/jsonnet/kube-prometheus/addons/podsecuritypolicies.libsonnet @@ -209,8 +209,8 @@ local restrictedPodSecurityPolicy = { hostPID: true, hostPorts: [ { - max: 9100, - min: 9100, + max: $.nodeExporter.config.port, + min: $.nodeExporter.config.port, }, ], readOnlyRootFilesystem: true,