From ab2473bdf4c4a7c90fafbaada97eeb31eaf6739e Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Tue, 20 Feb 2024 16:04:28 +0100 Subject: [PATCH] introduce internal for monitoring --- src/main/cljc/dda/c4k_common/monitoring.cljc | 81 +++++-------------- .../monitoring/monitoring_internal.cljc | 77 ++++++++++++++++++ ...e-state-metrics-cluster-role-binding.yaml} | 0 ...l => kube-state-metrics-cluster-role.yaml} | 0 ...aml => kube-state-metrics-deployment.yaml} | 0 ...> kube-state-metrics-service-account.yaml} | 0 ...e.yaml => kube-state-metrics-service.yaml} | 0 ...> node-exporter-cluster-role-binding.yaml} | 0 ...e.yaml => node-exporter-cluster-role.yaml} | 0 ...set.yaml => node-exporter-daemon-set.yaml} | 0 ...aml => node-exporter-service-account.yaml} | 0 ...ervice.yaml => node-exporter-service.yaml} | 0 ...l => prometheus-cluster-role-binding.yaml} | 0 ...role.yaml => prometheus-cluster-role.yaml} | 0 .../config.yaml => prometheus-config.yaml} | 0 ...oyment.yaml => prometheus-deployment.yaml} | 0 ...etheus.yaml => prometheus-prometheus.yaml} | 0 ...t.yaml => prometheus-service-account.yaml} | 0 .../service.yaml => prometheus-service.yaml} | 0 .../monitoring/monitoring_internal_test.cljc | 59 ++++++++++++++ .../cljc/dda/c4k_common/monitoring_test.cljc | 46 +---------- 21 files changed, 157 insertions(+), 106 deletions(-) create mode 100644 src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc rename src/main/resources/monitoring/{kube-state-metrics/cluster-role-binding.yaml => kube-state-metrics-cluster-role-binding.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/cluster-role.yaml => kube-state-metrics-cluster-role.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/deployment.yaml => kube-state-metrics-deployment.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/service-account.yaml => kube-state-metrics-service-account.yaml} (100%) rename src/main/resources/monitoring/{kube-state-metrics/service.yaml => kube-state-metrics-service.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/cluster-role-binding.yaml => node-exporter-cluster-role-binding.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/cluster-role.yaml => node-exporter-cluster-role.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/daemon-set.yaml => node-exporter-daemon-set.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/service-account.yaml => node-exporter-service-account.yaml} (100%) rename src/main/resources/monitoring/{node-exporter/service.yaml => node-exporter-service.yaml} (100%) rename src/main/resources/monitoring/{prometheus/cluster-role-binding.yaml => prometheus-cluster-role-binding.yaml} (100%) rename src/main/resources/monitoring/{prometheus/cluster-role.yaml => prometheus-cluster-role.yaml} (100%) rename src/main/resources/monitoring/{prometheus/config.yaml => prometheus-config.yaml} (100%) rename src/main/resources/monitoring/{prometheus/deployment.yaml => prometheus-deployment.yaml} (100%) rename src/main/resources/monitoring/{prometheus/prometheus.yaml => prometheus-prometheus.yaml} (100%) rename src/main/resources/monitoring/{prometheus/service-account.yaml => prometheus-service-account.yaml} (100%) rename src/main/resources/monitoring/{prometheus/service.yaml => prometheus-service.yaml} (100%) create mode 100644 src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc diff --git a/src/main/cljc/dda/c4k_common/monitoring.cljc b/src/main/cljc/dda/c4k_common/monitoring.cljc index 07893e5..1536443 100644 --- a/src/main/cljc/dda/c4k_common/monitoring.cljc +++ b/src/main/cljc/dda/c4k_common/monitoring.cljc @@ -1,20 +1,18 @@ (ns dda.c4k-common.monitoring (:require [clojure.spec.alpha :as s] - #?(:cljs [shadow.resource :as rc]) #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) + [clojure.string :as str] [dda.c4k-common.yaml :as yaml] [dda.c4k-common.predicate :as cp] - [dda.c4k-common.common :as cm] - [clojure.string :as str])) + [dda.c4k-common.monitoring.monitoring-internal :as int])) (s/def ::grafana-cloud-user cp/bash-env-string?) (s/def ::grafana-cloud-password cp/bash-env-string?) (s/def ::grafana-cloud-url string?) (s/def ::cluster-name string?) (s/def ::cluster-stage cp/stage?) -(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) (s/def ::node-regex string?) (s/def ::traefik-regex string?) (s/def ::kube-state-regex string?) @@ -23,11 +21,11 @@ ::cluster-stage])) (s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user ::grafana-cloud-password])) -(s/def ::storage (s/keys :opt-un [::pvc-storage-class-name])) (s/def ::filter-regex (s/keys :req-un [::node-regex ::traefik-regex ::kube-state-regex])) + (def metric-regex {:node-regex (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|" "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|" @@ -46,63 +44,24 @@ (def filter-regex-string (str/join "|" (vals metric-regex))) -#?(:cljs - (defmethod yaml/load-resource :monitoring [resource-name] - (get (inline-resources "monitoring") resource-name))) -(defn-spec generate-stateful-set cp/map-or-seq? - [config ::storage] - (let [{:keys [pvc-storage-class-name] - :or {pvc-storage-class-name :manual}} config] - (-> - (yaml/load-as-edn "monitoring/stateful-set.yaml") - (assoc-in [:spec :volumeClaimTemplates 0 :spec :storageClassName] (name pvc-storage-class-name))))) - -(defn-spec generate-prometheus-config cp/map-or-seq? - [config ::mon-cfg - auth ::mon-auth] - (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config - {:keys [grafana-cloud-user grafana-cloud-password]} auth] - (-> - (yaml/load-as-edn "monitoring/prometheus/prometheus.yaml") - (assoc-in [:global :external_labels :cluster] - cluster-name) - (assoc-in [:global :external_labels :stage] - cluster-stage) - (assoc-in [:remote_write 0 :url] - grafana-cloud-url) - (assoc-in [:remote_write 0 :basic_auth :username] - grafana-cloud-user) - (assoc-in [:remote_write 0 :basic_auth :password] - grafana-cloud-password) - (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string)))) - -(defn-spec generate-config cp/map-or-seq? - [config ::mon-cfg - auth ::mon-auth] - (-> - (yaml/load-as-edn "monitoring/prometheus/config.yaml") - (assoc-in [:stringData :prometheus.yaml] - (yaml/to-string - (generate-prometheus-config config auth))))) - -(defn-spec generate cp/map-or-seq? +(defn-spec generate seq? [config ::mon-cfg auth ::mon-auth] [(yaml/load-as-edn "monitoring/namespace.yaml") - (yaml/load-as-edn "monitoring/prometheus/cluster-role.yaml") - (yaml/load-as-edn "monitoring/prometheus/cluster-role-binding.yaml") - (yaml/load-as-edn "monitoring/prometheus/service.yaml") - (yaml/load-as-edn "monitoring/prometheus/service-account.yaml") - (generate-config config auth) - (yaml/load-as-edn "monitoring/prometheus/deployment.yaml") - (yaml/load-as-edn "monitoring/node-exporter/service-account.yaml") - (yaml/load-as-edn "monitoring/node-exporter/cluster-role.yaml") - (yaml/load-as-edn "monitoring/node-exporter/cluster-role-binding.yaml") - (yaml/load-as-edn "monitoring/node-exporter/daemon-set.yaml") - (yaml/load-as-edn "monitoring/node-exporter/service.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role-binding.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/deployment.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/service-account.yaml") - (yaml/load-as-edn "monitoring/kube-state-metrics/service.yaml")]) + (yaml/load-as-edn "monitoring/prometheus-cluster-role.yaml") + (yaml/load-as-edn "monitoring/prometheus-cluster-role-binding.yaml") + (yaml/load-as-edn "monitoring/prometheus-service.yaml") + (yaml/load-as-edn "monitoring/prometheus-service-account.yaml") + (int/generate-config config auth) + (yaml/load-as-edn "monitoring/prometheus-deployment.yaml") + (yaml/load-as-edn "monitoring/node-exporter-service-account.yaml") + (yaml/load-as-edn "monitoring/node-exporter-cluster-role.yaml") + (yaml/load-as-edn "monitoring/node-exporter-cluster-role-binding.yaml") + (yaml/load-as-edn "monitoring/node-exporter-daemon-set.yaml") + (yaml/load-as-edn "monitoring/node-exporter-service.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role-binding.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-deployment.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-service-account.yaml") + (yaml/load-as-edn "monitoring/kube-state-metrics-service.yaml")]) diff --git a/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc b/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc new file mode 100644 index 0000000..6dfc3a3 --- /dev/null +++ b/src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc @@ -0,0 +1,77 @@ +(ns dda.c4k-common.monitoring.monitoring-internal + (:require + [clojure.spec.alpha :as s] + #?(:clj [orchestra.core :refer [defn-spec]] + :cljs [orchestra.core :refer-macros [defn-spec]]) + #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]]) + [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.predicate :as cp] + [dda.c4k-common.common :as cm] + [clojure.string :as str])) + +(s/def ::grafana-cloud-user cp/bash-env-string?) +(s/def ::grafana-cloud-password cp/bash-env-string?) +(s/def ::grafana-cloud-url string?) +(s/def ::cluster-name string?) +(s/def ::cluster-stage cp/stage?) +(s/def ::node-regex string?) +(s/def ::traefik-regex string?) +(s/def ::kube-state-regex string?) +(s/def ::mon-cfg (s/keys :req-un [::grafana-cloud-url + ::cluster-name + ::cluster-stage])) +(s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user + ::grafana-cloud-password])) +(s/def ::filter-regex (s/keys :req-un [::node-regex + ::traefik-regex + ::kube-state-regex])) + +(def metric-regex {:node-regex + (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|" + "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|" + "node_network_receive_bytes_total|node_network_transmit_bytes_total") + :traefik-regex (str "traefik_entrypoint_.*_total|" + "traefik_entrypoint_.*_seconds_count|" + "traefik_router_.*_total|" + "traefik_router_.*_seconds_count|" + "traefik_service_.*_total|" + "traefik_service_.*_seconds_count|" + "traefik_tls_certs_not_after") + :kube-state-regex (str "kube_pod_container_status_restarts_total|" + "kube_pod_status_reason|kube_node_status_capacity|kube_node_status_allocatable|" + "kube_cronjob_status_active|kube_job_status_failed")}) + +(def filter-regex-string + (str/join "|" (vals metric-regex))) + +(defn-spec generate-prometheus-config cp/map-or-seq? + [config ::mon-cfg + auth ::mon-auth] + (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config + {:keys [grafana-cloud-user grafana-cloud-password]} auth] + (-> + (yaml/load-as-edn "monitoring/prometheus-prometheus.yaml") + (assoc-in [:global :external_labels :cluster] + cluster-name) + (assoc-in [:global :external_labels :stage] + cluster-stage) + (assoc-in [:remote_write 0 :url] + grafana-cloud-url) + (assoc-in [:remote_write 0 :basic_auth :username] + grafana-cloud-user) + (assoc-in [:remote_write 0 :basic_auth :password] + grafana-cloud-password) + (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string)))) + +(defn-spec generate-config cp/map-or-seq? + [config ::mon-cfg + auth ::mon-auth] + (-> + (yaml/load-as-edn "monitoring/prometheus-config.yaml") + (assoc-in [:stringData :prometheus.yaml] + (yaml/to-string + (generate-prometheus-config config auth))))) + +#?(:cljs + (defmethod yaml/load-resource :monitoring [resource-name] + (get (inline-resources "monitoring") resource-name))) diff --git a/src/main/resources/monitoring/kube-state-metrics/cluster-role-binding.yaml b/src/main/resources/monitoring/kube-state-metrics-cluster-role-binding.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/cluster-role-binding.yaml rename to src/main/resources/monitoring/kube-state-metrics-cluster-role-binding.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/cluster-role.yaml b/src/main/resources/monitoring/kube-state-metrics-cluster-role.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/cluster-role.yaml rename to src/main/resources/monitoring/kube-state-metrics-cluster-role.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/deployment.yaml b/src/main/resources/monitoring/kube-state-metrics-deployment.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/deployment.yaml rename to src/main/resources/monitoring/kube-state-metrics-deployment.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/service-account.yaml b/src/main/resources/monitoring/kube-state-metrics-service-account.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/service-account.yaml rename to src/main/resources/monitoring/kube-state-metrics-service-account.yaml diff --git a/src/main/resources/monitoring/kube-state-metrics/service.yaml b/src/main/resources/monitoring/kube-state-metrics-service.yaml similarity index 100% rename from src/main/resources/monitoring/kube-state-metrics/service.yaml rename to src/main/resources/monitoring/kube-state-metrics-service.yaml diff --git a/src/main/resources/monitoring/node-exporter/cluster-role-binding.yaml b/src/main/resources/monitoring/node-exporter-cluster-role-binding.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/cluster-role-binding.yaml rename to src/main/resources/monitoring/node-exporter-cluster-role-binding.yaml diff --git a/src/main/resources/monitoring/node-exporter/cluster-role.yaml b/src/main/resources/monitoring/node-exporter-cluster-role.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/cluster-role.yaml rename to src/main/resources/monitoring/node-exporter-cluster-role.yaml diff --git a/src/main/resources/monitoring/node-exporter/daemon-set.yaml b/src/main/resources/monitoring/node-exporter-daemon-set.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/daemon-set.yaml rename to src/main/resources/monitoring/node-exporter-daemon-set.yaml diff --git a/src/main/resources/monitoring/node-exporter/service-account.yaml b/src/main/resources/monitoring/node-exporter-service-account.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/service-account.yaml rename to src/main/resources/monitoring/node-exporter-service-account.yaml diff --git a/src/main/resources/monitoring/node-exporter/service.yaml b/src/main/resources/monitoring/node-exporter-service.yaml similarity index 100% rename from src/main/resources/monitoring/node-exporter/service.yaml rename to src/main/resources/monitoring/node-exporter-service.yaml diff --git a/src/main/resources/monitoring/prometheus/cluster-role-binding.yaml b/src/main/resources/monitoring/prometheus-cluster-role-binding.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/cluster-role-binding.yaml rename to src/main/resources/monitoring/prometheus-cluster-role-binding.yaml diff --git a/src/main/resources/monitoring/prometheus/cluster-role.yaml b/src/main/resources/monitoring/prometheus-cluster-role.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/cluster-role.yaml rename to src/main/resources/monitoring/prometheus-cluster-role.yaml diff --git a/src/main/resources/monitoring/prometheus/config.yaml b/src/main/resources/monitoring/prometheus-config.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/config.yaml rename to src/main/resources/monitoring/prometheus-config.yaml diff --git a/src/main/resources/monitoring/prometheus/deployment.yaml b/src/main/resources/monitoring/prometheus-deployment.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/deployment.yaml rename to src/main/resources/monitoring/prometheus-deployment.yaml diff --git a/src/main/resources/monitoring/prometheus/prometheus.yaml b/src/main/resources/monitoring/prometheus-prometheus.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/prometheus.yaml rename to src/main/resources/monitoring/prometheus-prometheus.yaml diff --git a/src/main/resources/monitoring/prometheus/service-account.yaml b/src/main/resources/monitoring/prometheus-service-account.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/service-account.yaml rename to src/main/resources/monitoring/prometheus-service-account.yaml diff --git a/src/main/resources/monitoring/prometheus/service.yaml b/src/main/resources/monitoring/prometheus-service.yaml similarity index 100% rename from src/main/resources/monitoring/prometheus/service.yaml rename to src/main/resources/monitoring/prometheus-service.yaml diff --git a/src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc b/src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc new file mode 100644 index 0000000..b763456 --- /dev/null +++ b/src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc @@ -0,0 +1,59 @@ +(ns dda.c4k-common.monitoring.monitoring-internal-test + (:require + #?(:clj [clojure.test :refer [deftest is are testing run-tests]] + :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + [clojure.string :as str] + [clojure.spec.test.alpha :as st] + [dda.c4k-common.monitoring.monitoring-internal :as cut])) + +(st/instrument `cut/generate-stateful-set) +(st/instrument `cut/generate-agent-config) +(st/instrument `cut/generate-config) + +(def conf {:cluster-name "clustername" + :cluster-stage "test" + :grafana-cloud-url "https://some.url/with/path"}) + +(def auth {:grafana-cloud-user "user" + :grafana-cloud-password "password" + :hetzner-cloud-ro-token "ro-token"}) + +(def invalid-conf {:cluster-name "clustername" + :cluster-stage "test" + :grafana-clud-url "https://some.url/with/path"}) + +(def invalid-auth {:grafana-cloud-user "user" + :grafana-clod-password "password" + :hetzner-cloud-ro-token "ro-token"}) + +(deftest should-not-generate-config + (is (thrown? + #?(:clj Exception :cljs js/Error) + (cut/generate-config invalid-conf auth)))) + +(deftest should-not-generate-auth + (is (thrown? + #?(:clj Exception :cljs js/Error) + (cut/generate-config conf invalid-auth)))) + + +(deftest should-generate-prometheus-remote-write-auth + (is (= {:username "user", + :password "password"} + (get-in + (cut/generate-prometheus-config conf auth) + [:remote_write 0 :basic_auth])))) + +(deftest should-generate-prometheus-external-labels + (is (= {:cluster "clustername", + :stage "test"} + (get-in + (cut/generate-prometheus-config conf auth) + [:global :external_labels])))) + +(deftest should-generate-config + (is (str/starts-with? + (get-in + (cut/generate-config conf auth) + [:stringData :prometheus.yaml]) + "global:\n scrape_interval:"))) \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_common/monitoring_test.cljc b/src/test/cljc/dda/c4k_common/monitoring_test.cljc index 76dfa2c..5e5332b 100644 --- a/src/test/cljc/dda/c4k_common/monitoring_test.cljc +++ b/src/test/cljc/dda/c4k_common/monitoring_test.cljc @@ -2,16 +2,10 @@ (:require #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - [clojure.string :as s] [clojure.spec.test.alpha :as st] - [dda.c4k-common.monitoring :as cut] - [dda.c4k-common.yaml :as yaml] - [clojure.string :as str])) + [dda.c4k-common.monitoring :as cut])) (st/instrument `cut/generate) -(st/instrument `cut/generate-stateful-set) -(st/instrument `cut/generate-agent-config) -(st/instrument `cut/generate-config) (def conf {:cluster-name "clustername" :cluster-stage "test" @@ -21,45 +15,7 @@ :grafana-cloud-password "password" :hetzner-cloud-ro-token "ro-token"}) -(def invalid-conf {:cluster-name "clustername" - :cluster-stage "test" - :grafana-clud-url "https://some.url/with/path"}) - -(def invalid-auth {:grafana-cloud-user "user" - :grafana-clod-password "password" - :hetzner-cloud-ro-token "ro-token"}) - -(deftest should-not-generate-config - (is (thrown? - #?(:clj Exception :cljs js/Error) - (cut/generate-config invalid-conf auth)))) - -(deftest should-not-generate-auth - (is (thrown? - #?(:clj Exception :cljs js/Error) - (cut/generate-config conf invalid-auth)))) (deftest should-generate (is (= 17 (count (cut/generate conf auth))))) - -(deftest should-generate-prometheus-remote-write-auth - (is (= {:username "user", - :password "password"} - (get-in - (cut/generate-prometheus-config conf auth) - [:remote_write 0 :basic_auth])))) - -(deftest should-generate-prometheus-external-labels - (is (= {:cluster "clustername", - :stage "test"} - (get-in - (cut/generate-prometheus-config conf auth) - [:global :external_labels])))) - -(deftest should-generate-config - (is (s/starts-with? - (get-in - (cut/generate-config conf auth) - [:stringData :prometheus.yaml]) - "global:\n scrape_interval:"))) \ No newline at end of file