Compare commits
2 commits
6794a24f0f
...
ab2473bdf4
Author | SHA1 | Date | |
---|---|---|---|
ab2473bdf4 | |||
b4718500b1 |
22 changed files with 158 additions and 106 deletions
|
@ -1,20 +1,18 @@
|
||||||
(ns dda.c4k-common.monitoring
|
(ns dda.c4k-common.monitoring
|
||||||
(:require
|
(:require
|
||||||
[clojure.spec.alpha :as s]
|
[clojure.spec.alpha :as s]
|
||||||
#?(:cljs [shadow.resource :as rc])
|
|
||||||
#?(:clj [orchestra.core :refer [defn-spec]]
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
:cljs [orchestra.core :refer-macros [defn-spec]])
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
|
[clojure.string :as str]
|
||||||
[dda.c4k-common.yaml :as yaml]
|
[dda.c4k-common.yaml :as yaml]
|
||||||
[dda.c4k-common.predicate :as cp]
|
[dda.c4k-common.predicate :as cp]
|
||||||
[dda.c4k-common.common :as cm]
|
[dda.c4k-common.monitoring.monitoring-internal :as int]))
|
||||||
[clojure.string :as str]))
|
|
||||||
|
|
||||||
(s/def ::grafana-cloud-user cp/bash-env-string?)
|
(s/def ::grafana-cloud-user cp/bash-env-string?)
|
||||||
(s/def ::grafana-cloud-password cp/bash-env-string?)
|
(s/def ::grafana-cloud-password cp/bash-env-string?)
|
||||||
(s/def ::grafana-cloud-url string?)
|
(s/def ::grafana-cloud-url string?)
|
||||||
(s/def ::cluster-name string?)
|
(s/def ::cluster-name string?)
|
||||||
(s/def ::cluster-stage cp/stage?)
|
(s/def ::cluster-stage cp/stage?)
|
||||||
(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?)
|
|
||||||
(s/def ::node-regex string?)
|
(s/def ::node-regex string?)
|
||||||
(s/def ::traefik-regex string?)
|
(s/def ::traefik-regex string?)
|
||||||
(s/def ::kube-state-regex string?)
|
(s/def ::kube-state-regex string?)
|
||||||
|
@ -23,11 +21,11 @@
|
||||||
::cluster-stage]))
|
::cluster-stage]))
|
||||||
(s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user
|
(s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user
|
||||||
::grafana-cloud-password]))
|
::grafana-cloud-password]))
|
||||||
(s/def ::storage (s/keys :opt-un [::pvc-storage-class-name]))
|
|
||||||
(s/def ::filter-regex (s/keys :req-un [::node-regex
|
(s/def ::filter-regex (s/keys :req-un [::node-regex
|
||||||
::traefik-regex
|
::traefik-regex
|
||||||
::kube-state-regex]))
|
::kube-state-regex]))
|
||||||
|
|
||||||
|
|
||||||
(def metric-regex {:node-regex
|
(def metric-regex {:node-regex
|
||||||
(str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|"
|
(str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|"
|
||||||
"node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|"
|
"node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|"
|
||||||
|
@ -46,63 +44,24 @@
|
||||||
(def filter-regex-string
|
(def filter-regex-string
|
||||||
(str/join "|" (vals metric-regex)))
|
(str/join "|" (vals metric-regex)))
|
||||||
|
|
||||||
#?(:cljs
|
|
||||||
(defmethod yaml/load-resource :monitoring [resource-name]
|
|
||||||
(get (inline-resources "monitoring") resource-name)))
|
|
||||||
|
|
||||||
(defn-spec generate-stateful-set cp/map-or-seq?
|
(defn-spec generate seq?
|
||||||
[config ::storage]
|
|
||||||
(let [{:keys [pvc-storage-class-name]
|
|
||||||
:or {pvc-storage-class-name :manual}} config]
|
|
||||||
(->
|
|
||||||
(yaml/load-as-edn "monitoring/stateful-set.yaml")
|
|
||||||
(assoc-in [:spec :volumeClaimTemplates 0 :spec :storageClassName] (name pvc-storage-class-name)))))
|
|
||||||
|
|
||||||
(defn-spec generate-prometheus-config cp/map-or-seq?
|
|
||||||
[config ::mon-cfg
|
|
||||||
auth ::mon-auth]
|
|
||||||
(let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config
|
|
||||||
{:keys [grafana-cloud-user grafana-cloud-password]} auth]
|
|
||||||
(->
|
|
||||||
(yaml/load-as-edn "monitoring/prometheus/prometheus.yaml")
|
|
||||||
(assoc-in [:global :external_labels :cluster]
|
|
||||||
cluster-name)
|
|
||||||
(assoc-in [:global :external_labels :stage]
|
|
||||||
cluster-stage)
|
|
||||||
(assoc-in [:remote_write 0 :url]
|
|
||||||
grafana-cloud-url)
|
|
||||||
(assoc-in [:remote_write 0 :basic_auth :username]
|
|
||||||
grafana-cloud-user)
|
|
||||||
(assoc-in [:remote_write 0 :basic_auth :password]
|
|
||||||
grafana-cloud-password)
|
|
||||||
(cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string))))
|
|
||||||
|
|
||||||
(defn-spec generate-config cp/map-or-seq?
|
|
||||||
[config ::mon-cfg
|
|
||||||
auth ::mon-auth]
|
|
||||||
(->
|
|
||||||
(yaml/load-as-edn "monitoring/prometheus/config.yaml")
|
|
||||||
(assoc-in [:stringData :prometheus.yaml]
|
|
||||||
(yaml/to-string
|
|
||||||
(generate-prometheus-config config auth)))))
|
|
||||||
|
|
||||||
(defn-spec generate cp/map-or-seq?
|
|
||||||
[config ::mon-cfg
|
[config ::mon-cfg
|
||||||
auth ::mon-auth]
|
auth ::mon-auth]
|
||||||
[(yaml/load-as-edn "monitoring/namespace.yaml")
|
[(yaml/load-as-edn "monitoring/namespace.yaml")
|
||||||
(yaml/load-as-edn "monitoring/prometheus/cluster-role.yaml")
|
(yaml/load-as-edn "monitoring/prometheus-cluster-role.yaml")
|
||||||
(yaml/load-as-edn "monitoring/prometheus/cluster-role-binding.yaml")
|
(yaml/load-as-edn "monitoring/prometheus-cluster-role-binding.yaml")
|
||||||
(yaml/load-as-edn "monitoring/prometheus/service.yaml")
|
(yaml/load-as-edn "monitoring/prometheus-service.yaml")
|
||||||
(yaml/load-as-edn "monitoring/prometheus/service-account.yaml")
|
(yaml/load-as-edn "monitoring/prometheus-service-account.yaml")
|
||||||
(generate-config config auth)
|
(int/generate-config config auth)
|
||||||
(yaml/load-as-edn "monitoring/prometheus/deployment.yaml")
|
(yaml/load-as-edn "monitoring/prometheus-deployment.yaml")
|
||||||
(yaml/load-as-edn "monitoring/node-exporter/service-account.yaml")
|
(yaml/load-as-edn "monitoring/node-exporter-service-account.yaml")
|
||||||
(yaml/load-as-edn "monitoring/node-exporter/cluster-role.yaml")
|
(yaml/load-as-edn "monitoring/node-exporter-cluster-role.yaml")
|
||||||
(yaml/load-as-edn "monitoring/node-exporter/cluster-role-binding.yaml")
|
(yaml/load-as-edn "monitoring/node-exporter-cluster-role-binding.yaml")
|
||||||
(yaml/load-as-edn "monitoring/node-exporter/daemon-set.yaml")
|
(yaml/load-as-edn "monitoring/node-exporter-daemon-set.yaml")
|
||||||
(yaml/load-as-edn "monitoring/node-exporter/service.yaml")
|
(yaml/load-as-edn "monitoring/node-exporter-service.yaml")
|
||||||
(yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role-binding.yaml")
|
(yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role-binding.yaml")
|
||||||
(yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role.yaml")
|
(yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role.yaml")
|
||||||
(yaml/load-as-edn "monitoring/kube-state-metrics/deployment.yaml")
|
(yaml/load-as-edn "monitoring/kube-state-metrics-deployment.yaml")
|
||||||
(yaml/load-as-edn "monitoring/kube-state-metrics/service-account.yaml")
|
(yaml/load-as-edn "monitoring/kube-state-metrics-service-account.yaml")
|
||||||
(yaml/load-as-edn "monitoring/kube-state-metrics/service.yaml")])
|
(yaml/load-as-edn "monitoring/kube-state-metrics-service.yaml")])
|
||||||
|
|
|
@ -0,0 +1,77 @@
|
||||||
|
(ns dda.c4k-common.monitoring.monitoring-internal
|
||||||
|
(:require
|
||||||
|
[clojure.spec.alpha :as s]
|
||||||
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
|
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])
|
||||||
|
[dda.c4k-common.yaml :as yaml]
|
||||||
|
[dda.c4k-common.predicate :as cp]
|
||||||
|
[dda.c4k-common.common :as cm]
|
||||||
|
[clojure.string :as str]))
|
||||||
|
|
||||||
|
(s/def ::grafana-cloud-user cp/bash-env-string?)
|
||||||
|
(s/def ::grafana-cloud-password cp/bash-env-string?)
|
||||||
|
(s/def ::grafana-cloud-url string?)
|
||||||
|
(s/def ::cluster-name string?)
|
||||||
|
(s/def ::cluster-stage cp/stage?)
|
||||||
|
(s/def ::node-regex string?)
|
||||||
|
(s/def ::traefik-regex string?)
|
||||||
|
(s/def ::kube-state-regex string?)
|
||||||
|
(s/def ::mon-cfg (s/keys :req-un [::grafana-cloud-url
|
||||||
|
::cluster-name
|
||||||
|
::cluster-stage]))
|
||||||
|
(s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user
|
||||||
|
::grafana-cloud-password]))
|
||||||
|
(s/def ::filter-regex (s/keys :req-un [::node-regex
|
||||||
|
::traefik-regex
|
||||||
|
::kube-state-regex]))
|
||||||
|
|
||||||
|
(def metric-regex {:node-regex
|
||||||
|
(str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|"
|
||||||
|
"node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|"
|
||||||
|
"node_network_receive_bytes_total|node_network_transmit_bytes_total")
|
||||||
|
:traefik-regex (str "traefik_entrypoint_.*_total|"
|
||||||
|
"traefik_entrypoint_.*_seconds_count|"
|
||||||
|
"traefik_router_.*_total|"
|
||||||
|
"traefik_router_.*_seconds_count|"
|
||||||
|
"traefik_service_.*_total|"
|
||||||
|
"traefik_service_.*_seconds_count|"
|
||||||
|
"traefik_tls_certs_not_after")
|
||||||
|
:kube-state-regex (str "kube_pod_container_status_restarts_total|"
|
||||||
|
"kube_pod_status_reason|kube_node_status_capacity|kube_node_status_allocatable|"
|
||||||
|
"kube_cronjob_status_active|kube_job_status_failed")})
|
||||||
|
|
||||||
|
(def filter-regex-string
|
||||||
|
(str/join "|" (vals metric-regex)))
|
||||||
|
|
||||||
|
(defn-spec generate-prometheus-config cp/map-or-seq?
|
||||||
|
[config ::mon-cfg
|
||||||
|
auth ::mon-auth]
|
||||||
|
(let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config
|
||||||
|
{:keys [grafana-cloud-user grafana-cloud-password]} auth]
|
||||||
|
(->
|
||||||
|
(yaml/load-as-edn "monitoring/prometheus-prometheus.yaml")
|
||||||
|
(assoc-in [:global :external_labels :cluster]
|
||||||
|
cluster-name)
|
||||||
|
(assoc-in [:global :external_labels :stage]
|
||||||
|
cluster-stage)
|
||||||
|
(assoc-in [:remote_write 0 :url]
|
||||||
|
grafana-cloud-url)
|
||||||
|
(assoc-in [:remote_write 0 :basic_auth :username]
|
||||||
|
grafana-cloud-user)
|
||||||
|
(assoc-in [:remote_write 0 :basic_auth :password]
|
||||||
|
grafana-cloud-password)
|
||||||
|
(cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string))))
|
||||||
|
|
||||||
|
(defn-spec generate-config cp/map-or-seq?
|
||||||
|
[config ::mon-cfg
|
||||||
|
auth ::mon-auth]
|
||||||
|
(->
|
||||||
|
(yaml/load-as-edn "monitoring/prometheus-config.yaml")
|
||||||
|
(assoc-in [:stringData :prometheus.yaml]
|
||||||
|
(yaml/to-string
|
||||||
|
(generate-prometheus-config config auth)))))
|
||||||
|
|
||||||
|
#?(:cljs
|
||||||
|
(defmethod yaml/load-resource :monitoring [resource-name]
|
||||||
|
(get (inline-resources "monitoring") resource-name)))
|
|
@ -4,6 +4,7 @@
|
||||||
#?(:cljs [shadow.resource :as rc])
|
#?(:cljs [shadow.resource :as rc])
|
||||||
#?(:clj [orchestra.core :refer [defn-spec]]
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
:cljs [orchestra.core :refer-macros [defn-spec]])
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
|
#?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])
|
||||||
[dda.c4k-common.yaml :as yaml]
|
[dda.c4k-common.yaml :as yaml]
|
||||||
[dda.c4k-common.base64 :as b64]
|
[dda.c4k-common.base64 :as b64]
|
||||||
[dda.c4k-common.predicate :as cp]
|
[dda.c4k-common.predicate :as cp]
|
||||||
|
|
|
@ -0,0 +1,59 @@
|
||||||
|
(ns dda.c4k-common.monitoring.monitoring-internal-test
|
||||||
|
(:require
|
||||||
|
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
||||||
|
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
||||||
|
[clojure.string :as str]
|
||||||
|
[clojure.spec.test.alpha :as st]
|
||||||
|
[dda.c4k-common.monitoring.monitoring-internal :as cut]))
|
||||||
|
|
||||||
|
(st/instrument `cut/generate-stateful-set)
|
||||||
|
(st/instrument `cut/generate-agent-config)
|
||||||
|
(st/instrument `cut/generate-config)
|
||||||
|
|
||||||
|
(def conf {:cluster-name "clustername"
|
||||||
|
:cluster-stage "test"
|
||||||
|
:grafana-cloud-url "https://some.url/with/path"})
|
||||||
|
|
||||||
|
(def auth {:grafana-cloud-user "user"
|
||||||
|
:grafana-cloud-password "password"
|
||||||
|
:hetzner-cloud-ro-token "ro-token"})
|
||||||
|
|
||||||
|
(def invalid-conf {:cluster-name "clustername"
|
||||||
|
:cluster-stage "test"
|
||||||
|
:grafana-clud-url "https://some.url/with/path"})
|
||||||
|
|
||||||
|
(def invalid-auth {:grafana-cloud-user "user"
|
||||||
|
:grafana-clod-password "password"
|
||||||
|
:hetzner-cloud-ro-token "ro-token"})
|
||||||
|
|
||||||
|
(deftest should-not-generate-config
|
||||||
|
(is (thrown?
|
||||||
|
#?(:clj Exception :cljs js/Error)
|
||||||
|
(cut/generate-config invalid-conf auth))))
|
||||||
|
|
||||||
|
(deftest should-not-generate-auth
|
||||||
|
(is (thrown?
|
||||||
|
#?(:clj Exception :cljs js/Error)
|
||||||
|
(cut/generate-config conf invalid-auth))))
|
||||||
|
|
||||||
|
|
||||||
|
(deftest should-generate-prometheus-remote-write-auth
|
||||||
|
(is (= {:username "user",
|
||||||
|
:password "password"}
|
||||||
|
(get-in
|
||||||
|
(cut/generate-prometheus-config conf auth)
|
||||||
|
[:remote_write 0 :basic_auth]))))
|
||||||
|
|
||||||
|
(deftest should-generate-prometheus-external-labels
|
||||||
|
(is (= {:cluster "clustername",
|
||||||
|
:stage "test"}
|
||||||
|
(get-in
|
||||||
|
(cut/generate-prometheus-config conf auth)
|
||||||
|
[:global :external_labels]))))
|
||||||
|
|
||||||
|
(deftest should-generate-config
|
||||||
|
(is (str/starts-with?
|
||||||
|
(get-in
|
||||||
|
(cut/generate-config conf auth)
|
||||||
|
[:stringData :prometheus.yaml])
|
||||||
|
"global:\n scrape_interval:")))
|
|
@ -2,16 +2,10 @@
|
||||||
(:require
|
(:require
|
||||||
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
#?(:clj [clojure.test :refer [deftest is are testing run-tests]]
|
||||||
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
:cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
|
||||||
[clojure.string :as s]
|
|
||||||
[clojure.spec.test.alpha :as st]
|
[clojure.spec.test.alpha :as st]
|
||||||
[dda.c4k-common.monitoring :as cut]
|
[dda.c4k-common.monitoring :as cut]))
|
||||||
[dda.c4k-common.yaml :as yaml]
|
|
||||||
[clojure.string :as str]))
|
|
||||||
|
|
||||||
(st/instrument `cut/generate)
|
(st/instrument `cut/generate)
|
||||||
(st/instrument `cut/generate-stateful-set)
|
|
||||||
(st/instrument `cut/generate-agent-config)
|
|
||||||
(st/instrument `cut/generate-config)
|
|
||||||
|
|
||||||
(def conf {:cluster-name "clustername"
|
(def conf {:cluster-name "clustername"
|
||||||
:cluster-stage "test"
|
:cluster-stage "test"
|
||||||
|
@ -21,45 +15,7 @@
|
||||||
:grafana-cloud-password "password"
|
:grafana-cloud-password "password"
|
||||||
:hetzner-cloud-ro-token "ro-token"})
|
:hetzner-cloud-ro-token "ro-token"})
|
||||||
|
|
||||||
(def invalid-conf {:cluster-name "clustername"
|
|
||||||
:cluster-stage "test"
|
|
||||||
:grafana-clud-url "https://some.url/with/path"})
|
|
||||||
|
|
||||||
(def invalid-auth {:grafana-cloud-user "user"
|
|
||||||
:grafana-clod-password "password"
|
|
||||||
:hetzner-cloud-ro-token "ro-token"})
|
|
||||||
|
|
||||||
(deftest should-not-generate-config
|
|
||||||
(is (thrown?
|
|
||||||
#?(:clj Exception :cljs js/Error)
|
|
||||||
(cut/generate-config invalid-conf auth))))
|
|
||||||
|
|
||||||
(deftest should-not-generate-auth
|
|
||||||
(is (thrown?
|
|
||||||
#?(:clj Exception :cljs js/Error)
|
|
||||||
(cut/generate-config conf invalid-auth))))
|
|
||||||
|
|
||||||
(deftest should-generate
|
(deftest should-generate
|
||||||
(is (= 17
|
(is (= 17
|
||||||
(count (cut/generate conf auth)))))
|
(count (cut/generate conf auth)))))
|
||||||
|
|
||||||
(deftest should-generate-prometheus-remote-write-auth
|
|
||||||
(is (= {:username "user",
|
|
||||||
:password "password"}
|
|
||||||
(get-in
|
|
||||||
(cut/generate-prometheus-config conf auth)
|
|
||||||
[:remote_write 0 :basic_auth]))))
|
|
||||||
|
|
||||||
(deftest should-generate-prometheus-external-labels
|
|
||||||
(is (= {:cluster "clustername",
|
|
||||||
:stage "test"}
|
|
||||||
(get-in
|
|
||||||
(cut/generate-prometheus-config conf auth)
|
|
||||||
[:global :external_labels]))))
|
|
||||||
|
|
||||||
(deftest should-generate-config
|
|
||||||
(is (s/starts-with?
|
|
||||||
(get-in
|
|
||||||
(cut/generate-config conf auth)
|
|
||||||
[:stringData :prometheus.yaml])
|
|
||||||
"global:\n scrape_interval:")))
|
|
Loading…
Reference in a new issue