introduce internal for monitoring

src/main/cljc/dda/c4k_common/monitoring.cljc

@@ -1,20 +1,18 @@
 (ns dda.c4k-common.monitoring
   (:require
    [clojure.spec.alpha :as s]
-   #?(:cljs [shadow.resource :as rc])
    #?(:clj [orchestra.core :refer [defn-spec]]
       :cljs [orchestra.core :refer-macros [defn-spec]])
+   [clojure.string :as str]
    [dda.c4k-common.yaml :as yaml]
    [dda.c4k-common.predicate :as cp]
-   [dda.c4k-common.common :as cm]
-   [clojure.string :as str]))
+   [dda.c4k-common.monitoring.monitoring-internal :as int]))
 
 (s/def ::grafana-cloud-user cp/bash-env-string?)
 (s/def ::grafana-cloud-password cp/bash-env-string?)
 (s/def ::grafana-cloud-url string?)
 (s/def ::cluster-name string?)
 (s/def ::cluster-stage cp/stage?)
-(s/def ::pvc-storage-class-name cp/pvc-storage-class-name?)
 (s/def ::node-regex string?)
 (s/def ::traefik-regex string?)
 (s/def ::kube-state-regex string?)
@@ -23,11 +21,11 @@
                                  ::cluster-stage]))
 (s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user 
                                ::grafana-cloud-password]))
-(s/def ::storage (s/keys :opt-un [::pvc-storage-class-name]))
 (s/def ::filter-regex (s/keys :req-un [::node-regex 
                                        ::traefik-regex 
                                        ::kube-state-regex]))
 
+
 (def metric-regex {:node-regex
                    (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|"
                         "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|"
@@ -46,63 +44,24 @@
 (def filter-regex-string
   (str/join "|" (vals metric-regex)))
 
-#?(:cljs
-   (defmethod yaml/load-resource :monitoring [resource-name]
-      (get (inline-resources "monitoring") resource-name)))
 
-(defn-spec generate-stateful-set cp/map-or-seq?
-  [config ::storage]
-  (let [{:keys [pvc-storage-class-name]
-         :or {pvc-storage-class-name :manual}} config]
-    (->
-     (yaml/load-as-edn "monitoring/stateful-set.yaml")
-     (assoc-in [:spec :volumeClaimTemplates 0 :spec :storageClassName] (name pvc-storage-class-name)))))
-
-(defn-spec generate-prometheus-config cp/map-or-seq?
-  [config ::mon-cfg
-   auth ::mon-auth]
-  (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config
-        {:keys [grafana-cloud-user grafana-cloud-password]} auth]
-    (->
-     (yaml/load-as-edn "monitoring/prometheus/prometheus.yaml")
-     (assoc-in [:global :external_labels :cluster]
-               cluster-name)
-     (assoc-in [:global :external_labels :stage]
-               cluster-stage)
-     (assoc-in [:remote_write 0 :url]
-               grafana-cloud-url)
-     (assoc-in [:remote_write 0 :basic_auth :username]
-               grafana-cloud-user)
-     (assoc-in [:remote_write 0 :basic_auth :password]
-               grafana-cloud-password)
-     (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string))))
-
-(defn-spec generate-config cp/map-or-seq?
-  [config ::mon-cfg
-   auth ::mon-auth]
-  (->
-   (yaml/load-as-edn "monitoring/prometheus/config.yaml")
-   (assoc-in [:stringData :prometheus.yaml]
-             (yaml/to-string
-              (generate-prometheus-config config auth)))))
-
-(defn-spec generate cp/map-or-seq?
+(defn-spec generate seq?
   [config ::mon-cfg
    auth ::mon-auth]
   [(yaml/load-as-edn "monitoring/namespace.yaml")
-   (yaml/load-as-edn "monitoring/prometheus/cluster-role.yaml")
-   (yaml/load-as-edn "monitoring/prometheus/cluster-role-binding.yaml")
-   (yaml/load-as-edn "monitoring/prometheus/service.yaml")
-   (yaml/load-as-edn "monitoring/prometheus/service-account.yaml")
-   (generate-config config auth)
-   (yaml/load-as-edn "monitoring/prometheus/deployment.yaml")
-   (yaml/load-as-edn "monitoring/node-exporter/service-account.yaml")
-   (yaml/load-as-edn "monitoring/node-exporter/cluster-role.yaml")
-   (yaml/load-as-edn "monitoring/node-exporter/cluster-role-binding.yaml")
-   (yaml/load-as-edn "monitoring/node-exporter/daemon-set.yaml")
-   (yaml/load-as-edn "monitoring/node-exporter/service.yaml")
-   (yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role-binding.yaml")
-   (yaml/load-as-edn "monitoring/kube-state-metrics/cluster-role.yaml")
-   (yaml/load-as-edn "monitoring/kube-state-metrics/deployment.yaml")
-   (yaml/load-as-edn "monitoring/kube-state-metrics/service-account.yaml")
-   (yaml/load-as-edn "monitoring/kube-state-metrics/service.yaml")])
+   (yaml/load-as-edn "monitoring/prometheus-cluster-role.yaml")
+   (yaml/load-as-edn "monitoring/prometheus-cluster-role-binding.yaml")
+   (yaml/load-as-edn "monitoring/prometheus-service.yaml")
+   (yaml/load-as-edn "monitoring/prometheus-service-account.yaml")
+   (int/generate-config config auth)
+   (yaml/load-as-edn "monitoring/prometheus-deployment.yaml")
+   (yaml/load-as-edn "monitoring/node-exporter-service-account.yaml")
+   (yaml/load-as-edn "monitoring/node-exporter-cluster-role.yaml")
+   (yaml/load-as-edn "monitoring/node-exporter-cluster-role-binding.yaml")
+   (yaml/load-as-edn "monitoring/node-exporter-daemon-set.yaml")
+   (yaml/load-as-edn "monitoring/node-exporter-service.yaml")
+   (yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role-binding.yaml")
+   (yaml/load-as-edn "monitoring/kube-state-metrics-cluster-role.yaml")
+   (yaml/load-as-edn "monitoring/kube-state-metrics-deployment.yaml")
+   (yaml/load-as-edn "monitoring/kube-state-metrics-service-account.yaml")
+   (yaml/load-as-edn "monitoring/kube-state-metrics-service.yaml")])

src/main/cljc/dda/c4k_common/monitoring/monitoring_internal.cljc

@@ -0,0 +1,77 @@
+(ns dda.c4k-common.monitoring.monitoring-internal
+  (:require
+   [clojure.spec.alpha :as s]
+   #?(:clj [orchestra.core :refer [defn-spec]]
+      :cljs [orchestra.core :refer-macros [defn-spec]])
+   #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])
+   [dda.c4k-common.yaml :as yaml]
+   [dda.c4k-common.predicate :as cp]
+   [dda.c4k-common.common :as cm]
+   [clojure.string :as str]))
+
+(s/def ::grafana-cloud-user cp/bash-env-string?)
+(s/def ::grafana-cloud-password cp/bash-env-string?)
+(s/def ::grafana-cloud-url string?)
+(s/def ::cluster-name string?)
+(s/def ::cluster-stage cp/stage?)
+(s/def ::node-regex string?)
+(s/def ::traefik-regex string?)
+(s/def ::kube-state-regex string?)
+(s/def ::mon-cfg (s/keys :req-un [::grafana-cloud-url
+                                 ::cluster-name
+                                 ::cluster-stage]))
+(s/def ::mon-auth (s/keys :req-un [::grafana-cloud-user 
+                               ::grafana-cloud-password]))
+(s/def ::filter-regex (s/keys :req-un [::node-regex 
+                                       ::traefik-regex 
+                                       ::kube-state-regex]))
+
+(def metric-regex {:node-regex
+                   (str "node_cpu_sec.+|node_load[0-9]+|node_memory_Buf.*|node_memory_Mem.*|"
+                        "node_memory_Cached.*|node_disk_[r,w,i].*|node_filesystem_[s,a].*|"
+                        "node_network_receive_bytes_total|node_network_transmit_bytes_total")
+                   :traefik-regex (str "traefik_entrypoint_.*_total|"
+                                       "traefik_entrypoint_.*_seconds_count|"
+                                       "traefik_router_.*_total|"
+                                       "traefik_router_.*_seconds_count|"
+                                       "traefik_service_.*_total|"
+                                       "traefik_service_.*_seconds_count|"
+                                       "traefik_tls_certs_not_after")
+                   :kube-state-regex (str "kube_pod_container_status_restarts_total|"
+                                          "kube_pod_status_reason|kube_node_status_capacity|kube_node_status_allocatable|"
+                                          "kube_cronjob_status_active|kube_job_status_failed")})
+
+(def filter-regex-string
+  (str/join "|" (vals metric-regex)))
+
+(defn-spec generate-prometheus-config cp/map-or-seq?
+  [config ::mon-cfg
+   auth ::mon-auth]
+  (let [{:keys [grafana-cloud-url cluster-name cluster-stage]} config
+        {:keys [grafana-cloud-user grafana-cloud-password]} auth]
+    (->
+     (yaml/load-as-edn "monitoring/prometheus-prometheus.yaml")
+     (assoc-in [:global :external_labels :cluster]
+               cluster-name)
+     (assoc-in [:global :external_labels :stage]
+               cluster-stage)
+     (assoc-in [:remote_write 0 :url]
+               grafana-cloud-url)
+     (assoc-in [:remote_write 0 :basic_auth :username]
+               grafana-cloud-user)
+     (assoc-in [:remote_write 0 :basic_auth :password]
+               grafana-cloud-password)
+     (cm/replace-all-matching-values-by-new-value "FILTER_REGEX" filter-regex-string))))
+
+(defn-spec generate-config cp/map-or-seq?
+  [config ::mon-cfg
+   auth ::mon-auth]
+  (->
+   (yaml/load-as-edn "monitoring/prometheus-config.yaml")
+   (assoc-in [:stringData :prometheus.yaml]
+             (yaml/to-string
+              (generate-prometheus-config config auth)))))
+
+#?(:cljs
+   (defmethod yaml/load-resource :monitoring [resource-name]
+     (get (inline-resources "monitoring") resource-name)))

src/main/cljc/dda/c4k_common/postgres.cljc

@@ -4,6 +4,7 @@
    #?(:cljs [shadow.resource :as rc])
    #?(:clj [orchestra.core :refer [defn-spec]]
       :cljs [orchestra.core :refer-macros [defn-spec]])
+   #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])
    [dda.c4k-common.yaml :as yaml]
    [dda.c4k-common.base64 :as b64]
    [dda.c4k-common.predicate :as cp]

src/test/cljc/dda/c4k_common/monitoring/monitoring_internal_test.cljc

@@ -0,0 +1,59 @@
+(ns dda.c4k-common.monitoring.monitoring-internal-test
+  (:require
+   #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
+      :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
+   [clojure.string :as str]
+   [clojure.spec.test.alpha :as st]
+   [dda.c4k-common.monitoring.monitoring-internal :as cut]))
+
+(st/instrument `cut/generate-stateful-set)
+(st/instrument `cut/generate-agent-config)
+(st/instrument `cut/generate-config)
+
+(def conf {:cluster-name "clustername"
+           :cluster-stage "test"
+           :grafana-cloud-url "https://some.url/with/path"})
+
+(def auth {:grafana-cloud-user "user"
+           :grafana-cloud-password "password"
+           :hetzner-cloud-ro-token "ro-token"})
+
+(def invalid-conf {:cluster-name "clustername"
+                   :cluster-stage "test"
+                   :grafana-clud-url "https://some.url/with/path"})
+
+(def invalid-auth {:grafana-cloud-user "user"
+                   :grafana-clod-password "password"
+                   :hetzner-cloud-ro-token "ro-token"})
+
+(deftest should-not-generate-config
+  (is (thrown?
+       #?(:clj Exception :cljs js/Error)
+       (cut/generate-config invalid-conf auth))))
+
+(deftest should-not-generate-auth
+  (is (thrown?
+       #?(:clj Exception :cljs js/Error)
+       (cut/generate-config conf invalid-auth))))
+
+
+(deftest should-generate-prometheus-remote-write-auth
+  (is (= {:username "user",
+          :password "password"}
+         (get-in
+          (cut/generate-prometheus-config conf auth)
+          [:remote_write 0 :basic_auth]))))
+
+(deftest should-generate-prometheus-external-labels
+  (is (= {:cluster "clustername",
+          :stage "test"}
+         (get-in
+          (cut/generate-prometheus-config conf auth)
+          [:global :external_labels]))))
+
+(deftest should-generate-config
+  (is (str/starts-with?
+       (get-in
+        (cut/generate-config conf auth)
+        [:stringData :prometheus.yaml])
+       "global:\n  scrape_interval:")))

src/test/cljc/dda/c4k_common/monitoring_test.cljc

@@ -2,16 +2,10 @@
   (:require
    #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
       :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
-   [clojure.string :as s]
    [clojure.spec.test.alpha :as st]
-   [dda.c4k-common.monitoring :as cut]
-   [dda.c4k-common.yaml :as yaml]
-   [clojure.string :as str]))
+   [dda.c4k-common.monitoring :as cut]))
 
 (st/instrument `cut/generate)
-(st/instrument `cut/generate-stateful-set)
-(st/instrument `cut/generate-agent-config)
-(st/instrument `cut/generate-config)
 
 (def conf {:cluster-name "clustername"
            :cluster-stage "test"
@@ -21,45 +15,7 @@
            :grafana-cloud-password "password"
            :hetzner-cloud-ro-token "ro-token"})
 
-(def invalid-conf {:cluster-name "clustername"
-                   :cluster-stage "test"
-                   :grafana-clud-url "https://some.url/with/path"})
-
-(def invalid-auth {:grafana-cloud-user "user"
-                   :grafana-clod-password "password"
-                   :hetzner-cloud-ro-token "ro-token"})
-
-(deftest should-not-generate-config
-  (is (thrown?
-       #?(:clj Exception :cljs js/Error)
-       (cut/generate-config invalid-conf auth))))
-
-(deftest should-not-generate-auth
-  (is (thrown?
-       #?(:clj Exception :cljs js/Error)
-       (cut/generate-config conf invalid-auth))))
 
 (deftest should-generate
   (is (= 17
          (count (cut/generate conf auth)))))
-
-(deftest should-generate-prometheus-remote-write-auth
-  (is (= {:username "user",
-          :password "password"}
-         (get-in
-          (cut/generate-prometheus-config conf auth)
-          [:remote_write 0 :basic_auth]))))
-
-(deftest should-generate-prometheus-external-labels
-  (is (= {:cluster "clustername",
-          :stage "test"}
-         (get-in
-          (cut/generate-prometheus-config conf auth)
-          [:global :external_labels]))))
-
-(deftest should-generate-config
-  (is (s/starts-with?
-       (get-in
-        (cut/generate-config conf auth)
-        [:stringData :prometheus.yaml])
-       "global:\n  scrape_interval:")))