Merge branch 'main' into password-rotation

infrastructure/backup/build.py

@@ -6,7 +6,7 @@ from ddadevops import *
 name = "c4k-cloud"
 MODULE = "backup"
 PROJECT_ROOT_PATH = "../.."
-version = "10.4.3-dev"
+version = "10.4.5-dev"
 
 
 @init

infrastructure/nextcloud/build.py

@@ -6,7 +6,7 @@ from ddadevops import *
 name = 'c4k-cloud'
 MODULE = 'not_set'
 PROJECT_ROOT_PATH = '../..'
-version = "10.4.3-dev"
+version = "10.4.5-dev"
 
 @init
 def initialize(project):

package.json

@@ -2,7 +2,7 @@
     "name": "c4k-nextcloud",
     "description": "Generate c4k yaml for a nextcloud deployment.",
     "author": "meissa GmbH",
-    "version": "10.4.3-SNAPSHOT",
+    "version": "10.4.5-SNAPSHOT",
     "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme",
     "repository": "https://www.npmjs.com/package/c4k-nextcloud",
     "license": "APACHE2",

project.clj

@@ -1,4 +1,4 @@
-(defproject org.domaindrivenarchitecture/c4k-nextcloud "10.4.3-SNAPSHOT"
+(defproject org.domaindrivenarchitecture/c4k-nextcloud "10.4.5-SNAPSHOT"
   :description "nextcloud c4k-installation package"
   :url "https://domaindrivenarchitecture.org"
   :license {:name "Apache License, Version 2.0"

src/main/clj/dda/c4k_nextcloud/uberjar.clj

@@ -2,14 +2,13 @@
   (:gen-class)
   (:require
    [dda.c4k-common.uberjar :as uberjar]
-   [dda.c4k-nextcloud.nextcloud :as nextcloud]
    [dda.c4k-nextcloud.core :as core]))
 
 (defn -main [& cmd-args]
   (uberjar/main-cm
    "c4k-nextcloud"
-   nextcloud/config?
+   core/config?
-   nextcloud/auth?
+   core/auth?
    core/config-defaults
    core/config-objects
    core/auth-objects

src/main/cljc/dda/c4k_nextcloud/backup.cljc

@@ -1,6 +1,8 @@
 (ns dda.c4k-nextcloud.backup
  (:require
   [clojure.spec.alpha :as s]
+  #?(:clj [orchestra.core :refer [defn-spec]]
+     :cljs [orchestra.core :refer-macros [defn-spec]])
   [dda.c4k-common.yaml :as yaml]
   [dda.c4k-common.base64 :as b64]
   [dda.c4k-common.common :as cm]
@@ -13,27 +15,36 @@
 (s/def ::restic-new-password cp/bash-env-string?)
 (s/def ::restic-repository cp/bash-env-string?)
 
+(s/def ::config (s/keys :req-un [::restic-repository]))
+
+(s/def ::auth (s/keys :req-un [::restic-password ::aws-access-key-id ::aws-secret-access-key]))
+
+
 #?(:cljs
    (defmethod yaml/load-resource :backup [resource-name]
      (get (inline-resources "backup") resource-name)))
 
-(defn generate-config [my-conf]
+(defn-spec generate-config map?
+  [my-conf ::config]
   (let [{:keys [restic-repository]} my-conf]
     (->
      (yaml/load-as-edn "backup/config.yaml")
      (cm/replace-key-value :restic-repository restic-repository))))
 
-(defn generate-cron []
+(defn-spec generate-cron map?
+  []
    (yaml/from-string (yaml/load-resource "backup/cron.yaml")))
 
-(defn generate-backup-restore-deployment [my-conf]
+(defn-spec generate-backup-restore-deployment map?
+  [my-conf ::config]
   (let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")]
     (if (and (contains? my-conf :local-integration-test) 
              (= true (:local-integration-test my-conf)))
       (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
       backup-restore-yaml)))
 
-(defn generate-secret [my-auth]
+(defn-spec generate-secret map?
+  [my-auth ::auth]
   (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
     (->
      (yaml/load-as-edn "backup/secret.yaml")

src/main/cljc/dda/c4k_nextcloud/core.cljc

@@ -1,5 +1,6 @@
 (ns dda.c4k-nextcloud.core
   (:require
+   [clojure.spec.alpha :as s]
    #?(:clj [orchestra.core :refer [defn-spec]]
       :cljs [orchestra.core :refer-macros [defn-spec]])
    [dda.c4k-common.common :as cm]
@@ -16,8 +17,15 @@
                       :pvc-storage-class-name "hcloud-volumes-encrypted"
                       :pv-storage-size-gb 200})
 
+(def config? (s/merge ::nextcloud/config
+                      ::backup/config))
+
+(def auth? (s/merge ::nextcloud/auth
+                    ::backup/auth))
+
+
 (defn-spec config-objects cp/map-or-seq?
-  [config nextcloud/config?]
+  [config config?]
   (let [resolved-config (merge config-defaults config)]
     (map yaml/to-string
          (filter
@@ -40,8 +48,8 @@
              (mon/generate-config)))))))
 
 (defn-spec auth-objects cp/map-or-seq?
-  [config nextcloud/config?
+  [config config?
-   auth nextcloud/auth?]
+   auth auth?]
   (let [resolved-config (merge config-defaults config)]
     (map yaml/to-string
          (filter

src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc

@@ -14,23 +14,20 @@
 
 (s/def ::fqdn cp/fqdn-string?)
 (s/def ::issuer cp/letsencrypt-issuer?)
-(s/def ::restic-repository string?)
 (s/def ::nextcloud-admin-user cp/bash-env-string?)
 (s/def ::nextcloud-admin-password cp/bash-env-string?)
 (s/def ::pvc-storage-class-name cp/pvc-storage-class-name?)
 (s/def ::pv-storage-size-gb pos?)
 
-(def config? (s/keys :req-un [::fqdn]
+(s/def ::config (s/keys :req-un [::fqdn]
                      :opt-un [::issuer
-                              ::restic-repository
                               ::pv-storage-size-gb
                               ::pvc-storage-class-name
                               ::mon/mon-cfg]))
 
-(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
+(s/def ::auth (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
                             ::nextcloud-admin-user ::nextcloud-admin-password
-                            ::aws-access-key-id ::aws-secret-access-key
+                            ::aws-access-key-id ::aws-secret-access-key]
-                            ::restic-password]
                    :opt-un [::mon/mon-auth]))
 
 #?(:cljs
@@ -38,13 +35,13 @@
      (get (inline-resources "nextcloud") resource-name)))
 
 (defn-spec generate-deployment cp/map-or-seq? 
-  [config config?]
+  [config ::config]
   (let [{:keys [fqdn]} config]
     (-> (yaml/load-as-edn "nextcloud/deployment.yaml")
         (cm/replace-all-matching "fqdn" fqdn))))
 
 (defn-spec generate-ingress-and-cert cp/map-or-seq?
-  [config config?]
+  [config ::config]
   (ing/generate-ingress-and-cert
    (merge
     {:service-name "cloud-service"
@@ -64,7 +61,7 @@
   (yaml/load-as-edn "nextcloud/service.yaml"))
 
 (defn-spec generate-secret cp/map-or-seq? 
-  [auth auth?]
+  [auth ::auth]
   (let [{:keys [nextcloud-admin-user nextcloud-admin-password]} auth]
     (->
      (yaml/load-as-edn "nextcloud/secret.yaml")

src/main/cljs/dda/c4k_nextcloud/browser.cljs

@@ -2,28 +2,27 @@
   (:require
    [clojure.tools.reader.edn :as edn]
    [dda.c4k-common.common :as cm]
-   [dda.c4k-common.monitoring :as mon]
    [dda.c4k-nextcloud.core :as core]
-   [dda.c4k-nextcloud.nextcloud :as nextcloud]
+   [dda.c4k-common.browser :as br]))
-   [dda.c4k-common.browser :as br]
-   [dda.c4k-common.postgres :as pgc]))
 
 (defn generate-content []
   (cm/concat-vec
    [(assoc
      (br/generate-needs-validation) :content
      (cm/concat-vec
-      (br/generate-group "domain"
+      (br/generate-group "config"
-                         (cm/concat-vec (br/generate-input-field "fqdn" "Your fqdn:" "nextcloud-neu.prod.meissa-gmbh.de")
+                         (br/generate-text-area "config" "Your config.edn:" "{:fqdn \"cloud.your.domain\"
-                                        (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")
+          :issuer \"staging\"
-                                        (br/generate-input-field "pv-storage-size-gb" "(Optional) Your nextcloud storage size in GB" "8")
+          :pv-storage-size-gb \"400\"
-                                        (br/generate-input-field "pvc-storage-class-name" "(Optional) Your storage class type (manual / local-path):" "local-path")
+          :pvc-storage-class-name \"local-path\"                                                                                  
-                                        (br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path:" "/var/postgres")
+          :postgres-data-volume-path \"/var/postgres\"
-                                        (br/generate-input-field "restic-repository" "(Optional) Your restic-repository:" "restic-repository")
+          :restic-repository \"s3://yourbucket/your-repo\"
-                                        (br/generate-input-field "mon-cluster-name" "(Optional) monitoring cluster name:" "keycloak")
+          :restic-password \"restic-password\"}
-                                        (br/generate-input-field "mon-cluster-stage" "(Optional) monitoring cluster stage:" "test")
+          :mon-cfg {:cluster-name \"cloud\"
-                                        (br/generate-input-field "mon-cloud-url" "(Optional) grafana cloud url:" "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push")))
+                    :cluster-stage \"test\"
-      (br/generate-group "credentials"
+                    :cloud-url \"https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push\"}"
+                                                "5"))
+      (br/generate-group "auth"
                          (br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\"
          :postgres-db-password \"nextcloud-db-password\"
          :nextcloud-admin-password \"nextcloud-admin-password\"
@@ -45,41 +44,9 @@
    :content
    (generate-content)})
 
-(defn config-from-document []
-  (let [pv-storage-size-gb (br/get-content-from-element "pv-storage-size-gb" :optional true)
-        pvc-storage-class-name (br/get-content-from-element "pvc-storage-class-name" :optional true)
-        postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true)
-        restic-repository (br/get-content-from-element "restic-repository" :optional true)
-        issuer (br/get-content-from-element "issuer" :optional true)
-        mon-cluster-name (br/get-content-from-element "mon-cluster-name" :optional true)
-        mon-cluster-stage (br/get-content-from-element "mon-cluster-stage" :optional true)
-        mon-cloud-url (br/get-content-from-element "mon-cloud-url" :optional true)]
-    (merge
-     {:fqdn (br/get-content-from-element "fqdn")}
-     (when (and (some? pv-storage-size-gb) (some? pvc-storage-class-name))
-       {:pv-storage-size-gb pv-storage-size-gb :pvc-storage-class-name pvc-storage-class-name})
-     (when (some? postgres-data-volume-path)
-       {:postgres-data-volume-path postgres-data-volume-path})
-     (when (some? restic-repository)
-       {:restic-repository restic-repository})
-     (when (some? issuer)
-       {:issuer issuer})
-     (when (some? mon-cluster-name)
-       {:mon-cfg {:cluster-name mon-cluster-name
-                  :cluster-stage (keyword mon-cluster-stage)
-                  :grafana-cloud-url mon-cloud-url}}))))
-
 (defn validate-all! []
-  (br/validate! "fqdn" ::nextcloud/fqdn)
+  (br/validate! "config" core/config? :deserializer edn/read-string)
-  (br/validate! "pv-storage-size-gb" ::nextcloud/pv-storage-size-gb :optional true)
+  (br/validate! "auth" core/auth? :deserializer edn/read-string)
-  (br/validate! "pvc-storage-class-name" ::nextcloud/pvc-storage-class-name :optional true)
-  (br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true)
-  (br/validate! "restic-repository" ::nextcloud/restic-repository :optional true)
-  (br/validate! "issuer" ::nextcloud/issuer :optional true)
-  (br/validate! "mon-cluster-name" ::mon/cluster-name :optional true)
-  (br/validate! "mon-cluster-stage" ::mon/cluster-stage :optional true)
-  (br/validate! "mon-cloud-url" ::mon/grafana-cloud-url :optional true)
-  (br/validate! "auth" nextcloud/auth? :deserializer edn/read-string)
   (br/set-validated!))
 
 (defn add-validate-listener [name]
@@ -92,19 +59,14 @@
       (.getElementById "generate-button")
       (.addEventListener "click"
                          #(do (validate-all!)
-                              (-> (cm/generate-common
+                              (-> (cm/generate-cm
-                                   (config-from-document)
+                                   (br/get-content-from-element "config" :deserializer edn/read-string)
                                    (br/get-content-from-element "auth" :deserializer edn/read-string)
-                                   {}
+                                   core/config-defaults
-                                   core/k8s-objects)
+                                   core/config-objects
+                                   core/auth-objects
+                                   false
+                                   false)
                                   (br/set-output!)))))
-  (add-validate-listener "fqdn")
+  (add-validate-listener "config")
-  (add-validate-listener "pv-storage-size-gb")
-  (add-validate-listener "pvc-storage-class-name")
-  (add-validate-listener "postgres-data-volume-path")
-  (add-validate-listener "restic-repository")
-  (add-validate-listener "issuer")
-  (add-validate-listener "mon-cluster-name")
-  (add-validate-listener "mon-cluster-stage")
-  (add-validate-listener "mon-cloud-url")
   (add-validate-listener "auth"))

src/test/cljc/dda/c4k_nextcloud/backup_test.cljc

@@ -2,8 +2,12 @@
   (:require
    #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
       :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
+   [clojure.spec.test.alpha :as st]
    [dda.c4k-nextcloud.backup :as cut]))
 
+(st/instrument `cut/generate-secret)
+(st/instrument `cut/generate-config)
+(st/instrument `cut/generate-cron)
 
 (deftest should-generate-secret
   (is (= {:apiVersion "v1"

src/test/cljc/dda/c4k_nextcloud/core_test.cljc

@@ -0,0 +1,19 @@
+(ns dda.c4k-nextcloud.core-test
+  (:require
+   #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
+      :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
+   [clojure.spec.alpha :as s]
+   [dda.c4k-common.yaml :as yaml]
+   [dda.c4k-nextcloud.core :as cut]
+   #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
+
+
+#?(:cljs
+   (defmethod yaml/load-resource :nextcloud-test [resource-name]
+     (get (inline-resources "nextcloud-test") resource-name)))
+
+(deftest validate-valid-resources
+  (is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml")))
+  (is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml")))
+  (is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml"))))
+  (is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml")))))

src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc

@@ -2,24 +2,20 @@
   (:require
    #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
       :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
-   [clojure.spec.alpha :as s]
    [clojure.spec.test.alpha :as st]
    [dda.c4k-common.yaml :as yaml]
    [dda.c4k-nextcloud.nextcloud :as cut]
    #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
 
-(st/instrument)
+(st/instrument `cut/generate-secret)
+(st/instrument `cut/generate-ingress-and-cert)
+(st/instrument `cut/generate-pvc)
+(st/instrument `cut/generate-deployment)
 
 #?(:cljs
    (defmethod yaml/load-resource :nextcloud-test [resource-name]
      (get (inline-resources "nextcloud-test") resource-name)))
 
-(deftest validate-valid-resources
-  (is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml")))
-  (is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml")))
-  (is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml"))))
-  (is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml")))))
-
 (deftest should-generate-secret
   (is (= {:apiVersion "v1"
           :kind "Secret"
@@ -95,7 +91,7 @@
            {:metadata {:labels {:app "cloud-app", :app.kubernetes.io/name "cloud-pod", :app.kubernetes.io/application "cloud", :redeploy "v3"}}
             :spec
             {:containers
-             [{:image "domaindrivenarchitecture/c4k-cloud:10.4.0"
+             [{:image "domaindrivenarchitecture/c4k-cloud:10.4.2"
                :name "cloud-app"
                :imagePullPolicy "IfNotPresent"
                :ports [{:containerPort 80}]