From 74505dbab7666911849653efe35971b8dfd32e6f Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Feb 2023 13:28:46 +0100 Subject: [PATCH 01/26] Bump c4k-common to newest version --- project.clj | 2 +- shadow-cljs.edn | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project.clj b/project.clj index 4572874..797ba25 100644 --- a/project.clj +++ b/project.clj @@ -5,7 +5,7 @@ :url "https://www.apache.org/licenses/LICENSE-2.0.html"} :dependencies [[org.clojure/clojure "1.11.1"] [org.clojure/tools.reader "1.3.6"] - [org.domaindrivenarchitecture/c4k-common-clj "2.0.3"]] + [org.domaindrivenarchitecture/c4k-common-clj "6.0.1"]] :target-path "target/%s/" :source-paths ["src/main/cljc" "src/main/clj"] diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 7578c71..fdc0345 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -4,7 +4,7 @@ "src/test/cljc" "src/test/cljs" "src/test/resources"] - :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "2.0.3"] + :dependencies [[org.domaindrivenarchitecture/c4k-common-cljs "6.0.1"] [hickory "0.7.1"]] :builds {:frontend {:target :browser :modules {:main {:init-fn dda.c4k-nextcloud.browser/init}} From 107b8c387240122eda1bcaed159afbe9c736dde5 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Feb 2023 13:29:10 +0100 Subject: [PATCH 02/26] Use common load-as-edn To avoid common code duplication --- src/main/cljc/dda/c4k_nextcloud/backup.cljc | 6 +++--- src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/cljc/dda/c4k_nextcloud/backup.cljc b/src/main/cljc/dda/c4k_nextcloud/backup.cljc index 4a499c9..1ef1694 100644 --- a/src/main/cljc/dda/c4k_nextcloud/backup.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/backup.cljc @@ -23,14 +23,14 @@ (defn generate-config [my-conf] (let [{:keys [restic-repository]} my-conf] (-> - (yaml/from-string (yaml/load-resource "backup/config.yaml")) + (yaml/load-as-edn "backup/config.yaml") (cm/replace-key-value :restic-repository restic-repository)))) (defn generate-cron [] (yaml/from-string (yaml/load-resource "backup/cron.yaml"))) (defn generate-backup-restore-deployment [my-conf] - (let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))] + (let [backup-restore-yaml (yaml/load-as-edn "backup/backup-restore-deployment.yaml")] (if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf))) (cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt") backup-restore-yaml))) @@ -38,7 +38,7 @@ (defn generate-secret [my-auth] (let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth] (-> - (yaml/from-string (yaml/load-resource "backup/secret.yaml")) + (yaml/load-as-edn "backup/secret.yaml") (cm/replace-key-value :aws-access-key-id (b64/encode aws-access-key-id)) (cm/replace-key-value :aws-secret-access-key (b64/encode aws-secret-access-key)) (cm/replace-key-value :restic-password (b64/encode restic-password))))) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index 70bed75..95b73d0 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -36,14 +36,14 @@ (let [{:keys [fqdn issuer]} config letsencrypt-issuer issuer] (-> - (yaml/from-string (yaml/load-resource "nextcloud/certificate.yaml")) + (yaml/load-as-edn "nextcloud/certificate.yaml") (assoc-in [:spec :commonName] fqdn) (assoc-in [:spec :dnsNames] [fqdn]) (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) (defn generate-deployment [config] (let [{:keys [fqdn]} config] - (-> (yaml/from-string (yaml/load-resource "nextcloud/deployment.yaml")) + (-> (yaml/load-as-edn "nextcloud/deployment.yaml") (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) (defn generate-ingress [config] @@ -51,7 +51,7 @@ :or {issuer "staging"}} config letsencrypt-issuer issuer] (-> - (yaml/from-string (yaml/load-resource "nextcloud/ingress.yaml")) + (yaml/load-as-edn "nextcloud/ingress.yaml") (assoc-in [:metadata :annotations :cert-manager.io/cluster-issuer] letsencrypt-issuer) (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) @@ -59,16 +59,16 @@ [config (s/keys :req-un [::pv-storage-size-gb ::pvc-storage-class-name])] (let [{:keys [pv-storage-size-gb pvc-storage-class-name]} config] (-> - (yaml/from-string (yaml/load-resource "nextcloud/pvc.yaml")) + (yaml/load-as-edn "nextcloud/pvc.yaml") (assoc-in [:spec :resources :requests :storage] (str pv-storage-size-gb "Gi")) (assoc-in [:spec :storageClassName] (name pvc-storage-class-name))))) (defn generate-service [] - (yaml/from-string (yaml/load-resource "nextcloud/service.yaml"))) + (yaml/load-as-edn "nextcloud/service.yaml")) (defn generate-secret [config] (let [{:keys [nextcloud-admin-user nextcloud-admin-password]} config] (-> - (yaml/from-string (yaml/load-resource "nextcloud/secret.yaml")) + (yaml/load-as-edn "nextcloud/secret.yaml") (cm/replace-key-value :nextcloud-admin-user (b64/encode nextcloud-admin-user)) (cm/replace-key-value :nextcloud-admin-password (b64/encode nextcloud-admin-password))))) From 88a077e776bb7fb82f39f4c7d8de91d3adfba8c4 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Feb 2023 13:40:56 +0100 Subject: [PATCH 03/26] Use standardized uberjar --- src/main/clj/dda/c4k_nextcloud/uberjar.clj | 60 ++++--------------- src/main/cljc/dda/c4k_nextcloud/core.cljc | 30 +++------- .../cljc/dda/c4k_nextcloud/nextcloud.cljc | 12 ++++ 3 files changed, 30 insertions(+), 72 deletions(-) diff --git a/src/main/clj/dda/c4k_nextcloud/uberjar.clj b/src/main/clj/dda/c4k_nextcloud/uberjar.clj index 865b92c..73c4c40 100644 --- a/src/main/clj/dda/c4k_nextcloud/uberjar.clj +++ b/src/main/clj/dda/c4k_nextcloud/uberjar.clj @@ -1,55 +1,15 @@ (ns dda.c4k-nextcloud.uberjar (:gen-class) (:require - [clojure.spec.alpha :as s] - [clojure.string :as cs] - [clojure.tools.reader.edn :as edn] - [expound.alpha :as expound] - [dda.c4k-common.yaml :as yaml] - [dda.c4k-nextcloud.core :as core] - [dda.c4k-nextcloud.nextcloud :as nextcloud])) - -(def usage - "usage: - - c4k-nextcloud {your configuraton file} {your authorization file}") - -(s/def ::options (s/* #{"-h"})) -(s/def ::filename (s/and string? - #(not (cs/starts-with? % "-")))) -(s/def ::cmd-args (s/cat :options ::options - :args (s/? - (s/cat :config ::filename - :auth ::filename)))) -(defn invalid-args-msg - [spec args] - (s/explain spec args) - (println (str "Bad commandline arguments\n" usage))) + [dda.c4k-common.uberjar :as uberjar] + [dda.c4k-nextcloud.jitsi :as jitsi] + [dda.c4k-nextcloud.core :as core])) (defn -main [& cmd-args] - (let [parsed-args-cmd (s/conform ::cmd-args cmd-args)] - (if (= ::s/invalid parsed-args-cmd) - (invalid-args-msg ::cmd-args cmd-args) - (let [{:keys [options args]} parsed-args-cmd - {:keys [config auth]} args] - (cond - (some #(= "-h" %) options) - (println usage) - :default - (let [config-str (slurp config) - auth-str (slurp auth) - config-parse-fn (if (yaml/is-yaml? config) yaml/from-string edn/read-string) - auth-parse-fn (if (yaml/is-yaml? auth) yaml/from-string edn/read-string) - parsed-config (config-parse-fn config-str) - parsed-auth (auth-parse-fn auth-str) - config-valid? (s/valid? ::core/config parsed-config) - auth-valid? (s/valid? ::core/auth parsed-auth)] - (if (and config-valid? auth-valid?) - (println (core/generate parsed-config parsed-auth)) - (do - (when (not config-valid?) - (println - (expound/expound-str ::core/config parsed-config {:print-specs? false}))) - (when (not auth-valid?) - (println - (expound/expound-str ::core/auth parsed-auth {:print-specs? false}))))))))))) + (uberjar/main-common + "c4k-nextcloud" + jitsi/config? + jitsi/auth? + core/config-defaults + core/generate + cmd-args)) diff --git a/src/main/cljc/dda/c4k_nextcloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc index 8cd5205..a04bd42 100644 --- a/src/main/cljc/dda/c4k_nextcloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -1,9 +1,9 @@ (ns dda.c4k-nextcloud.core (:require - [clojure.string :as cs] [clojure.spec.alpha :as s] #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) + [dda.c4k-common.common :as cm] [dda.c4k-common.yaml :as yaml] [dda.c4k-common.postgres :as postgres] [dda.c4k-nextcloud.nextcloud :as nextcloud] @@ -13,22 +13,8 @@ (def config-defaults {:issuer "staging"}) -(def config? (s/keys :req-un [::nextcloud/fqdn] - :opt-un [::nextcloud/issuer - ::nextcloud/restic-repository - ::nextcloud/pv-storage-size-gb - ::nextcloud/pvc-storage-class-name])) - -(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password - ::nextcloud/nextcloud-admin-user ::nextcloud/nextcloud-admin-password - ::aws-access-key-id ::aws-secret-access-key - ::restic-password])) - -(s/def ::config config?) -(s/def ::auth auth?) - (defn-spec k8s-objects any? - [config (s/merge config? auth?)] + [config (s/merge nextcloud/config? nextcloud/auth?)] (let [nextcloud-default-storage-config {:pvc-storage-class-name default-storage-class :pv-storage-size-gb 200}] (into @@ -52,9 +38,9 @@ (yaml/to-string (backup/generate-backup-restore-deployment config))]))))) (defn-spec generate any? - [my-config config? - my-auth auth?] - (let [resulting-config (merge config-defaults my-config my-auth)] - (cs/join - "\n---\n" - (k8s-objects resulting-config)))) + [my-config nextcloud/config? + my-auth nextcloud/auth?] + (cm/concat-vec + (map yaml/to-string + (filter #(not (nil? %)) + (merge config-defaults my-config my-auth))))) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index 95b73d0..9566430 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -7,6 +7,7 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] + [dda.c4k-common.postgres :as postgres] [dda.c4k-common.common :as cm])) (s/def ::fqdn cp/fqdn-string?) @@ -21,6 +22,17 @@ ::pvc-storage-class-name] :opt-un [::restic-repository])) +(def config? (s/keys :req-un [::fqdn] + :opt-un [::issuer + ::restic-repository + ::pv-storage-size-gb + ::pvc-storage-class-name])) + +(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password + ::nextcloud-admin-user ::nextcloud-admin-password + ::aws-access-key-id ::aws-secret-access-key + ::restic-password])) + #?(:cljs (defmethod yaml/load-resource :nextcloud [resource-name] (case resource-name From 91d88d2a689739f9820865857769e655aa714294 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 10:14:34 +0100 Subject: [PATCH 04/26] Use nextcloud instead of jitsi --- src/main/clj/dda/c4k_nextcloud/uberjar.clj | 6 +- src/main/cljc/dda/c4k_nextcloud/core.cljc | 37 +++++----- .../cljc/dda/c4k_nextcloud/core_test.cljc | 67 +++++-------------- 3 files changed, 38 insertions(+), 72 deletions(-) diff --git a/src/main/clj/dda/c4k_nextcloud/uberjar.clj b/src/main/clj/dda/c4k_nextcloud/uberjar.clj index 73c4c40..4983f70 100644 --- a/src/main/clj/dda/c4k_nextcloud/uberjar.clj +++ b/src/main/clj/dda/c4k_nextcloud/uberjar.clj @@ -2,14 +2,14 @@ (:gen-class) (:require [dda.c4k-common.uberjar :as uberjar] - [dda.c4k-nextcloud.jitsi :as jitsi] + [dda.c4k-nextcloud.nextcloud :as nextcloud] [dda.c4k-nextcloud.core :as core])) (defn -main [& cmd-args] (uberjar/main-common "c4k-nextcloud" - jitsi/config? - jitsi/auth? + nextcloud/config? + nextcloud/auth? core/config-defaults core/generate cmd-args)) diff --git a/src/main/cljc/dda/c4k_nextcloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc index a04bd42..27045f2 100644 --- a/src/main/cljc/dda/c4k_nextcloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -17,25 +17,24 @@ [config (s/merge nextcloud/config? nextcloud/auth?)] (let [nextcloud-default-storage-config {:pvc-storage-class-name default-storage-class :pv-storage-size-gb 200}] - (into - [] - (concat [(yaml/to-string (postgres/generate-config {:postgres-size :8gb})) - (yaml/to-string (postgres/generate-secret config)) - (yaml/to-string (postgres/generate-pvc {:pv-storage-size-gb 50 - :pvc-storage-class-name default-storage-class})) - (yaml/to-string (postgres/generate-deployment)) - (yaml/to-string (postgres/generate-service)) - (yaml/to-string (nextcloud/generate-secret config)) - (yaml/to-string (nextcloud/generate-pvc (merge nextcloud-default-storage-config config))) - (yaml/to-string (nextcloud/generate-deployment config)) - (yaml/to-string (nextcloud/generate-service)) - (yaml/to-string (nextcloud/generate-certificate config)) - (yaml/to-string (nextcloud/generate-ingress config))] - (when (contains? config :restic-repository) - [(yaml/to-string (backup/generate-config config)) - (yaml/to-string (backup/generate-secret config)) - (yaml/to-string (backup/generate-cron)) - (yaml/to-string (backup/generate-backup-restore-deployment config))]))))) + (map yaml/to-string + [(postgres/generate-config {:postgres-size :8gb}) + (postgres/generate-secret config) + (postgres/generate-pvc {:pv-storage-size-gb 50 + :pvc-storage-class-name default-storage-class}) + (postgres/generate-deployment) + (postgres/generate-service) + (nextcloud/generate-secret config) + (nextcloud/generate-pvc (merge nextcloud-default-storage-config config)) + (nextcloud/generate-deployment config) + (nextcloud/generate-service) + (nextcloud/generate-certificate config) + (nextcloud/generate-ingress config)] + (when (:contains? config :restic-repository) + [(backup/generate-config config) + (backup/generate-secret config) + (backup/generate-cron) + (backup/generate-backup-restore-deployment config)])))) (defn-spec generate any? [my-config nextcloud/config? diff --git a/src/test/cljc/dda/c4k_nextcloud/core_test.cljc b/src/test/cljc/dda/c4k_nextcloud/core_test.cljc index d4fbda6..0c88290 100644 --- a/src/test/cljc/dda/c4k_nextcloud/core_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/core_test.cljc @@ -2,54 +2,21 @@ (:require #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - [clojure.string :as st] - [dda.c4k-nextcloud.core :as cut] - )) + #?(:cljs [shadow-resource :as rc]) + [clojure.spec.alpha :as s] + [dda.c4k-common.yaml :as yaml] + [dda.c4k-nextcloud.core :as cut])) -(deftest should-k8s-objects - (is (= 15 - (count (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "nextcloud" - :postgres-db-password "nextcloud-db-password" - :nextcloud-admin-user "cloudadmin" - :nextcloud-admin-password "cloudpassword" - :issuer "prod" - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw" - :restic-repository "restic-repository"})))) - (is (= 11 - (count (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "nextcloud" - :postgres-db-password "nextcloud-db-password" - :nextcloud-admin-user "cloudadmin" - :nextcloud-admin-password "cloudpassword" - :issuer "prod" - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw"})))) - (is (st/includes? - (get-in (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "nextcloud" - :postgres-db-password "nextcloud-db-password" - :nextcloud-admin-user "cloudadmin" - :nextcloud-admin-password "cloudpassword" - :issuer "prod" - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw"}) - [0]) - "max_connections = 700")) - (is (st/includes? - (get-in (cut/k8s-objects {:fqdn "nextcloud-neu.prod.meissa-gmbh.de" - :postgres-db-user "nextcloud" - :postgres-db-password "nextcloud-db-password" - :nextcloud-admin-user "cloudadmin" - :nextcloud-admin-password "cloudpassword" - :issuer "prod" - :aws-access-key-id "aws-id" - :aws-secret-access-key "aws-secret" - :restic-password "restic-pw"}) - [6]) - "storageClassName: local-path")) -) +#?(:cljs + (defmethod yaml/load-resource :nextcloud-test [resource-name] + (case resource-name + "nextcloud-test/valid-auth.yaml" (rc/inline "nextcloud-test/valid-auth.yaml") + "nextcloud-test/valid-config.yaml" (rc/inline "nextcloud-test/valid-config.yaml") + "nextcloud-test/invalid-auth.yaml" (rc/inline "nextcloud-test/invalid-auth.yaml") + "nextcloud-test/invalid-config.yaml" (rc/inline "nextcloud-test/invalid-config.yaml")))) + +(deftest validate-valid-resources + (is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml"))) + (is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml"))) + (is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml")))) + (is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml"))))) From 19682eec36180da0cf335517b11f8de4534ec935 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 10:22:54 +0100 Subject: [PATCH 05/26] Add and test valid/invalid config/auth files --- .../cljc/dda/c4k_nextcloud/core_test.cljc | 22 ------------------- .../dda/c4k_nextcloud/nextcloud_test.cljc | 20 +++++++++++++++++ .../nextcloud-test/invalid-auth.yaml | 7 ++++++ .../nextcloud-test/invalid-config.yaml | 5 +++++ .../resources/nextcloud-test/valid-auth.edn | 0 .../resources/nextcloud-test/valid-auth.yaml | 0 .../resources/nextcloud-test/valid-config.edn | 0 .../nextcloud-test/valid-config.yaml | 0 8 files changed, 32 insertions(+), 22 deletions(-) delete mode 100644 src/test/cljc/dda/c4k_nextcloud/core_test.cljc create mode 100644 src/test/resources/nextcloud-test/invalid-auth.yaml create mode 100644 src/test/resources/nextcloud-test/invalid-config.yaml rename valid-auth.edn => src/test/resources/nextcloud-test/valid-auth.edn (100%) rename valid-auth.yaml => src/test/resources/nextcloud-test/valid-auth.yaml (100%) rename valid-config.edn => src/test/resources/nextcloud-test/valid-config.edn (100%) rename valid-config.yaml => src/test/resources/nextcloud-test/valid-config.yaml (100%) diff --git a/src/test/cljc/dda/c4k_nextcloud/core_test.cljc b/src/test/cljc/dda/c4k_nextcloud/core_test.cljc deleted file mode 100644 index 0c88290..0000000 --- a/src/test/cljc/dda/c4k_nextcloud/core_test.cljc +++ /dev/null @@ -1,22 +0,0 @@ -(ns dda.c4k-nextcloud.core-test - (:require - #?(:clj [clojure.test :refer [deftest is are testing run-tests]] - :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - #?(:cljs [shadow-resource :as rc]) - [clojure.spec.alpha :as s] - [dda.c4k-common.yaml :as yaml] - [dda.c4k-nextcloud.core :as cut])) - -#?(:cljs - (defmethod yaml/load-resource :nextcloud-test [resource-name] - (case resource-name - "nextcloud-test/valid-auth.yaml" (rc/inline "nextcloud-test/valid-auth.yaml") - "nextcloud-test/valid-config.yaml" (rc/inline "nextcloud-test/valid-config.yaml") - "nextcloud-test/invalid-auth.yaml" (rc/inline "nextcloud-test/invalid-auth.yaml") - "nextcloud-test/invalid-config.yaml" (rc/inline "nextcloud-test/invalid-config.yaml")))) - -(deftest validate-valid-resources - (is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml"))) - (is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml"))) - (is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml")))) - (is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml"))))) diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 88e3376..26ee033 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -2,8 +2,28 @@ (:require #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) + #?(:cljs [shadow-resource :as rc]) + [clojure.spec.alpha :as s] + [clojure.spec.test.alpha :as st] + [dda.c4k-common.yaml :as yaml] [dda.c4k-nextcloud.nextcloud :as cut])) +(st/instrument) + +#?(:cljs + (defmethod yaml/load-resource :nextcloud-test [resource-name] + (case resource-name + "nextcloud-test/valid-auth.yaml" (rc/inline "nextcloud-test/valid-auth.yaml") + "nextcloud-test/valid-config.yaml" (rc/inline "nextcloud-test/valid-config.yaml") + "nextcloud-test/invalid-auth.yaml" (rc/inline "nextcloud-test/invalid-auth.yaml") + "nextcloud-test/invalid-config.yaml" (rc/inline "nextcloud-test/invalid-config.yaml")))) + +(deftest validate-valid-resources + (is (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/valid-config.yaml"))) + (is (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/valid-auth.yaml"))) + (is (not (s/valid? cut/config? (yaml/load-as-edn "nextcloud-test/invalid-config.yaml")))) + (is (not (s/valid? cut/auth? (yaml/load-as-edn "nextcloud-test/invalid-auth.yaml"))))) + (deftest should-generate-secret (is (= {:apiVersion "v1" :kind "Secret" diff --git a/src/test/resources/nextcloud-test/invalid-auth.yaml b/src/test/resources/nextcloud-test/invalid-auth.yaml new file mode 100644 index 0000000..11b9684 --- /dev/null +++ b/src/test/resources/nextcloud-test/invalid-auth.yaml @@ -0,0 +1,7 @@ +postgres-db-user: "nextcloud" +postgres-db-password: "nextcloud-db-password" +nextcloud-admin-user: "cloudadmin" +nextcloud-admin-password: "cloudpassword" +aws-accss-key-id: "aws-id" +aws-secret-access-key: "aws-secret" +restic-password: "restic-password" \ No newline at end of file diff --git a/src/test/resources/nextcloud-test/invalid-config.yaml b/src/test/resources/nextcloud-test/invalid-config.yaml new file mode 100644 index 0000000..86120a4 --- /dev/null +++ b/src/test/resources/nextcloud-test/invalid-config.yaml @@ -0,0 +1,5 @@ +fqdns: "cloud.test.meissa-gmbh.de" +issuer: "none" +nextcloud-data-volume-path: "/var/cloud" +postgres-data-volume-path: "/var/postgres" +restic-repository: "s3:s3.amazonaws.com/your-bucket/your-folder" \ No newline at end of file diff --git a/valid-auth.edn b/src/test/resources/nextcloud-test/valid-auth.edn similarity index 100% rename from valid-auth.edn rename to src/test/resources/nextcloud-test/valid-auth.edn diff --git a/valid-auth.yaml b/src/test/resources/nextcloud-test/valid-auth.yaml similarity index 100% rename from valid-auth.yaml rename to src/test/resources/nextcloud-test/valid-auth.yaml diff --git a/valid-config.edn b/src/test/resources/nextcloud-test/valid-config.edn similarity index 100% rename from valid-config.edn rename to src/test/resources/nextcloud-test/valid-config.edn diff --git a/valid-config.yaml b/src/test/resources/nextcloud-test/valid-config.yaml similarity index 100% rename from valid-config.yaml rename to src/test/resources/nextcloud-test/valid-config.yaml From 16dd0c58283b0863378d63302439d718440a37aa Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 10:32:04 +0100 Subject: [PATCH 06/26] Spec and instrument generate functions --- .../cljc/dda/c4k_nextcloud/nextcloud.cljc | 12 ++++++++---- .../dda/c4k_nextcloud/nextcloud_test.cljc | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index 9566430..40d11ce 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -44,7 +44,8 @@ "nextcloud/secret.yaml" (rc/inline "nextcloud/secret.yaml") (throw (js/Error. "Undefined Resource!"))))) -(defn generate-certificate [config] +(defn-spec generate-certificate cp/map-or-seq? + [config config?] (let [{:keys [fqdn issuer]} config letsencrypt-issuer issuer] (-> @@ -53,12 +54,14 @@ (assoc-in [:spec :dnsNames] [fqdn]) (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) -(defn generate-deployment [config] +(defn-spec generate-deployment cp/map-or-seq? + [config config?] (let [{:keys [fqdn]} config] (-> (yaml/load-as-edn "nextcloud/deployment.yaml") (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) -(defn generate-ingress [config] +(defn-spec generate-ingress cp/map-or-seq? + [config config?] (let [{:keys [fqdn issuer] :or {issuer "staging"}} config letsencrypt-issuer issuer] @@ -78,7 +81,8 @@ (defn generate-service [] (yaml/load-as-edn "nextcloud/service.yaml")) -(defn generate-secret [config] +(defn-spec generate-secret cp/map-or-seq? + [config config?] (let [{:keys [nextcloud-admin-user nextcloud-admin-password]} config] (-> (yaml/load-as-edn "nextcloud/secret.yaml") diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 26ee033..2cbd51c 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -32,7 +32,8 @@ :data {:nextcloud-admin-user "Y2xvdWRhZG1pbg==" :nextcloud-admin-password "Y2xvdWRwYXNzd29yZA=="}} - (cut/generate-secret {:nextcloud-admin-user "cloudadmin" + (cut/generate-secret {:fqdn "somefqdn.de" + :nextcloud-admin-user "cloudadmin" :nextcloud-admin-password "cloudpassword"})))) (deftest should-generate-certificate @@ -43,11 +44,11 @@ {:secretName "cloud-cert" :duration "2160h" :renewBefore "360h", - :commonName "xx", - :dnsNames ["xx"] + :commonName "somefqdn.de", + :dnsNames ["somefqdn.de"] :issuerRef {:name "prod", :kind "ClusterIssuer"}}} - (cut/generate-certificate {:fqdn "xx" :issuer "prod"})))) + (cut/generate-certificate {:fqdn "somefqdn.de" :issuer "prod"})))) (deftest should-generate-ingress (is (= {:apiVersion "networking.k8s.io/v1" @@ -64,9 +65,9 @@ :ingress.kubernetes.io/proxy-read-timeout "300"} :namespace "default"} :spec - {:tls [{:hosts ["xx"], :secretName "cloud-cert"}] + {:tls [{:hosts ["somefqdn.de"], :secretName "cloud-cert"}] :rules - [{:host "xx" + [{:host "somefqdn.de" :http {:paths [{:path "/" @@ -74,7 +75,7 @@ :backend {:service {:name "cloud-service", :port {:number 80}}}}]}}]}} - (cut/generate-ingress {:fqdn "xx"})))) + (cut/generate-ingress {:fqdn "somefqdn.de"})))) (deftest should-generate-pvc (is (= {:apiVersion "v1" @@ -113,11 +114,11 @@ [{:name "NEXTCLOUD_ADMIN_USER", :valueFrom {:secretKeyRef {:name "cloud-secret", :key "nextcloud-admin-user"}}} {:name "NEXTCLOUD_ADMIN_PASSWORD" :valueFrom {:secretKeyRef {:name "cloud-secret", :key "nextcloud-admin-password"}}} - {:name "NEXTCLOUD_TRUSTED_DOMAINS", :value "xx"} + {:name "NEXTCLOUD_TRUSTED_DOMAINS", :value "somefqdn.de"} {:name "POSTGRES_USER", :valueFrom {:secretKeyRef {:name "postgres-secret", :key "postgres-user"}}} {:name "POSTGRES_PASSWORD", :valueFrom {:secretKeyRef {:name "postgres-secret", :key "postgres-password"}}} {:name "POSTGRES_DB", :valueFrom {:configMapKeyRef {:name "postgres-config", :key "postgres-db"}}} {:name "POSTGRES_HOST", :value "postgresql-service:5432"}] :volumeMounts [{:name "cloud-data-volume", :mountPath "/var/www/html"}]}] :volumes [{:name "cloud-data-volume", :persistentVolumeClaim {:claimName "cloud-pvc"}}]}}}} - (cut/generate-deployment {:fqdn "xx"})))) + (cut/generate-deployment {:fqdn "somefqdn.de"})))) From fe4c38179125b9561b3693d8f2a85372d405b450 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 10:48:03 +0100 Subject: [PATCH 07/26] Use common ingress --- src/main/cljc/dda/c4k_nextcloud/core.cljc | 6 +- .../cljc/dda/c4k_nextcloud/nextcloud.cljc | 14 ++-- src/main/resources/nextcloud/certificate.yaml | 15 ----- src/main/resources/nextcloud/ingress.yaml | 29 --------- .../dda/c4k_nextcloud/nextcloud_test.cljc | 65 ++++++++----------- 5 files changed, 36 insertions(+), 93 deletions(-) delete mode 100644 src/main/resources/nextcloud/certificate.yaml delete mode 100644 src/main/resources/nextcloud/ingress.yaml diff --git a/src/main/cljc/dda/c4k_nextcloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc index 27045f2..6353a10 100644 --- a/src/main/cljc/dda/c4k_nextcloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -21,15 +21,15 @@ [(postgres/generate-config {:postgres-size :8gb}) (postgres/generate-secret config) (postgres/generate-pvc {:pv-storage-size-gb 50 - :pvc-storage-class-name default-storage-class}) + :pvc-storage-class-name default-storage-class}) (postgres/generate-deployment) (postgres/generate-service) (nextcloud/generate-secret config) (nextcloud/generate-pvc (merge nextcloud-default-storage-config config)) (nextcloud/generate-deployment config) (nextcloud/generate-service) - (nextcloud/generate-certificate config) - (nextcloud/generate-ingress config)] + (nextcloud/generate-certificate config)] + (nextcloud/generate-ingress config) (when (:contains? config :restic-repository) [(backup/generate-config config) (backup/generate-secret config) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index 40d11ce..75a3951 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -5,6 +5,7 @@ #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) [dda.c4k-common.yaml :as yaml] + [dda.c4k-common.ingress :as ing] [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] [dda.c4k-common.postgres :as postgres] @@ -62,13 +63,12 @@ (defn-spec generate-ingress cp/map-or-seq? [config config?] - (let [{:keys [fqdn issuer] - :or {issuer "staging"}} config - letsencrypt-issuer issuer] - (-> - (yaml/load-as-edn "nextcloud/ingress.yaml") - (assoc-in [:metadata :annotations :cert-manager.io/cluster-issuer] letsencrypt-issuer) - (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) + (ing/generate-ingress-and-cert + (merge + {:service-name "nextcloud" + :service-port 80 + :fqdns [(:fqdn config)]} + config))) (defn-spec generate-pvc cp/map-or-seq? [config (s/keys :req-un [::pv-storage-size-gb ::pvc-storage-class-name])] diff --git a/src/main/resources/nextcloud/certificate.yaml b/src/main/resources/nextcloud/certificate.yaml deleted file mode 100644 index 461168d..0000000 --- a/src/main/resources/nextcloud/certificate.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: cloud-cert - namespace: default -spec: - secretName: cloud-cert - duration: 2160h # 90d - renewBefore: 360h # 15d - commonName: fqdn - dnsNames: - - fqdn - issuerRef: - name: staging - kind: ClusterIssuer \ No newline at end of file diff --git a/src/main/resources/nextcloud/ingress.yaml b/src/main/resources/nextcloud/ingress.yaml deleted file mode 100644 index 96671eb..0000000 --- a/src/main/resources/nextcloud/ingress.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: ingress-cloud - annotations: - cert-manager.io/cluster-issuer: letsencrypt-staging-issuer - ingress.kubernetes.io/ssl-redirect: "true" - ingress.kubernetes.io/rewrite-target: / - ingress.kubernetes.io/proxy-body-size: "256m" - ingress.kubernetes.io/proxy-connect-timeout: "300" - ingress.kubernetes.io/proxy-send-timeout: "300" - ingress.kubernetes.io/proxy-read-timeout: "300" - namespace: default -spec: - tls: - - hosts: - - fqdn - secretName: cloud-cert - rules: - - host: fqdn - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: cloud-service - port: - number: 80 diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 2cbd51c..83ea56d 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -36,45 +36,32 @@ :nextcloud-admin-user "cloudadmin" :nextcloud-admin-password "cloudpassword"})))) -(deftest should-generate-certificate - (is (= {:apiVersion "cert-manager.io/v1" - :kind "Certificate" - :metadata {:name "cloud-cert", :namespace "default"} - :spec - {:secretName "cloud-cert" - :duration "2160h" - :renewBefore "360h", - :commonName "somefqdn.de", - :dnsNames ["somefqdn.de"] - :issuerRef - {:name "prod", :kind "ClusterIssuer"}}} - (cut/generate-certificate {:fqdn "somefqdn.de" :issuer "prod"})))) - -(deftest should-generate-ingress - (is (= {:apiVersion "networking.k8s.io/v1" - :kind "Ingress" - :metadata - {:name "ingress-cloud" - :annotations - {:cert-manager.io/cluster-issuer "staging" - :ingress.kubernetes.io/proxy-body-size "256m" - :ingress.kubernetes.io/ssl-redirect "true" - :ingress.kubernetes.io/rewrite-target "/" - :ingress.kubernetes.io/proxy-connect-timeout "300" - :ingress.kubernetes.io/proxy-send-timeout "300" - :ingress.kubernetes.io/proxy-read-timeout "300"} - :namespace "default"} - :spec - {:tls [{:hosts ["somefqdn.de"], :secretName "cloud-cert"}] - :rules - [{:host "somefqdn.de" - :http - {:paths - [{:path "/" - :pathType "Prefix" - :backend - {:service - {:name "cloud-service", :port {:number 80}}}}]}}]}} +(deftest should-generate-ingress-and-cert + (is (= [{:apiVersion "cert-manager.io/v1", + :kind "Certificate", + :metadata {:name "nextcloud", :labels {:app.kubernetes.part-of "nextcloud"}, :namespace "default"}, + :spec + {:secretName "nextcloud", + :commonName "somefqdn.de", + :duration "2160h", + :renewBefore "360h", + :dnsNames ["somefqdn.de"], + :issuerRef {:name "staging", :kind "ClusterIssuer"}}} + {:apiVersion "networking.k8s.io/v1", + :kind "Ingress", + :metadata + {:name "nextcloud", + :namespace "default", + :labels {:app.kubernetes.part-of "nextcloud"}, + :annotations + {:traefik.ingress.kubernetes.io/router.entrypoints "web, websecure", + :traefik.ingress.kubernetes.io/router.middlewares "default-redirect-https@kubernetescrd", + :metallb.universe.tf/address-pool "public"}}, + :spec + {:tls [{:hosts ["somefqdn.de"], :secretName "nextcloud"}], + :rules + [{:host "somefqdn.de", + :http {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "nextcloud", :port {:number 80}}}}]}}]}}] (cut/generate-ingress {:fqdn "somefqdn.de"})))) (deftest should-generate-pvc From 84e70af7426d333a44b8f1c0823aee9ac579fe34 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 11:51:22 +0100 Subject: [PATCH 08/26] Use groups for webview --- src/main/cljc/dda/c4k_nextcloud/core.cljc | 27 +++----- .../cljc/dda/c4k_nextcloud/nextcloud.cljc | 18 +----- src/main/cljs/dda/c4k_nextcloud/browser.cljs | 61 +++++++++++-------- .../dda/c4k_nextcloud/nextcloud_test.cljc | 2 +- 4 files changed, 49 insertions(+), 59 deletions(-) diff --git a/src/main/cljc/dda/c4k_nextcloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc index 6353a10..78453bb 100644 --- a/src/main/cljc/dda/c4k_nextcloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -4,6 +4,7 @@ #?(:clj [orchestra.core :refer [defn-spec]] :cljs [orchestra.core :refer-macros [defn-spec]]) [dda.c4k-common.common :as cm] + [dda.c4k-common.predicate :as cp] [dda.c4k-common.yaml :as yaml] [dda.c4k-common.postgres :as postgres] [dda.c4k-nextcloud.nextcloud :as nextcloud] @@ -13,33 +14,25 @@ (def config-defaults {:issuer "staging"}) -(defn-spec k8s-objects any? - [config (s/merge nextcloud/config? nextcloud/auth?)] +(defn-spec k8s-objects cp/map-or-seq? + [config nextcloud/config? + auth nextcloud/auth?] (let [nextcloud-default-storage-config {:pvc-storage-class-name default-storage-class :pv-storage-size-gb 200}] (map yaml/to-string - [(postgres/generate-config {:postgres-size :8gb}) - (postgres/generate-secret config) + [(postgres/generate-config {:postgres-size :8gb :db-name "nextcloud"}) + (postgres/generate-secret auth) (postgres/generate-pvc {:pv-storage-size-gb 50 :pvc-storage-class-name default-storage-class}) (postgres/generate-deployment) (postgres/generate-service) - (nextcloud/generate-secret config) + (nextcloud/generate-secret auth) (nextcloud/generate-pvc (merge nextcloud-default-storage-config config)) (nextcloud/generate-deployment config) - (nextcloud/generate-service) - (nextcloud/generate-certificate config)] - (nextcloud/generate-ingress config) + (nextcloud/generate-service)] + (nextcloud/generate-ingress-and-cert config) (when (:contains? config :restic-repository) [(backup/generate-config config) - (backup/generate-secret config) + (backup/generate-secret auth) (backup/generate-cron) (backup/generate-backup-restore-deployment config)])))) - -(defn-spec generate any? - [my-config nextcloud/config? - my-auth nextcloud/auth?] - (cm/concat-vec - (map yaml/to-string - (filter #(not (nil? %)) - (merge config-defaults my-config my-auth))))) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index 75a3951..cf08a4a 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -37,31 +37,19 @@ #?(:cljs (defmethod yaml/load-resource :nextcloud [resource-name] (case resource-name - "nextcloud/certificate.yaml" (rc/inline "nextcloud/certificate.yaml") "nextcloud/deployment.yaml" (rc/inline "nextcloud/deployment.yaml") - "nextcloud/ingress.yaml" (rc/inline "nextcloud/ingress.yaml") "nextcloud/pvc.yaml" (rc/inline "nextcloud/pvc.yaml") "nextcloud/service.yaml" (rc/inline "nextcloud/service.yaml") "nextcloud/secret.yaml" (rc/inline "nextcloud/secret.yaml") (throw (js/Error. "Undefined Resource!"))))) -(defn-spec generate-certificate cp/map-or-seq? - [config config?] - (let [{:keys [fqdn issuer]} config - letsencrypt-issuer issuer] - (-> - (yaml/load-as-edn "nextcloud/certificate.yaml") - (assoc-in [:spec :commonName] fqdn) - (assoc-in [:spec :dnsNames] [fqdn]) - (assoc-in [:spec :issuerRef :name] letsencrypt-issuer)))) - (defn-spec generate-deployment cp/map-or-seq? [config config?] (let [{:keys [fqdn]} config] (-> (yaml/load-as-edn "nextcloud/deployment.yaml") (cm/replace-all-matching-values-by-new-value "fqdn" fqdn)))) -(defn-spec generate-ingress cp/map-or-seq? +(defn-spec generate-ingress-and-cert cp/map-or-seq? [config config?] (ing/generate-ingress-and-cert (merge @@ -82,8 +70,8 @@ (yaml/load-as-edn "nextcloud/service.yaml")) (defn-spec generate-secret cp/map-or-seq? - [config config?] - (let [{:keys [nextcloud-admin-user nextcloud-admin-password]} config] + [auth auth?] + (let [{:keys [nextcloud-admin-user nextcloud-admin-password]} auth] (-> (yaml/load-as-edn "nextcloud/secret.yaml") (cm/replace-key-value :nextcloud-admin-user (b64/encode nextcloud-admin-user)) diff --git a/src/main/cljs/dda/c4k_nextcloud/browser.cljs b/src/main/cljs/dda/c4k_nextcloud/browser.cljs index eee519c..411bd88 100644 --- a/src/main/cljs/dda/c4k_nextcloud/browser.cljs +++ b/src/main/cljs/dda/c4k_nextcloud/browser.cljs @@ -1,31 +1,36 @@ (ns dda.c4k-nextcloud.browser (:require [clojure.tools.reader.edn :as edn] + [dda.c4k-common.common :as cm] [dda.c4k-nextcloud.core :as core] [dda.c4k-nextcloud.nextcloud :as nextcloud] [dda.c4k-common.browser :as br] [dda.c4k-common.postgres :as pgc])) -(defn generate-content - [] - (into [] (concat [(assoc (br/generate-needs-validation) :content - (into [] (concat (br/generate-input-field "fqdn" "Your fqdn:" "nextcloud-neu.prod.meissa-gmbh.de") - (br/generate-input-field "nextcloud-data-volume-path" "(Optional) Your nextcloud-data-volume-path:" "/var/nextcloud") - (br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path:" "/var/postgres") - (br/generate-input-field "restic-repository" "(Optional) Your restic-repository:" "restic-repository") - (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "") - [(br/generate-br)] - (br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\" +(defn generate-content [] + (cm/concat-vec + [(assoc + (br/generate-needs-validation) :content + (cm/concat-vec + (br/generate-group "domain" + (cm/concat-vec (br/generate-input-field "fqdn" "Your fqdn:" "nextcloud-neu.prod.meissa-gmbh.de") + (br/generate-input-field "pv-storage-size-gb" "(Optional) Your nextcloud storage size in GB" "8") + (br/generate-input-field "pvc-storage-class-name" "(Optional) Your storage class type (manual / local-path):" "local-path") + (br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path:" "/var/postgres") + (br/generate-input-field "restic-repository" "(Optional) Your restic-repository:" "restic-repository") + (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" ""))) + (br/generate-group "credentials" + (br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\" :postgres-db-password \"nextcloud-db-password\" :nextcloud-admin-password \"nextcloud-admin-password\" :nextcloud-admin-user \"nextcloud-admin-user\" :aws-access-key-id \"aws-id\" :aws-secret-access-key \"aws-secret\" :restic-password \"restic-password\"}" - "5") - [(br/generate-br)] - (br/generate-button "generate-button" "Generate c4k yaml"))))] - (br/generate-output "c4k-nextcloud-output" "Your c4k deployment.yaml:" "25")))) + "5")) + [(br/generate-br)] + (br/generate-button "generate-button" "Generate c4k yaml")))] + (br/generate-output "c4k-nextcloud-output" "Your c4k deployment.yaml:" "25"))) (defn generate-content-div [] @@ -35,29 +40,30 @@ (generate-content)}) (defn config-from-document [] - (let [nextcloud-data-volume-path (br/get-content-from-element "nextcloud-data-volume-path" :optional true) + (let [pv-storage-size-gb (br/get-content-from-element "pv-storage-size-gb" :optional true) + pvc-storage-class-name (br/get-content-from-element "pvc-storage-class-name" :optional true) postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true) restic-repository (br/get-content-from-element "restic-repository" :optional true) issuer (br/get-content-from-element "issuer" :optional true :deserializer keyword)] (merge {:fqdn (br/get-content-from-element "fqdn")} - (when (some? nextcloud-data-volume-path) - {:nextcloud-data-volume-path nextcloud-data-volume-path}) + (when (and (some? pv-storage-size-gb) (some? pvc-storage-class-name)) + {:pv-storage-size-gb pv-storage-size-gb :pvc-storage-class-name pvc-storage-class-name}) (when (some? postgres-data-volume-path) {:postgres-data-volume-path postgres-data-volume-path}) (when (some? restic-repository) {:restic-repository restic-repository}) (when (some? issuer) - {:issuer issuer}) - ))) + {:issuer issuer})))) (defn validate-all! [] (br/validate! "fqdn" ::nextcloud/fqdn) - (br/validate! "nextcloud-data-volume-path" ::nextcloud/nextcloud-data-volume-path :optional true) + (br/validate! "pv-storage-size-gb" ::nextcloud/pv-storage-size-gb :optional true) + (br/validate! "pvc-storage-class-name" ::nextcloud/pvc-storage-class-name :optional true) (br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true) (br/validate! "restic-repository" ::nextcloud/restic-repository :optional true) - (br/validate! "issuer" ::nextcloud/issuer :optional true :deserializer keyword) - (br/validate! "auth" core/auth? :deserializer edn/read-string) + (br/validate! "issuer" ::nextcloud/issuer :optional true) + (br/validate! "auth" nextcloud/auth? :deserializer edn/read-string) (br/set-validated!)) (defn add-validate-listener [name] @@ -70,12 +76,15 @@ (.getElementById "generate-button") (.addEventListener "click" #(do (validate-all!) - (-> (core/generate - (config-from-document) - (br/get-content-from-element "auth" :deserializer edn/read-string)) + (-> (cm/generate-common + (config-from-document) + (br/get-content-from-element "auth" :deserializer edn/read-string) + {} + core/k8s-objects) (br/set-output!))))) (add-validate-listener "fqdn") - (add-validate-listener "nextcloud-data-volume-path") + (add-validate-listener "pv-storage-size-gb") + (add-validate-listener "pvc-storage-class-name") (add-validate-listener "postgres-data-volume-path") (add-validate-listener "restic-repository") (add-validate-listener "issuer") diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 83ea56d..9d28d7c 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -62,7 +62,7 @@ :rules [{:host "somefqdn.de", :http {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "nextcloud", :port {:number 80}}}}]}}]}}] - (cut/generate-ingress {:fqdn "somefqdn.de"})))) + (cut/generate-ingress-and-cert {:fqdn "somefqdn.de"})))) (deftest should-generate-pvc (is (= {:apiVersion "v1" From e74280bf78c7d71d9c795628b9a1e8a2603495d2 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 11:59:58 +0100 Subject: [PATCH 09/26] Fix spec for should-generate-secret --- src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 9d28d7c..e449782 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -32,7 +32,11 @@ :data {:nextcloud-admin-user "Y2xvdWRhZG1pbg==" :nextcloud-admin-password "Y2xvdWRwYXNzd29yZA=="}} - (cut/generate-secret {:fqdn "somefqdn.de" + (cut/generate-secret {:postgres-db-user "postgres-user" + :postgres-db-password "postgres-password" + :aws-access-key-id "aws-key" + :aws-secret-access-key "aws-secret-key" + :restic-password "restic-password" :nextcloud-admin-user "cloudadmin" :nextcloud-admin-password "cloudpassword"})))) @@ -108,4 +112,4 @@ {:name "POSTGRES_HOST", :value "postgresql-service:5432"}] :volumeMounts [{:name "cloud-data-volume", :mountPath "/var/www/html"}]}] :volumes [{:name "cloud-data-volume", :persistentVolumeClaim {:claimName "cloud-pvc"}}]}}}} - (cut/generate-deployment {:fqdn "somefqdn.de"})))) + (cut/generate-deployment {:fqdn "somefqdn.de"})))) \ No newline at end of file From d6ab7c555ae1ecf8b201125ef357e427dc8f7191 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:07:00 +0100 Subject: [PATCH 10/26] Use common monitoring --- doc/SetupOnHetzner.md | 14 ++++++++------ src/main/cljc/dda/c4k_nextcloud/core.cljc | 7 +++++-- src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc | 9 ++++++--- src/test/resources/nextcloud-test/valid-auth.edn | 4 +++- src/test/resources/nextcloud-test/valid-config.edn | 5 ++++- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/doc/SetupOnHetzner.md b/doc/SetupOnHetzner.md index 9872457..359eddf 100644 --- a/doc/SetupOnHetzner.md +++ b/doc/SetupOnHetzner.md @@ -45,15 +45,17 @@ output "ipv4" { ## k8s minicluster -For k8s installation we use our [dda-k8s-crate](https://github.com/DomainDrivenArchitecture/dda-k8s-crate) with the following configuation: +For k8s installation we use our [provs](https://repo.prod.meissa.de/meissa/provs) with the following configuation: ``` -{:user :k8s - :k8s {:external-ip "ip-from-above"} - :cert-manager :letsencrypt-prod-issuer - :persistent-dirs ["cloud", "postgres"] - } +postgres-db-user: "nextcloud" +postgres-db-password: "nextcloud-db-password" +nextcloud-admin-user: "cloudadmin" +nextcloud-admin-password: "cloudpassword" +aws-access-key-id: "aws-id" +aws-secret-access-key: "aws-secret" +restic-password: "restic-password" ``` ## kubectl apply c4k-nextcloud diff --git a/src/main/cljc/dda/c4k_nextcloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc index 78453bb..0156b89 100644 --- a/src/main/cljc/dda/c4k_nextcloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -8,7 +8,8 @@ [dda.c4k-common.yaml :as yaml] [dda.c4k-common.postgres :as postgres] [dda.c4k-nextcloud.nextcloud :as nextcloud] - [dda.c4k-nextcloud.backup :as backup])) + [dda.c4k-nextcloud.backup :as backup] + [dda.c4k-common.monitoring :as mon])) (def default-storage-class :local-path) @@ -35,4 +36,6 @@ [(backup/generate-config config) (backup/generate-secret auth) (backup/generate-cron) - (backup/generate-backup-restore-deployment config)])))) + (backup/generate-backup-restore-deployment config)]) + (when (:contains? config :mon-cfg) + (mon/generate (:mon-cfg config) (:mon-auth auth)))))) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index cf08a4a..d2f9e4b 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -9,7 +9,8 @@ [dda.c4k-common.base64 :as b64] [dda.c4k-common.predicate :as cp] [dda.c4k-common.postgres :as postgres] - [dda.c4k-common.common :as cm])) + [dda.c4k-common.common :as cm] + [dda.c4k-common.monitoring :as mon])) (s/def ::fqdn cp/fqdn-string?) (s/def ::issuer cp/letsencrypt-issuer?) @@ -27,12 +28,14 @@ :opt-un [::issuer ::restic-repository ::pv-storage-size-gb - ::pvc-storage-class-name])) + ::pvc-storage-class-name + ::mon/mon-cfg])) (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::nextcloud-admin-user ::nextcloud-admin-password ::aws-access-key-id ::aws-secret-access-key - ::restic-password])) + ::restic-password] + :opt-un [::mon/mon-auth])) #?(:cljs (defmethod yaml/load-resource :nextcloud [resource-name] diff --git a/src/test/resources/nextcloud-test/valid-auth.edn b/src/test/resources/nextcloud-test/valid-auth.edn index 60f0255..5cee93d 100644 --- a/src/test/resources/nextcloud-test/valid-auth.edn +++ b/src/test/resources/nextcloud-test/valid-auth.edn @@ -4,4 +4,6 @@ :nextcloud-admin-password "cloudpassword" :aws-access-key-id "aws-id" :aws-secret-access-key "aws-secret" - :restic-password "restic-password"} \ No newline at end of file + :restic-password "restic-password" + :mon-auth {:grafana-cloud-user "user" + :grafana-cloud-password "password"}} \ No newline at end of file diff --git a/src/test/resources/nextcloud-test/valid-config.edn b/src/test/resources/nextcloud-test/valid-config.edn index b50ced9..374f961 100644 --- a/src/test/resources/nextcloud-test/valid-config.edn +++ b/src/test/resources/nextcloud-test/valid-config.edn @@ -2,4 +2,7 @@ :issuer "staging" :nextcloud-data-volume-path "/var/cloud" :postgres-data-volume-path "/var/postgres" - :restic-repository "s3:s3.amazonaws.com/your-bucket/your-folder"} \ No newline at end of file + :restic-repository "s3:s3.amazonaws.com/your-bucket/your-folder" + :mon-cfg {:grafana-cloud-url "url-for-your-prom-remote-write-endpoint" + :k3s-cluster-name "jitsi" + :k3s-cluster-stage "test"}} \ No newline at end of file From 234c920c633f09e1dbaefe8576b061b4bc5a751b Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:07:17 +0100 Subject: [PATCH 11/26] Remove unused variable --- src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index d2f9e4b..1abc065 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -20,10 +20,6 @@ (s/def ::pvc-storage-class-name cp/pvc-storage-class-name?) (s/def ::pv-storage-size-gb pos?) -(def strong-config? (s/keys :req-un [::fqdn ::issuer ::pv-storage-size-gb - ::pvc-storage-class-name] - :opt-un [::restic-repository])) - (def config? (s/keys :req-un [::fqdn] :opt-un [::issuer ::restic-repository From 3ffd98786b45817f7f959fdf30755754939838c2 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:27:35 +0100 Subject: [PATCH 12/26] Fix k8s-objects --- src/main/clj/dda/c4k_nextcloud/uberjar.clj | 2 +- src/main/cljc/dda/c4k_nextcloud/core.cljc | 39 ++++++++++++---------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/main/clj/dda/c4k_nextcloud/uberjar.clj b/src/main/clj/dda/c4k_nextcloud/uberjar.clj index 4983f70..368a49a 100644 --- a/src/main/clj/dda/c4k_nextcloud/uberjar.clj +++ b/src/main/clj/dda/c4k_nextcloud/uberjar.clj @@ -11,5 +11,5 @@ nextcloud/config? nextcloud/auth? core/config-defaults - core/generate + core/k8s-objects cmd-args)) diff --git a/src/main/cljc/dda/c4k_nextcloud/core.cljc b/src/main/cljc/dda/c4k_nextcloud/core.cljc index 0156b89..32fb75d 100644 --- a/src/main/cljc/dda/c4k_nextcloud/core.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/core.cljc @@ -21,21 +21,24 @@ (let [nextcloud-default-storage-config {:pvc-storage-class-name default-storage-class :pv-storage-size-gb 200}] (map yaml/to-string - [(postgres/generate-config {:postgres-size :8gb :db-name "nextcloud"}) - (postgres/generate-secret auth) - (postgres/generate-pvc {:pv-storage-size-gb 50 - :pvc-storage-class-name default-storage-class}) - (postgres/generate-deployment) - (postgres/generate-service) - (nextcloud/generate-secret auth) - (nextcloud/generate-pvc (merge nextcloud-default-storage-config config)) - (nextcloud/generate-deployment config) - (nextcloud/generate-service)] - (nextcloud/generate-ingress-and-cert config) - (when (:contains? config :restic-repository) - [(backup/generate-config config) - (backup/generate-secret auth) - (backup/generate-cron) - (backup/generate-backup-restore-deployment config)]) - (when (:contains? config :mon-cfg) - (mon/generate (:mon-cfg config) (:mon-auth auth)))))) + (filter + #(not (nil? %)) + (cm/concat-vec + [(postgres/generate-config {:postgres-size :8gb :db-name "nextcloud"}) + (postgres/generate-secret auth) + (postgres/generate-pvc {:pv-storage-size-gb 50 + :pvc-storage-class-name default-storage-class}) + (postgres/generate-deployment) + (postgres/generate-service) + (nextcloud/generate-secret auth) + (nextcloud/generate-pvc (merge nextcloud-default-storage-config config)) + (nextcloud/generate-deployment config) + (nextcloud/generate-service)] + (nextcloud/generate-ingress-and-cert config) + (when (:contains? config :restic-repository) + [(backup/generate-config config) + (backup/generate-secret auth) + (backup/generate-cron) + (backup/generate-backup-restore-deployment config)]) + (when (:contains? config :mon-cfg) + (mon/generate (:mon-cfg config) (:mon-auth auth)))))))) From be684408a087ea77fcf5b2478abcb9605dd04272 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:33:49 +0100 Subject: [PATCH 13/26] Add monitoring config/auth to browser --- src/main/cljs/dda/c4k_nextcloud/browser.cljs | 27 +++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/cljs/dda/c4k_nextcloud/browser.cljs b/src/main/cljs/dda/c4k_nextcloud/browser.cljs index 411bd88..cb151c1 100644 --- a/src/main/cljs/dda/c4k_nextcloud/browser.cljs +++ b/src/main/cljs/dda/c4k_nextcloud/browser.cljs @@ -2,6 +2,7 @@ (:require [clojure.tools.reader.edn :as edn] [dda.c4k-common.common :as cm] + [dda.c4k-common.monitoring :as mon] [dda.c4k-nextcloud.core :as core] [dda.c4k-nextcloud.nextcloud :as nextcloud] [dda.c4k-common.browser :as br] @@ -14,11 +15,14 @@ (cm/concat-vec (br/generate-group "domain" (cm/concat-vec (br/generate-input-field "fqdn" "Your fqdn:" "nextcloud-neu.prod.meissa-gmbh.de") + (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "") (br/generate-input-field "pv-storage-size-gb" "(Optional) Your nextcloud storage size in GB" "8") (br/generate-input-field "pvc-storage-class-name" "(Optional) Your storage class type (manual / local-path):" "local-path") (br/generate-input-field "postgres-data-volume-path" "(Optional) Your postgres-data-volume-path:" "/var/postgres") (br/generate-input-field "restic-repository" "(Optional) Your restic-repository:" "restic-repository") - (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" ""))) + (br/generate-input-field "mon-cluster-name" "(Optional) monitoring cluster name:" "keycloak") + (br/generate-input-field "mon-cluster-stage" "(Optional) monitoring cluster stage:" "test") + (br/generate-input-field "mon-cloud-url" "(Optional) grafana cloud url:" "https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push"))) (br/generate-group "credentials" (br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"nextcloud\" :postgres-db-password \"nextcloud-db-password\" @@ -26,7 +30,9 @@ :nextcloud-admin-user \"nextcloud-admin-user\" :aws-access-key-id \"aws-id\" :aws-secret-access-key \"aws-secret\" - :restic-password \"restic-password\"}" + :restic-password \"restic-password\"} + :mon-auth {:grafana-cloud-user \"your-user-id\" + :grafana-cloud-password \"your-cloud-password\"}" "5")) [(br/generate-br)] (br/generate-button "generate-button" "Generate c4k yaml")))] @@ -44,7 +50,10 @@ pvc-storage-class-name (br/get-content-from-element "pvc-storage-class-name" :optional true) postgres-data-volume-path (br/get-content-from-element "postgres-data-volume-path" :optional true) restic-repository (br/get-content-from-element "restic-repository" :optional true) - issuer (br/get-content-from-element "issuer" :optional true :deserializer keyword)] + issuer (br/get-content-from-element "issuer" :optional true) + mon-cluster-name (br/get-content-from-element "mon-cluster-name" :optional true) + mon-cluster-stage (br/get-content-from-element "mon-cluster-stage" :optional true) + mon-cloud-url (br/get-content-from-element "mon-cloud-url" :optional true)] (merge {:fqdn (br/get-content-from-element "fqdn")} (when (and (some? pv-storage-size-gb) (some? pvc-storage-class-name)) @@ -54,7 +63,11 @@ (when (some? restic-repository) {:restic-repository restic-repository}) (when (some? issuer) - {:issuer issuer})))) + {:issuer issuer}) + (when (some? mon-cluster-name) + {:mon-cfg {:cluster-name mon-cluster-name + :cluster-stage (keyword mon-cluster-stage) + :grafana-cloud-url mon-cloud-url}})))) (defn validate-all! [] (br/validate! "fqdn" ::nextcloud/fqdn) @@ -63,6 +76,9 @@ (br/validate! "postgres-data-volume-path" ::pgc/postgres-data-volume-path :optional true) (br/validate! "restic-repository" ::nextcloud/restic-repository :optional true) (br/validate! "issuer" ::nextcloud/issuer :optional true) + (br/validate! "mon-cluster-name" ::mon/cluster-name :optional true) + (br/validate! "mon-cluster-stage" ::mon/cluster-stage :optional true) + (br/validate! "mon-cloud-url" ::mon/grafana-cloud-url :optional true) (br/validate! "auth" nextcloud/auth? :deserializer edn/read-string) (br/set-validated!)) @@ -88,4 +104,7 @@ (add-validate-listener "postgres-data-volume-path") (add-validate-listener "restic-repository") (add-validate-listener "issuer") + (add-validate-listener "mon-cluster-name") + (add-validate-listener "mon-cluster-stage") + (add-validate-listener "mon-cloud-url") (add-validate-listener "auth")) From efac24b6df4455381b763cb37195e1857f9c3723 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:43:20 +0100 Subject: [PATCH 14/26] Fix import name --- src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index e449782..0954a82 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -2,7 +2,7 @@ (:require #?(:clj [clojure.test :refer [deftest is are testing run-tests]] :cljs [cljs.test :refer-macros [deftest is are testing run-tests]]) - #?(:cljs [shadow-resource :as rc]) + #?(:cljs [shadow.resource :as rc]) [clojure.spec.alpha :as s] [clojure.spec.test.alpha :as st] [dda.c4k-common.yaml :as yaml] From 483fb69918a6a48d4779ab01e9ca3034c782e6c0 Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:44:26 +0100 Subject: [PATCH 15/26] Point to right valid config/auth file in ci --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35ee19c..c6fc919 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,7 +49,7 @@ test-schema: stage: build_and_test script: - lein uberjar - - java -jar target/uberjar/c4k-nextcloud-standalone.jar valid-config.edn valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip "Certificate,CronJob" - + - java -jar target/uberjar/c4k-nextcloud-standalone.jar src/test/resources/nextcloud-test/valid-config.edn src/test/resources/nextcloud-test/valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip "Certificate,CronJob" - artifacts: paths: - target/uberjar From f43e46ac3ba2ef1ae237e60deb4a24ec4a00c9ee Mon Sep 17 00:00:00 2001 From: bom Date: Fri, 3 Feb 2023 12:51:12 +0100 Subject: [PATCH 16/26] Fix ci and valid config/auths --- .gitlab-ci.yml | 2 +- src/test/resources/nextcloud-test/valid-auth.yaml | 5 ++++- src/test/resources/nextcloud-test/valid-config.yaml | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6fc919..f099ef9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -49,7 +49,7 @@ test-schema: stage: build_and_test script: - lein uberjar - - java -jar target/uberjar/c4k-nextcloud-standalone.jar src/test/resources/nextcloud-test/valid-config.edn src/test/resources/nextcloud-test/valid-auth.edn | kubeconform --kubernetes-version 1.19.0 --strict --skip "Certificate,CronJob" - + - java -jar target/uberjar/c4k-nextcloud-standalone.jar src/test/resources/nextcloud-test/valid-config.yaml src/test/resources/nextcloud-test/valid-auth.yaml | kubeconform --kubernetes-version 1.19.0 --strict --skip "Certificate,CronJob" - artifacts: paths: - target/uberjar diff --git a/src/test/resources/nextcloud-test/valid-auth.yaml b/src/test/resources/nextcloud-test/valid-auth.yaml index 247409a..6d88bf3 100644 --- a/src/test/resources/nextcloud-test/valid-auth.yaml +++ b/src/test/resources/nextcloud-test/valid-auth.yaml @@ -4,4 +4,7 @@ nextcloud-admin-user: "cloudadmin" nextcloud-admin-password: "cloudpassword" aws-access-key-id: "aws-id" aws-secret-access-key: "aws-secret" -restic-password: "restic-password" \ No newline at end of file +restic-password: "restic-password" +mon-auth: + grafana-cloud-user: "user" + grafana-cloud-password: "password" \ No newline at end of file diff --git a/src/test/resources/nextcloud-test/valid-config.yaml b/src/test/resources/nextcloud-test/valid-config.yaml index c716cf2..48178b0 100644 --- a/src/test/resources/nextcloud-test/valid-config.yaml +++ b/src/test/resources/nextcloud-test/valid-config.yaml @@ -2,4 +2,8 @@ fqdn: "cloud.test.meissa-gmbh.de" issuer: "staging" nextcloud-data-volume-path: "/var/cloud" postgres-data-volume-path: "/var/postgres" -restic-repository: "s3:s3.amazonaws.com/your-bucket/your-folder" \ No newline at end of file +restic-repository: "s3:s3.amazonaws.com/your-bucket/your-folder" +mon-cfg: + grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" + cluster-name: "keycloak" + cluster-stage: "test" \ No newline at end of file From 27c204491379391100341a8bedb9a45ed06697d8 Mon Sep 17 00:00:00 2001 From: jerger Date: Tue, 14 Feb 2023 09:15:27 +0100 Subject: [PATCH 17/26] update lein inst --- project.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/project.clj b/project.clj index 797ba25..49dcb1d 100644 --- a/project.clj +++ b/project.clj @@ -39,8 +39,7 @@ "-H:ResourceConfigurationFiles=graalvm-resource-config.json" "-H:Log=registerResource" "-H:Name=target/graalvm/${:name}"] - "inst" ["shell" "sudo" - "install" - "-m=755" - "target/uberjar/c4k-nextcloud-standalone.jar" - "/usr/local/bin/c4k-nextcloud-standalone.jar"]}) \ No newline at end of file + "inst" ["shell" + "sh" + "-c" + "lein uberjar && sudo install -m=755 target/uberjar/c4k-nextcloud-standalone.jar /usr/local/bin/c4k-nextcloud-standalone.jar"]}) \ No newline at end of file From 4cad3daa3f944f4a5fd58c7483b84c52cf878859 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:31:35 +0100 Subject: [PATCH 18/26] Update name of service in ingress Otherwise the ingress doesn't know the service it's pointing to --- src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc index 1abc065..f6fb875 100644 --- a/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc +++ b/src/main/cljc/dda/c4k_nextcloud/nextcloud.cljc @@ -52,7 +52,7 @@ [config config?] (ing/generate-ingress-and-cert (merge - {:service-name "nextcloud" + {:service-name "cloud-service" :service-port 80 :fqdns [(:fqdn config)]} config))) From 2198b2c76c5023cc35a65ee9b47da36dc4b81cee Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:31:55 +0100 Subject: [PATCH 19/26] Pin nextcloud image version --- src/main/resources/nextcloud/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/nextcloud/deployment.yaml b/src/main/resources/nextcloud/deployment.yaml index 4bca0a9..3f7bb26 100644 --- a/src/main/resources/nextcloud/deployment.yaml +++ b/src/main/resources/nextcloud/deployment.yaml @@ -18,7 +18,7 @@ spec: redeploy: v3 spec: containers: - - image: domaindrivenarchitecture/c4k-cloud + - image: domaindrivenarchitecture/c4k-cloud:4.0.3 name: cloud-app imagePullPolicy: IfNotPresent ports: From 621543419940aefdb706060d8e3157f0448e512f Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:32:20 +0100 Subject: [PATCH 20/26] Update nextcloud version to newest release --- infrastructure/docker-nextcloud/image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/docker-nextcloud/image/Dockerfile b/infrastructure/docker-nextcloud/image/Dockerfile index 61ac8ab..e65e42b 100644 --- a/infrastructure/docker-nextcloud/image/Dockerfile +++ b/infrastructure/docker-nextcloud/image/Dockerfile @@ -1,4 +1,4 @@ -FROM nextcloud:22 +FROM nextcloud:25 # Prepare Entrypoint Script ADD resources /tmp From 0e14a59f6dfc5517569b6153ae878dbd6531b7eb Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:32:40 +0100 Subject: [PATCH 21/26] Add docs for upgrading a live deployment --- doc/LiveUpgrade.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/LiveUpgrade.md diff --git a/doc/LiveUpgrade.md b/doc/LiveUpgrade.md new file mode 100644 index 0000000..cb422a4 --- /dev/null +++ b/doc/LiveUpgrade.md @@ -0,0 +1,17 @@ +# Upgrade major or minor versions of nextcloud + +## Nextcloud versions of c4k-nextcloud docker images + +- 4.0.3: nextcloud 22 +- 5.0.0: nextcloud 23 +- 6.0.0: nextcloud 24 +- 7.0.0: nextcloud 25 + +## Uprgrading process + +1. Change the version of the docker image in the deployment to the next major version + - `kubectl edit deploy cloud-deployment` + - change `image: domaindrivenarchitecture/c4k-cloud:4.0.3` +2. Wait for the pod to finish restarting +3. Verify the website is working and https://URL/settings/admin/overview shows the correct version +4. Repeat until desired version is reached From 861f1f1c21cdf0d9dcc18c4ff8fd16231f697066 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:42:45 +0100 Subject: [PATCH 22/26] Update pinned version --- src/main/resources/nextcloud/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/nextcloud/deployment.yaml b/src/main/resources/nextcloud/deployment.yaml index 3f7bb26..36bccec 100644 --- a/src/main/resources/nextcloud/deployment.yaml +++ b/src/main/resources/nextcloud/deployment.yaml @@ -18,7 +18,7 @@ spec: redeploy: v3 spec: containers: - - image: domaindrivenarchitecture/c4k-cloud:4.0.3 + - image: domaindrivenarchitecture/c4k-cloud:7.0.0 name: cloud-app imagePullPolicy: IfNotPresent ports: From 11323e283237f6e446276f27d05e6b8239b3a1fb Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:44:42 +0100 Subject: [PATCH 23/26] Fix tests --- .../dda/c4k_nextcloud/nextcloud_test.cljc | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc index 0954a82..583fa18 100644 --- a/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc +++ b/src/test/cljc/dda/c4k_nextcloud/nextcloud_test.cljc @@ -43,9 +43,12 @@ (deftest should-generate-ingress-and-cert (is (= [{:apiVersion "cert-manager.io/v1", :kind "Certificate", - :metadata {:name "nextcloud", :labels {:app.kubernetes.part-of "nextcloud"}, :namespace "default"}, + :metadata + {:name "cloud-service", + :labels {:app.kubernetes.part-of "cloud-service"}, + :namespace "default"}, :spec - {:secretName "nextcloud", + {:secretName "cloud-service", :commonName "somefqdn.de", :duration "2160h", :renewBefore "360h", @@ -54,18 +57,24 @@ {:apiVersion "networking.k8s.io/v1", :kind "Ingress", :metadata - {:name "nextcloud", + {:name "cloud-service", :namespace "default", - :labels {:app.kubernetes.part-of "nextcloud"}, + :labels {:app.kubernetes.part-of "cloud-service"}, :annotations {:traefik.ingress.kubernetes.io/router.entrypoints "web, websecure", - :traefik.ingress.kubernetes.io/router.middlewares "default-redirect-https@kubernetescrd", + :traefik.ingress.kubernetes.io/router.middlewares + "default-redirect-https@kubernetescrd", :metallb.universe.tf/address-pool "public"}}, :spec - {:tls [{:hosts ["somefqdn.de"], :secretName "nextcloud"}], + {:tls [{:hosts ["somefqdn.de"], :secretName "cloud-service"}], :rules [{:host "somefqdn.de", - :http {:paths [{:pathType "Prefix", :path "/", :backend {:service {:name "nextcloud", :port {:number 80}}}}]}}]}}] + :http + {:paths + [{:pathType "Prefix", + :path "/", + :backend + {:service {:name "cloud-service", :port {:number 80}}}}]}}]}}] (cut/generate-ingress-and-cert {:fqdn "somefqdn.de"})))) (deftest should-generate-pvc @@ -89,7 +98,7 @@ {:metadata {:labels {:app "cloud-app", :app.kubernetes.io/name "cloud-pod", :app.kubernetes.io/application "cloud", :redeploy "v3"}} :spec {:containers - [{:image "domaindrivenarchitecture/c4k-cloud" + [{:image "domaindrivenarchitecture/c4k-cloud:7.0.0" :name "cloud-app" :imagePullPolicy "IfNotPresent" :ports [{:containerPort 80}] From 53e4f0781fa7cadd2fc298271a16cf9d48beed72 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:45:10 +0100 Subject: [PATCH 24/26] Prepare release v7.0.0 --- package.json | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 256d0c5..28d7ecb 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "c4k-nextcloud", "description": "Generate c4k yaml for a nextcloud deployment.", "author": "meissa GmbH", - "version": "4.0.3-SNAPSHOT", + "version": "7.0.0", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme", "repository": "https://www.npmjs.com/package/c4k-nextcloud", "license": "APACHE2", diff --git a/project.clj b/project.clj index 49dcb1d..5d29fbc 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.domaindrivenarchitecture/c4k-nextcloud "4.0.4-SNAPSHOT" +(defproject org.domaindrivenarchitecture/c4k-nextcloud "7.0.0-SNAPSHOT" :description "nextcloud c4k-installation package" :url "https://domaindrivenarchitecture.org" :license {:name "Apache License, Version 2.0" From c3c2740e5a901939da01d70861505e2cdc8ec81b Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:45:16 +0100 Subject: [PATCH 25/26] Version 7.0.0 --- project.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project.clj b/project.clj index 5d29fbc..38e6c46 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.domaindrivenarchitecture/c4k-nextcloud "7.0.0-SNAPSHOT" +(defproject org.domaindrivenarchitecture/c4k-nextcloud "7.0.0" :description "nextcloud c4k-installation package" :url "https://domaindrivenarchitecture.org" :license {:name "Apache License, Version 2.0" From 8afc7ee9e1bafd4f96cd85229a84528c5bee9a20 Mon Sep 17 00:00:00 2001 From: bom Date: Thu, 2 Mar 2023 11:45:46 +0100 Subject: [PATCH 26/26] Version bump --- package.json | 2 +- project.clj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 28d7ecb..85d98af 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "c4k-nextcloud", "description": "Generate c4k yaml for a nextcloud deployment.", "author": "meissa GmbH", - "version": "7.0.0", + "version": "7.0.1-SNAPSHOT", "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-nextcloud#readme", "repository": "https://www.npmjs.com/package/c4k-nextcloud", "license": "APACHE2", diff --git a/project.clj b/project.clj index 38e6c46..38f38fe 100644 --- a/project.clj +++ b/project.clj @@ -1,4 +1,4 @@ -(defproject org.domaindrivenarchitecture/c4k-nextcloud "7.0.0" +(defproject org.domaindrivenarchitecture/c4k-nextcloud "7.0.1-SNAPSHOT" :description "nextcloud c4k-installation package" :url "https://domaindrivenarchitecture.org" :license {:name "Apache License, Version 2.0"