bump version to: 1.2.7-SNAPSHOT

.gitignore

@@ -27,8 +27,7 @@ public/js/
 *.iml
 .idea/
 
-auth.edn
-config.edn
-
 build-and-move-frontend.sh
 website.yaml
+
+.envrc

doc/BackupAndRestore.md

@@ -1,77 +1,9 @@
+
 # Backup Architecture details
 
-![](backup.svg)
+Use process documented at https://repo.prod.meissa.de/meissa/dda-backup/src/branch/main/docs/Backup.md
 
-* we use restic to produce small & encrypted backups
-* backup is scheduled at `schedule: "10 23 * * *"`
-* Cloud stores files on `/var/jira`, these files are backuped. If you create a jira xml backup located in /var/jira this file will also be backed up.
-* postgres db is backed up as pgdump
+Parameters are:
 
-## Manual backup
-
-1. Scale gateway and front deployment down:   
-   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=0`
-   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=0`
-2. Scale backup-restore deployment up:   
-   `kubectl -n taiga scale deployment backup-restore --replicas=1`
-3. exec into pod and execute restore pod   
-   `kubectl -n taiga exec -it backup-restore -- backup.bb`
-4. Scale backup-restore deployment down:   
-  `kubectl -n taiga scale deployment backup-restore --replicas=0`
-1. Scale gateway and front deployment up:   
-   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=1`
-   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=1`
-
-## Manual restore
-
-1. Scale gateway and front deployment down:   
-   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=0`
-   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=0`
-2. Scale backup-restore deployment up:   
-   `kubectl -n taiga scale deployment backup-restore --replicas=1`
-3. exec into pod and execute restore pod   
-   `kubectl -n taiga exec -it backup-restore -- restore.bb`
-4. Scale backup-restore deployment down:   
-  `kubectl -n taiga scale deployment backup-restore --replicas=0`
-5. Scale gateway and front deployment up:   
-   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=1`
-   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=1`
-
-## Change Password
-
-1. Check restic-new-password env is set in backup deployment   
-   ```
-   kind: Deployment
-   metadata:
-     name: backup-restore
-   spec:
-       spec:
-         containers:
-         - name: backup-app
-           env:
-           - name: RESTIC_NEW_PASSWORD_FILE
-             value: /var/run/secrets/backup-secrets/restic-new-password
-   ```
-2. Add restic-new-password to secret   
-   ```
-   kind: Secret
-   metadata:
-     name: backup-secret
-   data:
-     restic-password: old
-     restic-new-password: new
-   ```
-3. Scale backup-restore deployment up:   
-   `kubectl -n taiga scale deployment backup-restore --replicas=1`
-4. exec into pod and execute restore pod   
-   `kubectl -n taiga exec -it backup-restore -- change-password.bb`
-5. Scale backup-restore deployment down:   
-  `kubectl -n taiga scale deployment backup-restore --replicas=0`
-6. Replace restic-password with restic-new-password in secret   
-   ```
-   kind: Secret
-   metadata:
-     name: backup-secret
-   data:
-     restic-password: new
-   ```
+1. **deployment-name**: taiga-front-deployment, taiga-gateway-deployment
+2. **deployment-namespace**: taiga

infrastructure/backup/build.py

@@ -6,7 +6,7 @@ from ddadevops import *
 name = "c4k-taiga"
 MODULE = "backup"
 PROJECT_ROOT_PATH = "../.."
-version = "1.2.0"
+version = "1.2.7-dev"
 
 
 @init

infrastructure/backup/image/resources/backup.bb

@@ -1,6 +1,5 @@
 #!/usr/bin/env bb
 (require
- '[babashka.tasks :as t]
  '[dda.backup.core :as bc]
  '[dda.backup.config :as cfg]
  '[dda.backup.restic :as rc]
@@ -25,8 +24,6 @@
   (bak/backup-file! (:file-config config))
   (bak/backup-db! (:db-config config)))
 
-(t/shell "start-maintenance.sh")
 (prepare!)
 (restic-repo-init!)
 (restic-backup!)
-(t/shell "end-maintenance.sh")

infrastructure/backup/image/resources/config.edn

@@ -0,0 +1,24 @@
+{:restic-repo {:password-file #env-or-file "RESTIC_PASSWORD_FILE"
+               :restic-repository #env-or-file "RESTIC_REPOSITORY"}
+ 
+ :file-config #merge [#ref [:restic-repo]
+                      {:backup-path "files"
+                       :execution-directory "/media"
+                       :files ["."]}]
+ 
+ :file-restore-config #merge [#ref [:restic-repo]
+                              {:backup-path "files"
+                               :restore-target-directory "/media/"
+                               :clean-up-elements ["user"]}]
+ 
+ :db-config #merge [#ref [:restic-repo] {:backup-path "pg-database"
+                                         :pg-host #env-or-file "POSTGRES_SERVICE"
+                                         :pg-port #env-or-file "POSTGRES_PORT"
+                                         :pg-db #env-or-file "POSTGRES_DB"
+                                         :pg-user #env-or-file "POSTGRES_USER"
+                                         :pg-password #env-or-file "POSTGRES_PASSWORD"}]
+
+ :aws-config {:aws-access-key-id #env-or-file "AWS_ACCESS_KEY_ID"
+              :aws-secret-access-key #env-or-file "AWS_SECRET_ACCESS_KEY"}
+
+ :dry-run {:dry-run true :debug true}}

infrastructure/backup/image/resources/restore.bb

@@ -18,7 +18,6 @@
   (pg/drop-create-db! (:db-config config))
   (rs/restore-db! (:db-config config))
   (rs/restore-file! (:file-restore-config config))
-  (t/shell "mv /var/backups/restore/* /media")
   (t/shell "chown -R 999:999 /media"))
 
 (prepare!)

package.json

@@ -2,7 +2,7 @@
     "name": "c4k-taiga",
     "description": "Generate c4k yaml for a taiga project management deployment.",
     "author": "meissa GmbH",
-    "version": "1.2.0",
+    "version": "1.2.7-SNAPSHOT",
     "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-taiga#readme",
     "repository": "https://www.npmjs.com/package/c4k-taiga",
     "license": "APACHE2",

project.clj

@@ -1,4 +1,4 @@
-(defproject org.domaindrivenarchitecture/c4k-taiga "1.2.0"
+(defproject org.domaindrivenarchitecture/c4k-taiga "1.2.7-SNAPSHOT"
   :description "taiga c4k-installation package"
   :url "https://domaindrivenarchitecture.org"
   :license {:name "Apache License, Version 2.0"

src/main/clj/dda/c4k_taiga/uberjar.clj

@@ -9,7 +9,7 @@
    "c4k-taiga"
    core/config?
    core/auth?
-   core/defaults
+   core/config-defaults
    core/config-objects
    core/auth-objects
    cmd-args))

src/main/cljc/dda/c4k_taiga/core.cljc

@@ -12,16 +12,16 @@
    [dda.c4k-common.postgres :as postgres]
    [dda.c4k-common.namespace :as ns]))
 
-(def defaults {:namespace "taiga"
-               :issuer "staging"
-               :storage-class-name "local-path"
-               :pv-storage-size-gb "5"
-               :storage-media-size "5"
-               :storage-static-size "5"
-               :storage-async-rabbitmq-size "5"
-               :storage-events-rabbitmq-size "5"
-               :public-register-enabled "false"
-               :enable-telemetry "false"})
+(def config-defaults {:namespace "taiga"
+                      :issuer "staging"
+                      :storage-class-name "local-path"
+                      :pv-storage-size-gb "5"
+                      :storage-media-size "5"
+                      :storage-static-size "5"
+                      :storage-async-rabbitmq-size "5"
+                      :storage-events-rabbitmq-size "5"
+                      :public-register-enabled "false"
+                      :enable-telemetry "false"})
 
 (def config? (s/merge
               ::backup/config
@@ -54,7 +54,7 @@
 
 (defn-spec config-objects cp/map-or-seq?
   [config config?]
-  (let [resolved-config (merge defaults config)]
+  (let [resolved-config (merge config-defaults config)]
     (cm/concat-vec
      (map yaml/to-string
           (filter
@@ -97,7 +97,7 @@
 (defn-spec auth-objects cp/map-or-seq?
   [config config?
    auth auth?]
-  (let [resolved-config (merge defaults config)]
+  (let [resolved-config (merge config-defaults config)]
     (cm/concat-vec
      (map yaml/to-string
           (filter

src/main/cljs/dda/c4k_taiga/browser.cljs

@@ -12,35 +12,39 @@
      (cm/concat-vec
       (br/generate-group
        "config"
-       (br/generate-text-area "config" "Your config.edn:" "{:fqdn \"cloud.your.domain\"
-          :issuer \"staging\"
-          :restic-repository \"s3://yourbucket/your-repo\"
-          :mon-cfg {:cluster-name \"cloud\"
-                    :cluster-stage \"test\"
-                    :cloud-url \"https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push\"}}"
-                              "5"))
+       (br/generate-text-area
+        "config" "Your config.edn:"
+        "{:fqdn \"taiga.your.domain\"
+ :issuer \"staging\"
+ :restic-repository \"s3://yourbucket/your-repo\"
+ :mon-cfg {:cluster-name \"taiga\"
+                 :cluster-stage \"test\"
+                 :grafana-cloud-url \"https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push\"}}"
+        "6"))
       (br/generate-group
        "auth"
-       (br/generate-text-area "auth" "Your auth.edn:" "{:postgres-db-user \"taiga\"
-                :postgres-db-password \"db-password\"
-                :mailer-user \"mail[at]example.com\"
-                :mailer-pw \"change-me\"
-                :django-superuser-username \"admin\"
-                :django-superuser-password \"change-me\"
-                :django-superuser-email \"mail[at]example.com\"
-                :rabbitmq-user \"user\"
-                :rabbitmq-pw \"change-me\"
-                :rabbitmq-erlang-cookie \"change-me\"
-                :taiga-secret-key \"change-me\"
-                :aws-access-key-id \"aws-id\"
-                :aws-secret-access-key \"aws-secret\"
-                :restic-password \"restic-password\"}
-                :mon-auth {:grafana-cloud-user \"your-user-id\"
-                           :grafana-cloud-password \"your-cloud-password\"}"
-                              "5"))
+       (br/generate-text-area 
+        "auth" "Your auth.edn:" 
+        "{:postgres-db-user \"taiga\"
+ :postgres-db-password \"db-password\"
+ :mailer-user \"mail[at]example.com\"
+ :mailer-pw \"change-me\"
+ :django-superuser-username \"admin\"
+ :django-superuser-password \"change-me\"
+ :django-superuser-email \"mail[at]example.com\"
+ :rabbitmq-user \"user\"
+ :rabbitmq-pw \"change-me\"
+ :rabbitmq-erlang-cookie \"change-me\"
+ :taiga-secret-key \"change-me\"
+ :aws-access-key-id \"aws-id\"
+ :aws-secret-access-key \"aws-secret\"
+ :restic-password \"restic-password\"
+ :mon-auth {:grafana-cloud-user \"your-user-id\"
+                   :grafana-cloud-password \"your-cloud-password\"}}"
+                              "16"))
       [(br/generate-br)]
       (br/generate-button "generate-button" "Generate c4k yaml")))]
-   (br/generate-output "c4k-taiga-output" "Your c4k deployment.yaml:" "15")))
+   (br/generate-output "c4k-taiga-output" "Your c4k deployment.yaml:" "25")))
 
 (defn generate-content-div
   []
@@ -66,12 +70,12 @@
                          #(do (validate-all!)
                               (-> (cm/generate-cm
                                    (br/get-content-from-element "config" :deserializer edn/read-string)
-                                    (br/get-content-from-element "auth" :deserializer edn/read-string)
-                                    core/config-defaults
-                                    core/config-objects
-                                    core/auth-objects
-                                    false
-                                    false)
+                                   (br/get-content-from-element "auth" :deserializer edn/read-string)
+                                   core/config-defaults
+                                   core/config-objects
+                                   core/auth-objects
+                                   false
+                                   false)
                                   (br/set-output!)))))
   (add-validate-listener "config")
-  (add-validate-listener "authr"))
+  (add-validate-listener "auth"))

src/main/resources/backup/backup-restore-deployment.yaml

@@ -15,7 +15,7 @@ spec:
       labels:
         app: backup-restore
         app.kubernetes.io/name: backup-restore
-        app.kubernetes.io/part-of: taiga
+        app.kubernetes.io/part-of: c4k-taiga
     spec:
       containers:
       - image: domaindrivenarchitecture/c4k-taiga-backup

src/main/resources/backup/config.yaml

@@ -5,6 +5,6 @@ metadata:
   namespace: taiga
   labels:
     app.kubernetes.io/name: backup
-    app.kubernetes.io/part-of: taiga
+    app.kubernetes.io/part-of: c4k-taiga
 data:
   restic-repository: restic-repository

src/main/resources/backup/cron.yaml

@@ -4,7 +4,7 @@ metadata:
   name: taiga-backup
   namespace: taiga
   labels:
-    app.kubernetes.part-of: taiga
+    app.kubernetes.part-of: c4k-taiga
 spec:
   schedule: "10 23 * * *"
   successfulJobsHistoryLimit: 1

src/main/resources/taiga/async-rabbitmq-deployment.yaml

@@ -48,7 +48,6 @@ spec:
             configMapKeyRef:
               name: taiga-configmap
               key: RABBITMQ_DEFAULT_VHOST
-      
       volumes:
         - name: taiga-async-rabbitmq-data
           persistentVolumeClaim:

src/main/resources/taiga/async-rabbitmq-service.yaml

@@ -6,7 +6,6 @@ metadata:
   labels:
     app.kubernetes.part-of: c4k-taiga
     app.kubernetes.io/component: taiga-async-rabbitmq
-  namespace: taiga
 spec:
   type: ClusterIP
   selector:

src/main/resources/taiga/async-service.yaml

@@ -6,7 +6,6 @@ metadata:
   labels:
     app.kubernetes.part-of: c4k-taiga
     app.kubernetes.io/component: taiga-async
-  namespace: taiga
 spec:
   type: ClusterIP
   selector:

src/main/resources/taiga/configmap.yaml

@@ -3,6 +3,8 @@ kind: ConfigMap
 metadata:
   name: taiga-configmap
   namespace: taiga
+  labels:
+    app.kubernetes.part-of: c4k-taiga
 data:
   # These environment variables will be used by taiga-back and taiga-async.
   # Database settings handled in deployment

src/main/resources/taiga/events-rabbitmq-deployment.yaml

@@ -48,7 +48,6 @@ spec:
             secretKeyRef:
               name: taiga-secret
               key: RABBITMQ_PASS
-      
       volumes:
         - name: taiga-events-rabbitmq-data
           persistentVolumeClaim:

src/main/resources/taiga/events-rabbitmq-service.yaml

@@ -6,7 +6,6 @@ metadata:
   labels:
     app.kubernetes.part-of: c4k-taiga
     app.kubernetes.io/component: taiga-events-rabbitmq
-  namespace: taiga
 spec:
   type: ClusterIP
   selector:

src/main/resources/taiga/gateway-configmap.yaml

@@ -3,6 +3,8 @@ kind: ConfigMap
 metadata:
   name: taiga-gateway-configmap
   namespace: taiga
+  labels:
+    app.kubernetes.part-of: c4k-taiga
 data:
   default.conf: |
     server {

src/main/resources/taiga/gateway-deployment.yaml

@@ -34,7 +34,6 @@ spec:
         - name: taiga-media
           mountPath: /taiga/media
           readOnly: false
-
       volumes: 
         - name: taiga-gateway-configmap
           configMap:

src/main/resources/taiga/pvc-taiga-media-data.yaml

@@ -4,8 +4,7 @@ metadata:
   name: taiga-media-data
   namespace: taiga
   labels:
-    app: taiga
-    app.kubernetes.part-of: taiga
+    app.kubernetes.part-of: c4k-taiga
 spec:
   storageClassName: REPLACEME
   accessModes:

src/main/resources/taiga/pvc-taiga-static-data.yaml

@@ -4,8 +4,7 @@ metadata:
   name: taiga-static-data
   namespace: taiga
   labels:
-    app: taiga
-    app.kubernetes.part-of: taiga
+    app.kubernetes.part-of: c4k-taiga
 spec:
   storageClassName: REPLACEME
   accessModes:

src/main/resources/taiga/rabbitmq-pvc-async.yaml

@@ -4,8 +4,7 @@ metadata:
   name: taiga-async-rabbitmq-data
   namespace: taiga
   labels:
-    app: taiga
-    app.kubernetes.part-of: taiga
+    app.kubernetes.part-of: c4k-taiga
 spec:
   storageClassName: REPLACEME
   accessModes:

src/main/resources/taiga/rabbitmq-pvc-events.yaml

@@ -4,8 +4,7 @@ metadata:
   name: taiga-events-rabbitmq-data
   namespace: taiga
   labels:
-    app: taiga
-    app.kubernetes.part-of: taiga
+    app.kubernetes.part-of: c4k-taiga
 spec:
   storageClassName: REPLACEME
   accessModes:

src/main/resources/taiga/secret.yaml

@@ -4,7 +4,7 @@ metadata:
   name: taiga-secret
   namespace: taiga
   labels:
-    app.kubernetes.part-of: taiga
+    app.kubernetes.part-of: c4k-taiga
 data:
   # Taiga settings
   TAIGA_SECRET_KEY: TAIGA_SECRET_KEY

src/test/cljc/dda/c4k_taiga/backup_test.cljc

@@ -15,7 +15,7 @@
           :metadata {:name "backup-config"
                      :namespace "taiga"
                      :labels {:app.kubernetes.io/name "backup"
-                              :app.kubernetes.io/part-of "taiga"}}
+                              :app.kubernetes.io/part-of "c4k-taiga"}}
           :data
           {:restic-repository "s3:restic-repository"}}
          (cut/generate-config {:restic-repository "s3:restic-repository"}))))

src/test/cljc/dda/c4k_taiga/core_test.cljc

@@ -14,3 +14,14 @@
 (deftest validate-valid-resources
   (is (s/valid? cut/config? (yaml/load-as-edn "taiga-test/valid-config.yaml")))
   (is (s/valid? cut/auth? (yaml/load-as-edn "taiga-test/valid-auth.yaml"))))
+
+(deftest test-whole-generation 
+  (is (= 49
+         (count 
+          (cut/config-objects 
+           (yaml/load-as-edn "taiga-test/valid-config.yaml")))))
+  (is (= 4
+         (count
+          (cut/auth-objects 
+           (yaml/load-as-edn "taiga-test/valid-config.yaml")
+           (yaml/load-as-edn "taiga-test/valid-auth.yaml"))))))

src/test/cljc/dda/c4k_taiga/taiga_test.cljc

@@ -20,7 +20,9 @@
 (deftest should-generate-configmap
   (is (= {:apiVersion "v1",
           :kind "ConfigMap",
-          :metadata {:name "taiga-configmap", :namespace "taiga"},
+          :metadata 
+          {:name "taiga-configmap", :namespace "taiga"
+           :labels {:app.kubernetes.part-of "c4k-taiga"}},
           :data
           {:ENABLE_TELEMETRY "false",
            :TAIGA_SITES_SCHEME "https",
@@ -43,7 +45,7 @@
           :metadata
           {:name "taiga-media-data",
            :namespace "taiga"
-           :labels {:app "taiga", :app.kubernetes.part-of "taiga"}},
+           :labels {:app.kubernetes.part-of "c4k-taiga"}},
           :spec
           {:storageClassName "local-path",
            :accessModes ["ReadWriteOnce"],
@@ -54,7 +56,7 @@
           :metadata
           {:name "taiga-static-data",
            :namespace "taiga"
-           :labels {:app "taiga", :app.kubernetes.part-of "taiga"}},
+           :labels {:app.kubernetes.part-of "c4k-taiga"}},
           :spec
           {:storageClassName "local-path",
            :accessModes ["ReadWriteOnce"],
@@ -67,7 +69,7 @@
           :metadata
           {:name "taiga-async-rabbitmq-data",
            :namespace "taiga"
-           :labels {:app "taiga", :app.kubernetes.part-of "taiga"}},
+           :labels {:app.kubernetes.part-of "c4k-taiga"}},
           :spec
           {:storageClassName "local-path",
            :accessModes ["ReadWriteOnce"],
@@ -80,7 +82,7 @@
           :metadata
           {:name "taiga-events-rabbitmq-data",
            :namespace "taiga"
-           :labels {:app "taiga", :app.kubernetes.part-of "taiga"}},
+           :labels {:app.kubernetes.part-of "c4k-taiga"}},
           :spec
           {:storageClassName "local-path",
            :accessModes ["ReadWriteOnce"],
@@ -91,7 +93,7 @@
   (is (= {:apiVersion "v1",
           :kind "Secret",
           :metadata
-          {:name "taiga-secret", :namespace "taiga" :labels {:app.kubernetes.part-of "taiga"}},
+          {:name "taiga-secret", :namespace "taiga" :labels {:app.kubernetes.part-of "c4k-taiga"}},
           :data
           {:TAIGA_SECRET_KEY "c29tZS1rZXk=",
            :EMAIL_HOST_USER "bWFpbGVyLXVzZXI=",