introduce new backup & change-password

README.md

@@ -98,7 +98,7 @@ For more details about our repository model see: https://repo.prod.meissa.de/mei
 
 ## License
 
-Copyright © 2024 meissa GmbH
+Copyright © 2024, 2025 meissa GmbH
 Licensed under the [Apache License, Version 2.0](LICENSE) (the "License")
 Pls. find licenses of our subcomponents [here](doc/SUBCOMPONENT_LICENSE)
 

doc/BackupAndRestore.md

@@ -7,41 +7,71 @@
 * Cloud stores files on `/var/jira`, these files are backuped. If you create a jira xml backup located in /var/jira this file will also be backed up.
 * postgres db is backed up as pgdump
 
-## Manual init the restic repository for the first time
+## Manual backup
 
-1. Scale backup-restore deployment up:   
-   `kubectl scale deployment backup-restore --replicas=1`
-1. exec into pod and execute restore pod   
-   `kubectl exec -it backup-restore -- /usr/local/bin/init.sh`
-1. Scale backup-restore deployment down:   
-  `kubectl scale deployment backup-restore --replicas=0`
-
-## Manual backup the restic repository for the first time
-
-1. Scale gateway and front deployment down:
-   `kubectl scale deployment taiga-gateway-deployment --replicas=0`
-   `kubectl scale deployment taiga-front-deployment --replicas=0`
-1. Scale backup-restore deployment up:
-   `kubectl scale deployment backup-restore --replicas=1`
-1. exec into pod and execute restore pod
-   `kubectl exec -it backup-restore -- /usr/local/bin/backup.sh`
-1. Scale backup-restore deployment down:
-   `kubectl scale deployment backup-restore --replicas=0`
-1. Scale gateway and front deployment up:
-   `kubectl scale deployment taiga-front-deployment --replicas=1`
-   `kubectl scale deployment taiga-gateway-deployment --replicas=1`
+1. Scale gateway and front deployment down:   
+   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=0`
+   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=0`
+2. Scale backup-restore deployment up:   
+   `kubectl -n taiga scale deployment backup-restore --replicas=1`
+3. exec into pod and execute restore pod   
+   `kubectl -n taiga exec -it backup-restore -- backup.bb`
+4. Scale backup-restore deployment down:   
+  `kubectl -n taiga scale deployment backup-restore --replicas=0`
+1. Scale gateway and front deployment up:   
+   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=1`
+   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=1`
 
 ## Manual restore
 
-1. Scale gateway and front deployment down:
-   `kubectl scale deployment taiga-gateway-deployment --replicas=0`
-   `kubectl scale deployment taiga-front-deployment --replicas=0`
-2. Scale backup-restore deployment up:
-   `kubectl scale deployment backup-restore --replicas=1`
-3. exec into pod and execute restore pod:
-   `kubectl exec -it backup-restore -- /usr/local/bin/restore.sh`
-4. Scale backup-restore deployment down:
-   `kubectl scale deployment backup-restore --replicas=0`
-5. Scale gateway and front deployment up:
-   `kubectl scale deployment taiga-front-deployment --replicas=1`
-   `kubectl scale deployment taiga-gateway-deployment --replicas=1`
+1. Scale gateway and front deployment down:   
+   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=0`
+   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=0`
+2. Scale backup-restore deployment up:   
+   `kubectl -n taiga scale deployment backup-restore --replicas=1`
+3. exec into pod and execute restore pod   
+   `kubectl -n taiga exec -it backup-restore -- restore.bb`
+4. Scale backup-restore deployment down:   
+  `kubectl -n taiga scale deployment backup-restore --replicas=0`
+5. Scale gateway and front deployment up:   
+   `kubectl -n taiga scale deployment taiga-gateway-deployment --replicas=1`
+   `kubectl -n taiga scale deployment taiga-front-deployment --replicas=1`
+
+## Change Password
+
+1. Check restic-new-password env is set in backup deployment   
+   ```
+   kind: Deployment
+   metadata:
+     name: backup-restore
+   spec:
+       spec:
+         containers:
+         - name: backup-app
+           env:
+           - name: RESTIC_NEW_PASSWORD_FILE
+             value: /var/run/secrets/backup-secrets/restic-new-password
+   ```
+2. Add restic-new-password to secret   
+   ```
+   kind: Secret
+   metadata:
+     name: backup-secret
+   data:
+     restic-password: old
+     restic-new-password: new
+   ```
+3. Scale backup-restore deployment up:   
+   `kubectl -n taiga scale deployment backup-restore --replicas=1`
+4. exec into pod and execute restore pod   
+   `kubectl -n taiga exec -it backup-restore -- change-password.bb`
+5. Scale backup-restore deployment down:   
+  `kubectl -n taiga scale deployment backup-restore --replicas=0`
+6. Replace restic-password with restic-new-password in secret   
+   ```
+   kind: Secret
+   metadata:
+     name: backup-secret
+   data:
+     restic-password: new
+   ```

infrastructure/backup/image/Dockerfile

@@ -1,5 +1,5 @@
-FROM domaindrivenarchitecture/dda-backup:latest
+FROM domaindrivenarchitecture/dda-backup:5.3.0
 
 # Prepare Entrypoint Script
 ADD resources /tmp
-RUN /tmp/install.sh
+RUN /tmp/install.bb

infrastructure/backup/image/resources/backup.bb

@@ -0,0 +1,32 @@
+#!/usr/bin/env bb
+(require
+ '[babashka.tasks :as t]
+ '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
+ '[dda.backup.restic :as rc]
+ '[dda.backup.postgresql :as pg]
+ '[dda.backup.backup :as bak])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+
+(defn prepare!
+  []
+  (bc/create-aws-credentials! (:aws-config config))
+  (pg/create-pg-pass! (:db-config config)))
+
+(defn restic-repo-init!
+  []
+  (rc/init! (:file-config config))
+  (rc/init! (:db-config config)))
+
+(defn restic-backup!
+  []
+  (bak/backup-file! (:file-config config))
+  (bak/backup-db! (:db-config config)))
+
+(t/shell "start-maintenance.sh")
+(prepare!)
+(restic-repo-init!)
+(restic-backup!)
+(t/shell "end-maintenance.sh")

infrastructure/backup/image/resources/backup.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-function main() {
-    file_env AWS_ACCESS_KEY_ID
-    file_env AWS_SECRET_ACCESS_KEY
-    file_env RESTIC_DAYS_TO_KEEP 30
-    file_env RESTIC_MONTHS_TO_KEEP 12
-
-    backup-db-dump
-    backup-directory "/media"
-}
-
-source /usr/local/lib/functions.sh
-source /usr/local/lib/pg-functions.sh
-source /usr/local/lib/file-functions.sh
-
-main

infrastructure/backup/image/resources/bb-backup.edn

@@ -0,0 +1,4 @@
+{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
+        orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        aero/aero {:mvn/version "1.1.6"}
+        org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

infrastructure/backup/image/resources/bb.edn

@@ -0,0 +1,3 @@
+{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
+        orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        org.domaindrivenarchitecture/dda-build {:mvn/version "0.2.0"}}}

infrastructure/backup/image/resources/change-password.bb

@@ -0,0 +1,24 @@
+#!/usr/bin/env bb
+(require
+ '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
+ '[dda.backup.restic :as rc])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+(def file-pw-change-config (merge (:file-config config)
+                                  {:new-password-file (bc/env-or-file "RESTIC_NEW_PASSWORD_FILE")}))
+(def db-pw-change-config (merge (:db-config config) 
+                                {:new-password-file (bc/env-or-file "RESTIC_NEW_PASSWORD_FILE")}))
+
+(defn prepare!
+  []
+  (bc/create-aws-credentials! (:aws-config config)))
+
+(defn change-password!
+  []
+  (rc/change-password! file-pw-change-config)
+  (rc/change-password! db-pw-change-config))
+
+(prepare!)
+(change-password!)

infrastructure/backup/image/resources/entrypoint-start-and-wait.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-
-function main() {
-    create-pg-pass
-
-    while true; do
-        sleep 1m
-    done
-}
-
-source /usr/local/lib/functions.sh
-source /usr/local/lib/pg-functions.sh
-main

infrastructure/backup/image/resources/entrypoint.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-function main() {
-    create-pg-pass
-
-	/usr/local/bin/backup.sh
-}
-
-source /usr/local/lib/functions.sh
-source /usr/local/lib/pg-functions.sh
-main

infrastructure/backup/image/resources/init.bb

@@ -0,0 +1,3 @@
+#!/usr/bin/env bb
+
+(println "initialized")

infrastructure/backup/image/resources/init.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-function main() {
-    file_env AWS_ACCESS_KEY_ID
-    file_env AWS_SECRET_ACCESS_KEY
-
-    init-database-repo
-    init-file-repo
-}
-
-source /usr/local/lib/functions.sh
-source /usr/local/lib/pg-functions.sh
-source /usr/local/lib/file-functions.sh
-main

infrastructure/backup/image/resources/install.bb

@@ -0,0 +1,18 @@
+#!/usr/bin/env bb
+
+(require
+ '[dda.image.ubuntu :as ub]
+ '[dda.image.install :as in])
+
+(ub/upgrade-system!)
+(in/install! "bb-backup.edn" :target-name "bb.edn" :mod "0440")
+(in/install! "config.edn" :mod "0440")
+(in/install! "init.bb")
+(in/install! "backup.bb")
+(in/install! "restore.bb")
+(in/install! "list-snapshots.bb")
+(in/install! "change-password.bb")
+(in/install! "restore.bb")
+(in/install! "wait.bb")
+
+(ub/cleanup-container!)

infrastructure/backup/image/resources/install.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -exo pipefail
-
-function main()
-{
-    {
-        install -m 0700 /tmp/entrypoint.sh /
-        install -m 0700 /tmp/entrypoint-start-and-wait.sh /
-
-        install -m 0700 /tmp/init.sh /usr/local/bin/
-        install -m 0700 /tmp/backup.sh /usr/local/bin/
-        install -m 0700 /tmp/restore.sh /usr/local/bin/
-        install -m 0700 /tmp/restic-snapshots.sh /usr/local/bin/
-        
-        cleanupDocker
-    } > /dev/null
-}
-
-source /tmp/install_functions_debian.sh
-DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes main

infrastructure/backup/image/resources/list-snapshots.bb

@@ -0,0 +1,20 @@
+#!/usr/bin/env bb
+(require
+ '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
+ '[dda.backup.restic :as rc])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+
+(defn prepare!
+  []
+  (bc/create-aws-credentials! (:aws-config config)))
+
+(defn list-snapshots!
+  []
+  (rc/list-snapshots! (:file-config config))
+  (rc/list-snapshots! (:db-config config)))
+
+(prepare!)
+(list-snapshots!)

infrastructure/backup/image/resources/restic-snapshots.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-function main() {
-    file_env AWS_ACCESS_KEY_ID
-    file_env AWS_SECRET_ACCESS_KEY
-
-    restic -r ${RESTIC_REPOSITORY}/files snapshots
-    restic -r ${RESTIC_REPOSITORY}/pg-database snapshots
-}
-
-source /usr/local/lib/functions.sh
-source /usr/local/lib/file-functions.sh
-
-main

infrastructure/backup/image/resources/restore.bb

@@ -0,0 +1,25 @@
+#!/usr/bin/env bb
+(require
+ '[babashka.tasks :as t]
+ '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
+ '[dda.backup.postgresql :as pg]
+ '[dda.backup.restore :as rs])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+(defn prepare!
+  []
+  (bc/create-aws-credentials! (:aws-config config))
+  (pg/create-pg-pass! (:db-config config)))
+
+(defn restic-restore!
+  []
+  (pg/drop-create-db! (:db-config config))
+  (rs/restore-db! (:db-config config))
+  (rs/restore-file! (:file-restore-config config))
+  (t/shell "mv /var/backups/restore/* /media")
+  (t/shell "chown -R 999:999 /media"))
+
+(prepare!)
+(restic-restore!)

infrastructure/backup/image/resources/restore.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-
-function main() {
-
-    file_env AWS_ACCESS_KEY_ID
-    file_env AWS_SECRET_ACCESS_KEY
-
-    file_env POSTGRES_DB
-    file_env POSTGRES_PASSWORD
-    file_env POSTGRES_USER
-
-    # Restore latest snapshot into /var/backups/restore
-    restore-directory '/var/backups/restore'
-    
-    mv /var/backups/restore/* /media
-
-    # adjust file permissions for the taiga user
-    chown -R 999:999 /media
-
-    # Restore db
-    drop-create-db
-    restore-db
-}
-
-source /usr/local/lib/functions.sh
-source /usr/local/lib/pg-functions.sh
-source /usr/local/lib/file-functions.sh
-
-main

infrastructure/backup/image/resources/wait.bb

@@ -0,0 +1,19 @@
+#!/usr/bin/env bb
+(require
+ '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
+ '[dda.backup.postgresql :as pg])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+(defn prepare!
+  []
+  (bc/create-aws-credentials! (:aws-config config))
+  (pg/create-pg-pass! (:db-config config)))
+
+(defn wait! []
+  (while true
+    (Thread/sleep 1000)))
+
+(prepare!)
+(wait!)

infrastructure/backup/test/Dockerfile

@@ -0,0 +1,4 @@
+FROM c4k-taiga-backup:latest
+
+ADD resources /tmp/
+RUN RESTIC_PASSWORD_FILE=/tmp/file_password RESTIC_NEW_PASSWORD_FILE=/tmp/new_file_password RESTIC_REPOSITORY=restic-repo POSTGRES_SERVICE=dummy POSTGRES_PORT=dummy POSTGRES_DB=dummy POSTGRES_USER=dummy POSTGRES_PASSWORD=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /tmp/test.bb

infrastructure/backup/test/resources/bb.edn

@@ -0,0 +1,4 @@
+{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
+        orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        aero/aero {:mvn/version "1.1.6"}
+        org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

infrastructure/backup/test/resources/file_password

@@ -0,0 +1 @@
+oldPassword

infrastructure/backup/test/resources/new_file_password

@@ -0,0 +1 @@
+newPassword

infrastructure/backup/test/resources/test.bb

@@ -0,0 +1,59 @@
+#!/usr/bin/env bb
+(require '[babashka.tasks :as tasks]
+         '[dda.backup.core :as bc]
+         '[dda.backup.config :as cfg]
+         '[dda.backup.restic :as rc]
+         '[dda.backup.postgresql :as pg]
+         '[dda.backup.backup :as bak]
+         '[dda.backup.restore :as rs])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+(def file-pw-change-config (merge (:file-config config)
+                                  {:new-password-file (bc/env-or-file "RESTIC_NEW_PASSWORD_FILE")}))
+
+(defn prepare!
+  []
+  (tasks/shell "mkdir" "-p" "/var/backups/")
+  (spit "/var/backups/file" "I was here")
+  (tasks/shell "mkdir" "-p" "/var/restore"))
+
+(defn restic-repo-init!
+  []
+  (rc/init! (:file-config config))
+  (rc/init! (merge (:db-config config)
+                   (:dry-run config))))
+
+(defn restic-backup!
+  []
+  (bak/backup-file! (:file-config config))
+  (bak/backup-db! (merge (:db-config config)
+                         (:dry-run config))))
+
+(defn list-snapshots!
+  []
+  (rc/list-snapshots! (:file-config config))
+  (rc/list-snapshots! (merge (:db-config config)
+                             (:dry-run config))))
+
+
+(defn restic-restore!
+  []
+  (pg/drop-create-db! (merge (:db-config config)
+                             (:dry-run config)))
+  (rs/restore-db! (merge (:db-config config)
+                         (:dry-run config)))
+  (rs/restore-file! (merge (:file-restore-config config)
+                           (:dry-run config))))
+
+(defn change-password!
+  []
+  (println "change-password!")
+  (rc/change-password! file-pw-change-config))
+
+(prepare!)
+(restic-repo-init!)
+(restic-backup!)
+(list-snapshots!)
+(restic-restore!)
+(change-password!)

src/main/resources/backup/backup-restore-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       - image: domaindrivenarchitecture/c4k-taiga-backup
         name: backup-app
         imagePullPolicy: IfNotPresent
-        command: ["/entrypoint-start-and-wait.sh"]
+        command: ["wait.bb"]
         env:
         - name: POSTGRES_USER
           valueFrom:

src/main/resources/backup/cron.yaml

@@ -17,7 +17,7 @@ spec:
           - name: backup-app
             image: domaindrivenarchitecture/c4k-taiga-backup
             imagePullPolicy: IfNotPresent
-            command: ["/entrypoint.sh"]
+            command: ["backup.bb"]
             env:
             - name: POSTGRES_USER
               valueFrom:
@@ -53,8 +53,6 @@ spec:
                   key: restic-repository
             - name: RESTIC_PASSWORD_FILE
               value: /var/run/secrets/backup-secrets/restic-password
-            - name: CERTIFICATE_FILE
-              value: ""
             volumeMounts:
             - name: taiga-media
               mountPath: /media