infra work

doc/BackupAndRestore.md

@@ -7,35 +7,67 @@
 * Forgejo stores files in `/data/gitea` and `/data/git/repositories`, these files are backed up. 
 * The postgres db is also backed up
 
-## Manual init the restic repository for the first time
+## Manual backup
 
-1. apply backup-and-restore pod:   
+1. Scale down forgejo deployment:   
+   `kubectl -n forgejo scale deployment forgejo --replicas=0`
+2. apply backup-and-restore pod:   
   `kubectl -n forgejo scale deployment backup-restore --replicas=1`
-2. exec into pod and execute restore pod (press tab to get your exact pod name)   
+3. exec into pod and execute backup pod (press tab to get your exact pod name)   
-   `kubectl -n forgejo exec -it backup-restore-... -- /usr/local/bin/init.bb`
-3. remove backup-and-restore pod:   
-   `kubectl -n forgejo scale deployment backup-restore --replicas=0`
-
-
-## Manual backup the restic repository for the first time
-
-1. apply backup-and-restore pod:   
-  `kubectl -n forgejo scale deployment backup-restore --replicas=1`
-2. exec into pod and execute backup pod (press tab to get your exact pod name)   
    `kubectl -n forgejo exec -it backup-restore-... -- /usr/local/bin/backup.bb`
-3. remove backup-and-restore pod:   
+4. remove backup-and-restore pod:   
    `kubectl -n forgejo scale deployment backup-restore --replicas=0`
-
+5. Scale up forgejo deployment:   
+   `kubectl -n forgejo scale deployment forgejo --replicas=1`
 
 ## Manual restore
 
-1. apply backup-and-restore pod:   
+1. Scale down forgejo deployment:
-  `kubectl -n forgejo scale deployment backup-restore --replicas=1`
-2. Scale down forgejo deployment:
    `kubectl -n forgejo scale deployment forgejo --replicas=0`
+2. apply backup-and-restore pod:   
+  `kubectl -n forgejo scale deployment backup-restore --replicas=1`
 3. exec into pod and execute restore pod (press tab to get your exact pod name)   
    `kubectl -n forgejo exec -it backup-restore-... -- /usr/local/bin/restore.bb`
 4. Start forgejo again:
    `kubectl -n forgejo scale deployment forgejo --replicas=1`
 5. remove backup-and-restore pod:   
    `kubectl -n forgejo scale deployment backup-restore --replicas=0`
+
+## Change Password
+
+1. Check restic-new-password env is set in backup deployment   
+   ```
+   kind: Deployment
+   metadata:
+     name: backup-restore
+   spec:
+       spec:
+         containers:
+         - name: backup-app
+           env:
+           - name: RESTIC_NEW_PASSWORD_FILE
+             value: /var/run/secrets/backup-secrets/restic-new-password
+   ```
+2. Add restic-new-password to secret   
+   ```
+   kind: Secret
+   metadata:
+     name: backup-secret
+   data:
+     restic-password: old
+     restic-new-password: new
+   ```
+3. Scale backup-restore deployment up:   
+   `kubectl -n nextcloud scale deployment backup-restore --replicas=1`
+4. exec into pod and execute restore pod   
+   `kubectl -n nextcloud exec -it backup-restore -- change-password.bb`
+5. Scale backup-restore deployment down:   
+  `kubectl -n nextcloud scale deployment backup-restore --replicas=0`
+6. Replace restic-password with restic-new-password in secret   
+   ```
+   kind: Secret
+   metadata:
+     name: backup-secret
+   data:
+     restic-password: new
+   ```

infrastructure/backup/image/Dockerfile

@@ -1,4 +1,5 @@
-FROM domaindrivenarchitecture/dda-backup:latest
+FROM domaindrivenarchitecture/dda-backup:5.3.0
 
 ADD resources /tmp
 RUN /tmp/install.bb
+RUN init.bb

infrastructure/backup/image/resources/backup.bb

@@ -2,44 +2,27 @@
 
 (require
  '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
  '[dda.backup.restic :as rc]
  '[dda.backup.postgresql :as pg]
  '[dda.backup.backup :as bak])
 
-(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
-                  :restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
-
-(def file-config (merge restic-repo {:backup-path "files"
-                                     :execution-directory "/var/backups/"
-                                     :files ["gitea/" "git/repositories/"]}))
-
-
-(def db-config (merge restic-repo {:backup-path "pg-database"
-                                   :pg-host (bc/env-or-file "POSTGRES_SERVICE")
-                                   :pg-port (bc/env-or-file "POSTGRES_PORT")
-                                   :pg-db (bc/env-or-file "POSTGRES_DB")
-                                   :pg-user (bc/env-or-file "POSTGRES_USER")
-                                   :pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
-
-(def aws-config {:aws-access-key-id (bc/env-or-file "AWS_ACCESS_KEY_ID")
-                 :aws-secret-access-key (bc/env-or-file "AWS_SECRET_ACCESS_KEY")})
-
-(def dry-run {:dry-run true :debug true})
 
 (defn prepare!
   []
-  (bc/create-aws-credentials! aws-config)
+  (bc/create-aws-credentials! (:aws-config config))
-  (pg/create-pg-pass! db-config))
+  (pg/create-pg-pass! (:db-config config)))
 
 (defn restic-repo-init!
   []
-  (rc/init! file-config)
+  (rc/init! (:file-config config))
-  (rc/init! db-config))
+  (rc/init! (:db-config config)))
 
 (defn restic-backup!
   []
-  (bak/backup-file! file-config)
+  (bak/backup-file! (:file-config config))
-  (bak/backup-db! db-config))
+  (bak/backup-db! (:db-config config)))
 
 (prepare!)
 (restic-repo-init!)

infrastructure/backup/image/resources/bb-backup.edn

@@ -1,3 +1,4 @@
 {:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
         orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        aero/aero {:mvn/version "1.1.6"}
         org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

infrastructure/backup/image/resources/bb.edn

@@ -1,3 +1,3 @@
 {:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
         orchestra/orchestra {:mvn/version "2021.01.01-1"}
-        org.domaindrivenarchitecture/dda-build {:mvn/version "0.1.1-SNAPSHOT"}}}
+        org.domaindrivenarchitecture/dda-build {:mvn/version "0.2.0"}}}

infrastructure/backup/image/resources/change-password.bb

@@ -0,0 +1,24 @@
+#!/usr/bin/env bb
+(require
+ '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
+ '[dda.backup.restic :as rc])
+
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
+
+(def file-pw-change-config (merge (:file-config config)
+                                  {:new-password-file (bc/env-or-file "RESTIC_NEW_PASSWORD_FILE")}))
+(def db-pw-change-config (merge (:db-config config) 
+                                {:new-password-file (bc/env-or-file "RESTIC_NEW_PASSWORD_FILE")}))
+
+(defn prepare!
+  []
+  (bc/create-aws-credentials! (:aws-config config)))
+
+(defn change-password!
+  []
+  (rc/change-password! file-pw-change-config)
+  (rc/change-password! db-pw-change-config))
+
+(prepare!)
+(change-password!)

infrastructure/backup/image/resources/install.bb

@@ -7,8 +7,11 @@
 
 (ub/upgrade-system!)
 (in/install! "bb-backup.edn" :target-name "bb.edn" :mod "0400")
+(in/install! "config.edn" :mod "0440")
+(in/install! "init.bb")
 (in/install! "backup.bb")
 (in/install! "restore.bb")
 (in/install! "list-snapshots.bb")
+(in/install! "change-password.bb")
 (in/install! "wait.bb")
 (ub/cleanup-container!)

infrastructure/backup/image/resources/list-snapshots.bb

@@ -2,27 +2,19 @@
 
 (require
  '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
  '[dda.backup.restic :as rc])
 
-(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
-                  :restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
-
-(def file-config (merge restic-repo {:backup-path "files"}))
-
-
-(def db-config (merge restic-repo {:backup-path "pg-database"}))
-
-(def aws-config {:aws-access-key-id (bc/env-or-file "AWS_ACCESS_KEY_ID")
-                 :aws-secret-access-key (bc/env-or-file "AWS_SECRET_ACCESS_KEY")})
 
 (defn prepare!
   []
-  (bc/create-aws-credentials! aws-config))
+  (bc/create-aws-credentials! (:aws-config config)))
 
 (defn list-snapshots!
   []
-  (rc/list-snapshots! file-config)
+  (rc/list-snapshots! (:file-config config))
-  (rc/list-snapshots! db-config))
+  (rc/list-snapshots! (:db-config config)))
 
 (prepare!)
 (list-snapshots!)

infrastructure/backup/image/resources/restore.bb

@@ -2,34 +2,28 @@
 
 (require '[babashka.tasks :as tasks]
          '[dda.backup.core :as bc]
+         '[dda.backup.config :as cfg]
          '[dda.backup.postgresql :as pg]
          '[dda.backup.restore :as rs])
 
-(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
-                  :restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
 
-(def file-config (merge restic-repo {:backup-path "files"
+(def file-config (merge
+                  (:restic-repo config)
+                  {:backup-path "files"
                    :restore-target-directory "/var/backups/restore"
                    :snapshot-id "latest"}))
 
+(def db-config (merge (:db-config config)
+                      {:snapshot-id "latest"}))
 
-(def db-config (merge restic-repo {:backup-path "pg-database"
-                                   :pg-host (bc/env-or-file "POSTGRES_SERVICE")
-                                   :pg-port (bc/env-or-file "POSTGRES_PORT")
-                                   :pg-db (bc/env-or-file "POSTGRES_DB")
-                                   :pg-user (bc/env-or-file "POSTGRES_USER")
-                                   :pg-password (bc/env-or-file "POSTGRES_PASSWORD")
-                                   :snapshot-id "latest"}))
-
-(def aws-config {:aws-access-key-id (bc/env-or-file "AWS_ACCESS_KEY_ID")
-                 :aws-secret-access-key (bc/env-or-file "AWS_SECRET_ACCESS_KEY")})
 
 (def dry-run {:dry-run true :debug true})
 
 (defn prepare!
   []
-  (pg/create-pg-pass! db-config)
+  (bc/create-aws-credentials! (:aws-config config))
-  (bc/create-aws-credentials! aws-config))
+  (pg/create-pg-pass! db-config))
 
 (defn restic-restore!
   []
@@ -39,8 +33,8 @@
   (tasks/shell ["mv" "/var/backups/restore/gitea" "/var/backups/"])
   (tasks/shell ["mv" "/var/backups/restore/git/repositories" "/var/backups/git/"])
   (tasks/shell ["chown" "-R" "1000:1000" "/var/backups"])
-  (pg/drop-create-db! (merge db-config {:debug true}))
+  (pg/drop-create-db! db-config)
-  (rs/restore-db! (merge db-config {:debug true})))
+  (rs/restore-db! db-config))
 
 (prepare!)
 (restic-restore!)

infrastructure/backup/image/resources/wait.bb

@@ -1,23 +1,15 @@
 #!/usr/bin/env bb
-
 (require
  '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
  '[dda.backup.postgresql :as pg])
 
-
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
-(def restic-repo {:password-file (bc/env-or-file "RESTIC_PASSWORD_FILE")
-                  :restic-repository (bc/env-or-file "RESTIC_REPOSITORY")})
-
-(def db-config (merge restic-repo {:backup-path "pg-database"
-                                   :pg-host (bc/env-or-file "POSTGRES_SERVICE")
-                                   :pg-port (bc/env-or-file "POSTGRES_PORT")
-                                   :pg-db (bc/env-or-file "POSTGRES_DB")
-                                   :pg-user (bc/env-or-file "POSTGRES_USER")
-                                   :pg-password (bc/env-or-file "POSTGRES_PASSWORD")}))
 
 (defn prepare!
   []
-  (pg/create-pg-pass! db-config))
+  (bc/create-aws-credentials! (:aws-config config))
+  (pg/create-pg-pass! (:db-config config)))
 
 (defn wait! []
   (while true

infrastructure/backup/test/Dockerfile

@@ -1,4 +1,4 @@
 FROM c4k-forgejo-backup:latest
 
 ADD resources /tmp/
-RUN ENV_PASSWORD=env-password FILE_PASSWORD_FILE=/tmp/file_password /tmp/test.bb
+RUN RESTIC_PASSWORD_FILE=/tmp/file_password RESTIC_NEW_PASSWORD_FILE=/tmp/new_file_password RESTIC_REPOSITORY=/tmp/restic-repo POSTGRES_SERVICE=dummy POSTGRES_PORT=dummy POSTGRES_DB=dummy POSTGRES_USER=dummy POSTGRES_PASSWORD=dummy AWS_ACCESS_KEY_ID=dummy AWS_SECRET_ACCESS_KEY=dummy /tmp/test.bb

infrastructure/backup/test/resources/bb.edn

@@ -1,3 +1,4 @@
 {:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
         orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        aero/aero {:mvn/version "1.1.6"}
         org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

infrastructure/backup/test/resources/file_password

@@ -0,0 +1 @@
+oldPassword

infrastructure/backup/test/resources/new_file_password

@@ -0,0 +1 @@
+newPassword

infrastructure/backup/test/resources/test.bb

@@ -2,61 +2,66 @@
 
 (require '[babashka.tasks :as tasks]
          '[dda.backup.core :as bc]
+         '[dda.backup.config :as cfg]
          '[dda.backup.restic :as rc]
          '[dda.backup.postgresql :as pg]
          '[dda.backup.backup :as bak]
          '[dda.backup.restore :as rs])
 
-(def restic-repo {:password-file "restic-pwd"
+(def config (cfg/read-config "/usr/local/bin/config.edn"))
-                  :restic-repository "restic-repo"})
 
-(def file-config (merge restic-repo {:backup-path "files"
+(def file-pw-change-config (merge (:file-config config)
-                                     :files ["test-backup"]
+                                  {:new-password-file (bc/env-or-file "RESTIC_NEW_PASSWORD_FILE")}))
-                                     :restore-target-directory "test-restore"}))
 
-
+(def file-restore-config (merge
-(def db-config (merge restic-repo {:backup-path "db"
+                          (:restic-repo config)
-                                   :pg-db "mydb"
+                          {:backup-path "files"
-                                   :pg-user "user"
+                           :restore-target-directory "/var/backups/restore"
-                                   :pg-password "password"}))
+                           :snapshot-id "latest"}))
-
-(def dry-run {:dry-run true :debug true})
 
 (defn prepare!
   []
-  (spit "/tmp/file_password" "file-password")
+  (tasks/shell "mkdir" "-p" "/tmp/restic-repo")
-  (println (bc/env-or-file "FILE_PASSWORD"))
+  (tasks/shell "mkdir" "-p" "/var/backups/gitea")
-  (println (bc/env-or-file "ENV_PASSWORD"))
+  (tasks/shell "mkdir" "-p" "/var/backups/git/repositories")
-  (spit "restic-pwd" "ThePassword")
+  (spit "/var/backups/gitea/file" "I was here")
-  (tasks/shell "mkdir" "-p" "test-backup")
+  (tasks/shell "mkdir" "-p" "test-restore"))
-  (spit "test-backup/file" "I was here")
-  (tasks/shell "mkdir" "-p" "test-restore")
-  (pg/create-pg-pass! db-config))
 
 (defn restic-repo-init!
   []
-  (rc/init! file-config)
+  (rc/init! (:file-config config))
-  (rc/init! (merge db-config dry-run)))
+  (rc/init! (merge (:db-config config)
+                   (:dry-run config))))
 
 (defn restic-backup!
   []
-  (bak/backup-file! file-config)
+  (bak/backup-file! (:file-config config))
-  (bak/backup-db! (merge db-config dry-run)))
+  (bak/backup-db! (merge (:db-config config)
+                         (:dry-run config))))
 
 (defn list-snapshots!
   []
-  (rc/list-snapshots! file-config)
+  (rc/list-snapshots! (:file-config config))
-  (rc/list-snapshots! (merge db-config dry-run)))
+  (rc/list-snapshots! (merge (:db-config config)
+                             (:dry-run config))))
 
 
 (defn restic-restore!
   []
-  (rs/restore-file! file-config)
+  (pg/drop-create-db! (merge (:db-config config)
-  (pg/drop-create-db! (merge db-config dry-run))
+                             (:dry-run config)))
-  (rs/restore-db! (merge db-config dry-run)))
+  (rs/restore-db! (merge (:db-config config)
+                         (:dry-run config)))
+  (rs/restore-file! file-restore-config))
+
+(defn change-password!
+  []
+  (println "change-password!")
+  (rc/change-password! file-pw-change-config))
 
 (prepare!)
 (restic-repo-init!)
 (restic-backup!)
 (list-snapshots!)
 (restic-restore!)
+(change-password!)