From 09b005ecd8356c3bd60e99d9007a248cf6c39433 Mon Sep 17 00:00:00 2001
From: Michael Jerger <michael.jerger@meissa-gmbh.de>
Date: Mon, 13 Jan 2025 12:49:19 +0100
Subject: [PATCH] feat: add config load feature
---
deps.edn | 2 +-
infrastructure/backup/image/Dockerfile | 3 +-
.../image/resources2/change-password.bb | 34 ---------
.../backup/image/resources2/check.bb | 23 ------
.../backup/image/resources2/config.edn | 24 ++++++
.../backup/image/resources2/file_new_password | 1 +
.../backup/image/resources2/file_password | 1 +
.../backup/image/resources2/test.bb | 68 ++++++++++-------
infrastructure/backup/test/Dockerfile | 5 +-
infrastructure/backup/test/resources/bb.edn | 3 -
.../backup/test/resources/config.edn | 24 ++++++
.../backup/test/resources/file_new_password | 1 +
.../backup/test/resources/file_password | 1 +
.../backup/test/resources/install-test.bb | 29 -------
infrastructure/backup/test/resources/spec.yml | 7 --
infrastructure/backup/test/resources/test.bb | 76 +++++--------------
src/dda/backup/config.clj | 22 ++++++
17 files changed, 139 insertions(+), 185 deletions(-)
delete mode 100755 infrastructure/backup/image/resources2/change-password.bb
delete mode 100755 infrastructure/backup/image/resources2/check.bb
create mode 100644 infrastructure/backup/image/resources2/config.edn
create mode 100644 infrastructure/backup/image/resources2/file_new_password
create mode 100644 infrastructure/backup/image/resources2/file_password
delete mode 100644 infrastructure/backup/test/resources/bb.edn
create mode 100644 infrastructure/backup/test/resources/config.edn
create mode 100644 infrastructure/backup/test/resources/file_new_password
create mode 100644 infrastructure/backup/test/resources/file_password
delete mode 100755 infrastructure/backup/test/resources/install-test.bb
delete mode 100644 infrastructure/backup/test/resources/spec.yml
create mode 100644 src/dda/backup/config.clj
diff --git a/deps.edn b/deps.edn
index 1c60088..0e8b3c2 100644
--- a/deps.edn
+++ b/deps.edn
@@ -9,9 +9,9 @@
;; ---------------------------------------------------------
:deps
{;; Application
- org.clojure/clojure {:mvn/version "1.11.4"}
org.clojure/spec.alpha {:mvn/version "0.5.238"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
+ aero/aero {:mvn/version "1.1.6"}
cheshire/cheshire {:mvn/version "5.13.0"}
com.widdindustries/cljc.java-time {:mvn/version "0.1.21"}}
;; ---------------------------------------------------------
diff --git a/infrastructure/backup/image/Dockerfile b/infrastructure/backup/image/Dockerfile
index 5954507..26c0145 100644
--- a/infrastructure/backup/image/Dockerfile
+++ b/infrastructure/backup/image/Dockerfile
@@ -7,5 +7,4 @@ ADD local/ /usr/local/lib/dda-backup
RUN init-bb.bb
#ADD resources2 /tmp/
#RUN install -m 0700 -o root -g root /tmp/test.bb /usr/local/bin/
-#RUN install -m 0700 -o root -g root /tmp/check.bb /usr/local/bin/
-#RUN test.bb
+#RUN FILE_PASSWORD_FILE=/tmp/file_password test.bb
diff --git a/infrastructure/backup/image/resources2/change-password.bb b/infrastructure/backup/image/resources2/change-password.bb
deleted file mode 100755
index ab0d700..0000000
--- a/infrastructure/backup/image/resources2/change-password.bb
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env bb
-
-(require '[dda.backup.cred-rot :as cr])
-
-(def restic-repo {:password-file "/restic-pwd"
- :restic-repository "/restic-repo"
- :debug true})
-
-(def file-config (merge restic-repo {:backup-path "files"
- :files ["/test-backup"]
- :restore-target-directory "/test-restore"}))
-
-(def cred-config (merge file-config
- {:restic-repository "/restic-repo/files"
- :new-password-config {:new-password-file "/new-pw"
- :valid-from "2024-12-17 00:00:00"}}))
-
-
-(def dry-run {:dry-run true :debug true})
-
-(defn prepare!
- []
- (spit "/restic-pwd" "ThePassword")
- (spit "/new-pw" "newPassword"))
-
-
-(defn change-password!
- []
- (println "change-password!")
- (cr/change-password! cred-config))
-
-
-(prepare!)
-(change-password!)
diff --git a/infrastructure/backup/image/resources2/check.bb b/infrastructure/backup/image/resources2/check.bb
deleted file mode 100755
index 183feec..0000000
--- a/infrastructure/backup/image/resources2/check.bb
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bb
-
-(require '[dda.backup.restic :as rc])
-
-(def restic-repo {:password-file "/restic-pwd"
- :restic-repository "/restic-repo"
- :debug true})
-
-(def file-config (merge restic-repo {:backup-path "files"
- :files ["/test-backup"]
- :restore-target-directory "/test-restore"}))
-
-(def cred-config (merge file-config {:new-password-file "new-pw"}))
-
-
-(def dry-run {:dry-run true :debug true})
-
-(defn restic-repo-check
- []
- (println "restic-repo-check")
- (println (rc/check file-config)))
-
-(restic-repo-check)
\ No newline at end of file
diff --git a/infrastructure/backup/image/resources2/config.edn b/infrastructure/backup/image/resources2/config.edn
new file mode 100644
index 0000000..423fdb2
--- /dev/null
+++ b/infrastructure/backup/image/resources2/config.edn
@@ -0,0 +1,24 @@
+{:restic-repo {:password-file #env-or-file "FILE_PASSWORD_FILE"
+ :restic-repository "/restic-repo"
+ :debug true}
+ :file-config #merge [#ref [:restic-repo]
+ {:new-password-file "/tmp/file_new_password"
+ :backup-path "files"
+ :execution-directory "/var/backups/"
+ :files ["test-backup"]
+ :restore-target-directory "test-restore"}]
+ :file-config-with-new #merge [#ref [:file-config]
+ {:password-file "/tmp/file_new_password"}]
+ :db-config #merge [#ref [:restic-repo]
+ {:new-password-file "/tmp/file_new_password"
+ :backup-path "db"
+ :pg-db "mydb"
+ :pg-user "user"
+ :pg-password "password"}]
+ :db-roles-config #merge [#ref [:restic-repo]
+ {:new-password-file "/tmp/file_new_password"
+ :backup-path "db-roles"
+ :pg-db "mydb"
+ :pg-user "user"
+ :pg-password "password"}]
+ :dry-run {:dry-run true :debug true}}
\ No newline at end of file
diff --git a/infrastructure/backup/image/resources2/file_new_password b/infrastructure/backup/image/resources2/file_new_password
new file mode 100644
index 0000000..3e5126c
--- /dev/null
+++ b/infrastructure/backup/image/resources2/file_new_password
@@ -0,0 +1 @@
+new
\ No newline at end of file
diff --git a/infrastructure/backup/image/resources2/file_password b/infrastructure/backup/image/resources2/file_password
new file mode 100644
index 0000000..489ce0f
--- /dev/null
+++ b/infrastructure/backup/image/resources2/file_password
@@ -0,0 +1 @@
+old
\ No newline at end of file
diff --git a/infrastructure/backup/image/resources2/test.bb b/infrastructure/backup/image/resources2/test.bb
index d6278a9..b6adea7 100755
--- a/infrastructure/backup/image/resources2/test.bb
+++ b/infrastructure/backup/image/resources2/test.bb
@@ -1,58 +1,65 @@
#!/usr/bin/env bb
(require '[babashka.tasks :as tasks]
+ '[dda.backup.config :as cfg]
'[dda.backup.restic :as rc]
+ '[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs])
-(def restic-repo {:password-file "/restic-pwd"
- :new-password-file "/new-restic-pwd"
- :restic-repository "/restic-repo"
- :debug true})
-
-(def file-config (merge restic-repo {:backup-path "files"
- :execution-directory "/var/backups/"
- :files ["test-backup"]
- :restore-target-directory "test-restore"}))
-
-(def dry-run {:dry-run true :debug true})
+(def config (cfg/read-config "/tmp/config.edn"))
(defn prepare!
[]
- (spit "/tmp/file_password" "file-password")
-
- (spit "/restic-pwd" "oldPassword")
- (spit "/new-restic-pwd" "newPassword")
-
+ (println config)
(tasks/shell "mkdir" "-p" "/var/backups/test-backup")
(spit "/var/backups/test-backup/file" "I was here")
- (tasks/shell "mkdir" "-p" "/var/backups/test-restore"))
+ (tasks/shell "mkdir" "-p" "/var/backups/test-restore")
+ (pg/create-pg-pass! (:db-config config)))
(defn restic-repo-init!
[]
- (println "restic-repo-init!")
- (rc/init! file-config))
+ (println "\nrestic-repo-init!")
+ (rc/init! (:file-config config))
+ (rc/init! (merge (:db-config config) (:dry-run config)))
+ (rc/init! (merge (:db-roles-config config) (:dry-run config))))
(defn restic-backup!
[]
- (println "restic-backup!")
- (bak/backup-file! file-config))
+ (println "\nrestic-backup!")
+ (bak/backup-file! (:file-config config))
+ (bak/backup-db-roles! (merge (:db-roles-config config) (:dry-run config)))
+ (bak/backup-db! (merge (:db-config config) (:dry-run config))))
(defn list-snapshots!
[]
- (println "list-snapshots!")
- (rc/list-snapshots! file-config))
-
+ (println "\nlist-snapshots!")
+ (rc/list-snapshots! (:file-config config))
+ (rc/list-snapshots! (merge (:db-roles-config config) (:dry-run config)))
+ (rc/list-snapshots! (merge (:db-config config) (:dry-run config))))
(defn restic-restore!
[]
- (println "restic-restore!")
- (rs/restore-file! file-config))
+ (println "\nrestic-restore!")
+ (rs/restore-file! (:file-config config))
+ (pg/drop-create-db! (merge (:db-config config) (:dry-run config)))
+ (rs/restore-db-roles! (merge (:db-roles-config config) (:dry-run config)))
+ (rs/restore-db! (merge (:db-config config) (:dry-run config))))
(defn change-password!
[]
- (println "change-password!")
- (rc/change-password! file-config))
+ (println "\nchange-password!")
+ (rc/change-password! (:file-config config)))
+
+(defn restic-backup-with-new!
+ []
+ (println "\nrestic-backup with new!")
+ (bak/backup-file! (:file-config-with-new config)))
+
+(defn list-snapshots-with-new!
+ []
+ (println "\nlist-snapshots with new!")
+ (rc/list-snapshots! (:file-config-with-new config)))
(prepare!)
@@ -61,3 +68,8 @@
(list-snapshots!)
(restic-restore!)
(change-password!)
+(restic-backup!)
+(list-snapshots!)
+(restic-restore!)
+(restic-backup-with-new!)
+(list-snapshots-with-new!)
diff --git a/infrastructure/backup/test/Dockerfile b/infrastructure/backup/test/Dockerfile
index 0a66d92..59db2b7 100644
--- a/infrastructure/backup/test/Dockerfile
+++ b/infrastructure/backup/test/Dockerfile
@@ -1,6 +1,5 @@
FROM dda-backup:latest
-# install it
-#ADD local/ /usr/local/lib/dda-backup
ADD resources /tmp/
-RUN ENV_PASSWORD=env-password FILE_PASSWORD_FILE=/tmp/file_password /tmp/test.bb
+RUN install -m 0700 -o root -g root /tmp/test.bb /usr/local/bin/
+RUN FILE_PASSWORD_FILE=/tmp/file_password test.bb
diff --git a/infrastructure/backup/test/resources/bb.edn b/infrastructure/backup/test/resources/bb.edn
deleted file mode 100644
index 1a7297a..0000000
--- a/infrastructure/backup/test/resources/bb.edn
+++ /dev/null
@@ -1,3 +0,0 @@
-{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
- orchestra/orchestra {:mvn/version "2021.01.01-1"}
- org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}
diff --git a/infrastructure/backup/test/resources/config.edn b/infrastructure/backup/test/resources/config.edn
new file mode 100644
index 0000000..423fdb2
--- /dev/null
+++ b/infrastructure/backup/test/resources/config.edn
@@ -0,0 +1,24 @@
+{:restic-repo {:password-file #env-or-file "FILE_PASSWORD_FILE"
+ :restic-repository "/restic-repo"
+ :debug true}
+ :file-config #merge [#ref [:restic-repo]
+ {:new-password-file "/tmp/file_new_password"
+ :backup-path "files"
+ :execution-directory "/var/backups/"
+ :files ["test-backup"]
+ :restore-target-directory "test-restore"}]
+ :file-config-with-new #merge [#ref [:file-config]
+ {:password-file "/tmp/file_new_password"}]
+ :db-config #merge [#ref [:restic-repo]
+ {:new-password-file "/tmp/file_new_password"
+ :backup-path "db"
+ :pg-db "mydb"
+ :pg-user "user"
+ :pg-password "password"}]
+ :db-roles-config #merge [#ref [:restic-repo]
+ {:new-password-file "/tmp/file_new_password"
+ :backup-path "db-roles"
+ :pg-db "mydb"
+ :pg-user "user"
+ :pg-password "password"}]
+ :dry-run {:dry-run true :debug true}}
\ No newline at end of file
diff --git a/infrastructure/backup/test/resources/file_new_password b/infrastructure/backup/test/resources/file_new_password
new file mode 100644
index 0000000..3e5126c
--- /dev/null
+++ b/infrastructure/backup/test/resources/file_new_password
@@ -0,0 +1 @@
+new
\ No newline at end of file
diff --git a/infrastructure/backup/test/resources/file_password b/infrastructure/backup/test/resources/file_password
new file mode 100644
index 0000000..489ce0f
--- /dev/null
+++ b/infrastructure/backup/test/resources/file_password
@@ -0,0 +1 @@
+old
\ No newline at end of file
diff --git a/infrastructure/backup/test/resources/install-test.bb b/infrastructure/backup/test/resources/install-test.bb
deleted file mode 100755
index ecbda5c..0000000
--- a/infrastructure/backup/test/resources/install-test.bb
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bb
-
-(require '[babashka.tasks :as tasks])
-
-(defn curl-and-check!
- [filename artifact-url sha256-url]
- (let [filepath (str "/tmp/" filename)]
- (tasks/shell "curl" "-SsLo" filepath artifact-url)
- (tasks/shell "curl" "-SsLo" "/tmp/checksum" sha256-url)
- (tasks/shell "bash" "-c" (str "echo \" " filepath "\"|tee -a /tmp/checksum"))
- ;(tasks/shell "sha256sum" "-c" "--status" "/tmp/checksum")
- ))
-
-(defn tar-install!
- [filename binname]
- (let [filepath (str "/tmp/" filename)]
- (tasks/shell "tar" "-C" "/tmp" "-xzf" filepath)
- (tasks/shell "install" "-m" "0700" "-o" "root" "-g" "root" (str "/tmp/" binname) "/usr/local/bin/")))
-
-(defn install!
- [filename]
- (tasks/shell "install" "-m" "0700" "-o" "root" "-g" "root" (str "/tmp/" filename) "/usr/local/bin/"))
-
-(curl-and-check!
- "provs-syspec.jar"
- "https://repo.prod.meissa.de/attachments/0a1da41e-aa5b-4a3e-a3b1-215cf2d5b021"
- "https://repo.prod.meissa.de/attachments/f227cf65-cb0f-46a7-a6cd-28f46917412a")
-(install! "provs-syspec.jar")
-(tasks/shell "java" "-jar" "/usr/local/bin/provs-syspec.jar" "local" "-c" "/tmp/spec.yml" )
diff --git a/infrastructure/backup/test/resources/spec.yml b/infrastructure/backup/test/resources/spec.yml
deleted file mode 100644
index 19d0edf..0000000
--- a/infrastructure/backup/test/resources/spec.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-package:
- - name: "restic"
-
-command:
- - command: "bb -h"
- - command: "/tmp/test.bb"
-
\ No newline at end of file
diff --git a/infrastructure/backup/test/resources/test.bb b/infrastructure/backup/test/resources/test.bb
index 2b0ba30..b6adea7 100755
--- a/infrastructure/backup/test/resources/test.bb
+++ b/infrastructure/backup/test/resources/test.bb
@@ -1,102 +1,68 @@
#!/usr/bin/env bb
(require '[babashka.tasks :as tasks]
- '[dda.backup.core :as bc]
+ '[dda.backup.config :as cfg]
'[dda.backup.restic :as rc]
'[dda.backup.postgresql :as pg]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs])
-(def restic-repo {:password-file "/restic-pwd"
- :new-password-file "/new-restic-pwd"
- :restic-repository "/restic-repo"
- :debug true})
-
-(def file-config (merge restic-repo {:backup-path "files"
- :execution-directory "/var/backups/"
- :files ["test-backup"]
- :restore-target-directory "test-restore"}))
-
-(def file-config-with-new (merge (dissoc file-config :new-password-file)
- {:password-file "/new-restic-pwd"}))
-
-(def db-config (merge restic-repo {:backup-path "db"
- :pg-db "mydb"
- :pg-user "user"
- :pg-password "password"}))
-
-(def db-roles-config (merge restic-repo {:backup-path "db-roles"
- :pg-db "mydb"
- :pg-user "user"
- :pg-password "password"}))
-
-(def dry-run {:dry-run true :debug true})
+(def config (cfg/read-config "/tmp/config.edn"))
(defn prepare!
[]
- (spit "/tmp/file_password" "file-password")
-
- (spit "/restic-pwd" "oldPassword")
- (spit "/new-restic-pwd" "newPassword")
-
+ (println config)
(tasks/shell "mkdir" "-p" "/var/backups/test-backup")
(spit "/var/backups/test-backup/file" "I was here")
(tasks/shell "mkdir" "-p" "/var/backups/test-restore")
- (pg/create-pg-pass! db-config))
-
-(defn check-env-or-file
- []
- (println "\ncheck-env-or-file")
- (println (bc/env-or-file "FILE_PASSWORD"))
- (println (bc/env-or-file "ENV_PASSWORD")))
+ (pg/create-pg-pass! (:db-config config)))
(defn restic-repo-init!
[]
(println "\nrestic-repo-init!")
- (rc/init! file-config)
- (rc/init! (merge db-config dry-run))
- (rc/init! (merge db-roles-config dry-run)))
+ (rc/init! (:file-config config))
+ (rc/init! (merge (:db-config config) (:dry-run config)))
+ (rc/init! (merge (:db-roles-config config) (:dry-run config))))
(defn restic-backup!
[]
(println "\nrestic-backup!")
- (bak/backup-file! file-config)
- (bak/backup-db-roles! (merge db-roles-config dry-run))
- (bak/backup-db! (merge db-config dry-run)))
+ (bak/backup-file! (:file-config config))
+ (bak/backup-db-roles! (merge (:db-roles-config config) (:dry-run config)))
+ (bak/backup-db! (merge (:db-config config) (:dry-run config))))
(defn list-snapshots!
[]
(println "\nlist-snapshots!")
- (rc/list-snapshots! file-config)
- (rc/list-snapshots! (merge db-roles-config dry-run))
- (rc/list-snapshots! (merge db-config dry-run)))
+ (rc/list-snapshots! (:file-config config))
+ (rc/list-snapshots! (merge (:db-roles-config config) (:dry-run config)))
+ (rc/list-snapshots! (merge (:db-config config) (:dry-run config))))
(defn restic-restore!
[]
(println "\nrestic-restore!")
- (rs/restore-file! file-config)
- (pg/drop-create-db! (merge db-config dry-run))
- (rs/restore-db-roles! (merge db-roles-config dry-run))
- (rs/restore-db! (merge db-config dry-run)))
+ (rs/restore-file! (:file-config config))
+ (pg/drop-create-db! (merge (:db-config config) (:dry-run config)))
+ (rs/restore-db-roles! (merge (:db-roles-config config) (:dry-run config)))
+ (rs/restore-db! (merge (:db-config config) (:dry-run config))))
(defn change-password!
[]
(println "\nchange-password!")
- (rc/change-password! file-config))
+ (rc/change-password! (:file-config config)))
(defn restic-backup-with-new!
[]
(println "\nrestic-backup with new!")
- (bak/backup-file! file-config-with-new))
+ (bak/backup-file! (:file-config-with-new config)))
(defn list-snapshots-with-new!
[]
(println "\nlist-snapshots with new!")
- (rc/list-snapshots! file-config-with-new))
+ (rc/list-snapshots! (:file-config-with-new config)))
(prepare!)
-(check-env-or-file)
(restic-repo-init!)
(restic-backup!)
(list-snapshots!)
@@ -106,4 +72,4 @@
(list-snapshots!)
(restic-restore!)
(restic-backup-with-new!)
-(list-snapshots-with-new!)
\ No newline at end of file
+(list-snapshots-with-new!)
diff --git a/src/dda/backup/config.clj b/src/dda/backup/config.clj
new file mode 100644
index 0000000..4fea21e
--- /dev/null
+++ b/src/dda/backup/config.clj
@@ -0,0 +1,22 @@
+(ns dda.backup.config
+ "{:test huhu :long-name #env LOGNAME :gopass-pw #gopass [sopra/test.de] :gopass-field-url #gopass [sopra/test.de url]}"
+ (:require [aero.core :as aero]
+ [dda.backup.core :as bc]
+ [dda.backup.infrastructure :as i]))
+
+(defmethod aero/reader 'env-or-file
+ [{:keys [profile] :as opts} tag value]
+ (bc/env-or-file value))
+
+(defmethod aero/reader 'gopass
+ [{:keys [profile] :as opts} tag value]
+ (i/execute-out! (into ["gopass" "show" "-y" "-o"] value) {}))
+
+(defn read-config
+ [file]
+ (try
+ (aero/read-config file)
+ (catch Exception e
+ (do (println (str "Warn: " e))
+ {}))
+ ))