meissa/dda-backup
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request 'credential-rotation' (#1) from credential-rotation into main

Browse source 
Reviewed-on: #1
This commit is contained in:
jem 2024-12-27 09:25:08 +00:00
commit d057592ba7
15 changed files with 353 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.gitlab-ci.yml
View file

@ -5,7 +5,7 @@ stages:
- image
.img: &img
image: "domaindrivenarchitecture/ddadevops-dind:4.13.0"
image: "domaindrivenarchitecture/ddadevops-dind:4.13.1"
services:
- docker:dind
before_script:
@ -15,13 +15,16 @@ stages:
- export IMAGE_TAG=$CI_COMMIT_TAG
.clj-job: &clj
image: "domaindrivenarchitecture/ddadevops-clj:4.13.0"
image: "domaindrivenarchitecture/ddadevops-clj:4.13.1"
cache:
key: ${CI_COMMIT_REF_SLUG}
paths:
- .m2
before_script:
- export RELEASE_ARTIFACT_TOKEN=$MEISSA_REPO_BUERO_RW
- curl -L -O https://github.com/clojure/brew-install/releases/latest/download/linux-install.sh
- chmod +x linux-install.sh
- ./linux-install.sh
- mkdir -p /root/.lein
- echo "{:auth {:repository-auth {#\"clojars\" {:username \"${CLOJARS_USER}\" :password \"${CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE}\" }}}}" > ~/.lein/profiles.clj

4
deps.edn
View file

@ -11,7 +11,9 @@
{;; Application
org.clojure/clojure {:mvn/version "1.11.4"}
org.clojure/spec.alpha {:mvn/version "0.5.238"}
orchestra/orchestra {:mvn/version "2021.01.01-1"}}
orchestra/orchestra {:mvn/version "2021.01.01-1"}
cheshire/cheshire {:mvn/version "5.13.0"}
com.widdindustries/cljc.java-time {:mvn/version "0.1.21"}}
;; ---------------------------------------------------------
;; ---------------------------------------------------------

27
docs/CredentialRotation.md
View file

@ -1,5 +1,30 @@
# Credential Rotation
## change password step
```mermaid
stateDiagram-v2
noAction: no-pwd-change-needed
wait: wait-for-new-pwd
new: change-pwd
finished: pwd-change-finished
state configExist? <<choice>>
state valid? <<choice>>
state finished? <<choice>>
[*] --> configExist?
configExist? --> valid?: new-password-config-exist?
configExist? --> noAction
valid? --> finished?: valid-from > now?
valid? --> wait
finished? --> finished: current > valid-from?
finished? --> new
new --> [*]
finished --> [*]
noAction --> [*]
wait --> [*]
```
## Example Data
Default
@ -86,7 +111,7 @@ Validation:
Steps to perform:
- Add new password
- `restic -r <repo> key add --new-password-file <file>`
- `restic -r <repo> --new-password-file <file> key passwd`
#### New password has been added

4
infrastructure/backup/image/Dockerfile
View file

@ -5,3 +5,7 @@ ADD resources /tmp/
RUN /tmp/install.sh
ADD local/ /usr/local/lib/dda-backup
RUN init-bb.bb
# ADD resources2 /tmp/
# RUN install -m 0700 -o root -g root /tmp/test.bb /usr/local/bin/
# RUN install -m 0700 -o root -g root /tmp/check.bb /usr/local/bin/
# RUN test.bb

2
infrastructure/backup/image/resources/install.sh
View file

@ -18,12 +18,12 @@ function main() {
apt-get install -qqy ca-certificates curl gnupg postgresql-client-16 restic openjdk-21-jre-headless nano
curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg
sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
upgradeSystem
babashka_install
} > /dev/null
update-ca-certificates
install -m 0700 -o root -g root /tmp/init-bb.bb /usr/local/bin/
install -m 0600 -o root -g root /tmp/bb.edn /usr/local/bin/
cleanupDocker
}

34
infrastructure/backup/image/resources2/change-password.bb Executable file
View file

@ -0,0 +1,34 @@
#!/usr/bin/env bb
(require '[dda.backup.cred-rot :as cr])
(def restic-repo {:password-file "/restic-pwd"
:restic-repository "/restic-repo"
:debug true})
(def file-config (merge restic-repo {:backup-path "files"
:files ["/test-backup"]
:restore-target-directory "/test-restore"}))
(def cred-config (merge file-config
{:restic-repository "/restic-repo/files"
:new-password-config {:new-password-file "/new-pw"
:valid-from "2024-12-17 00:00:00"}}))
(def dry-run {:dry-run true :debug true})
(defn prepare!
[]
(spit "/restic-pwd" "ThePassword")
(spit "/new-pw" "newPassword"))
(defn change-password!
[]
(println "change-password!")
(cr/change-password! cred-config))
(prepare!)
(change-password!)

23
infrastructure/backup/image/resources2/check.bb Executable file
View file

@ -0,0 +1,23 @@
#!/usr/bin/env bb
(require '[dda.backup.restic :as rc])
(def restic-repo {:password-file "/restic-pwd"
:restic-repository "/restic-repo"
:debug true})
(def file-config (merge restic-repo {:backup-path "files"
:files ["/test-backup"]
:restore-target-directory "/test-restore"}))
(def cred-config (merge file-config {:new-password-file "new-pw"}))
(def dry-run {:dry-run true :debug true})
(defn restic-repo-check
[]
(println "restic-repo-check")
(println (rc/check file-config)))
(restic-repo-check)

62
infrastructure/backup/image/resources2/test.bb Executable file
View file

@ -0,0 +1,62 @@
#!/usr/bin/env bb
(require '[babashka.tasks :as tasks]
'[dda.backup.restic :as rc]
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs])
(def restic-repo {:password-file "/restic-pwd"
:new-password-file "/new-restic-pwd"
:restic-repository "/restic-repo"
:debug true})
(def file-config (merge restic-repo {:backup-path "files"
:files ["/test-backup"]
:restore-target-directory "/test-restore"}))
(def dry-run {:dry-run true :debug true})
(defn prepare!
[]
(spit "/tmp/file_password" "file-password")
(spit "/restic-pwd" "oldPassword")
(spit "/new-restic-pwd" "newPassword")
(tasks/shell "mkdir" "-p" "/test-backup")
(spit "/test-backup/file" "I was here")
(tasks/shell "mkdir" "-p" "/test-restore"))
(defn restic-repo-init!
[]
(println "restic-repo-init!")
(rc/init! file-config))
(defn restic-backup!
[]
(println "restic-backup!")
(bak/backup-file! file-config))
(defn list-snapshots!
[]
(println "list-snapshots!")
(rc/list-snapshots! file-config))
(defn restic-restore!
[]
(println "restic-restore!")
(rs/restore-file! file-config))
(defn change-password!
[]
(println "change-password!")
(rc/change-password! file-config))
(prepare!)
(restic-repo-init!)
(restic-backup!)
(list-snapshots!)
(restic-restore!)
(change-password!)

50
infrastructure/backup/test/resources/test.bb
View file

@ -7,13 +7,14 @@
'[dda.backup.backup :as bak]
'[dda.backup.restore :as rs])
(def restic-repo {:password-file "restic-pwd"
:restic-repository "restic-repo"})
(def restic-repo {:password-file "/restic-pwd"
:new-password-file "/new-restic-pwd"
:restic-repository "/restic-repo"
:debug true})
(def file-config (merge restic-repo {:backup-path "files"
:files ["test-backup"]
:restore-target-directory "test-restore"}))
:files ["/test-backup"]
:restore-target-directory "/test-restore"}))
(def db-config (merge restic-repo {:backup-path "db"
:pg-db "mydb"
@ -25,38 +26,59 @@
(defn prepare!
[]
(spit "/tmp/file_password" "file-password")
(println (bc/env-or-file "FILE_PASSWORD"))
(println (bc/env-or-file "ENV_PASSWORD"))
(spit "restic-pwd" "ThePassword")
(tasks/shell "mkdir" "-p" "test-backup")
(spit "test-backup/file" "I was here")
(tasks/shell "mkdir" "-p" "test-restore")
(spit "/restic-pwd" "oldPassword")
(spit "/new-restic-pwd" "newPassword")
(tasks/shell "mkdir" "-p" "/test-backup")
(spit "/test-backup/file" "I was here")
(tasks/shell "mkdir" "-p" "/test-restore")
(pg/create-pg-pass! db-config))
(defn check-env-or-file
[]
(println "\ncheck-env-or-file")
(println (bc/env-or-file "FILE_PASSWORD"))
(println (bc/env-or-file "ENV_PASSWORD")))
(defn restic-repo-init!
[]
[]
(println "\nrestic-repo-init!")
(rc/init! file-config)
(rc/init! (merge db-config dry-run)))
(defn restic-backup!
[]
[]
(println "\nrestic-backup!")
(bak/backup-file! file-config)
(bak/backup-db! (merge db-config dry-run)))
(defn list-snapshots!
[]
(println "\nlist-snapshots!")
(rc/list-snapshots! file-config)
(rc/list-snapshots! (merge db-config dry-run)))
(defn restic-restore!
[]
[]
(println "\nrestic-restore!")
(rs/restore-file! file-config)
(pg/drop-create-db! (merge db-config dry-run))
(rs/restore-db! (merge db-config dry-run)))
(defn change-password!
[]
(println "\nchange-password!")
(rc/change-password! file-config))
(prepare!)
(check-env-or-file)
(restic-repo-init!)
(restic-backup!)
(list-snapshots!)
(restic-restore!)
(change-password!)
(restic-backup!)
(list-snapshots!)
(restic-restore!)

32
src/dda/backup/backup.clj
View file

@ -26,25 +26,31 @@
(s/merge ::pg/pg-config
::restic/restic-config))
(defn- config-w-defaults
[config]
(if (restic/use-new-password? config)
(merge default config {:password-file (:new-password-file config)})
(merge default config)))
(defn-spec backup-file! nil?
[config ::backup-file-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(let [config-2-use (config-w-defaults config)]
(restic/unlock! config-2-use)
(i/execute!
(domain/backup-files-command config-w-defaults)
config-w-defaults)
(restic/forget! config-w-defaults)))
(domain/backup-files-command config-2-use)
config-2-use)
(restic/forget! config-2-use)))
(defn-spec backup-db-roles! nil?
[config ::pg-role-dump-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute! (domain/backup-role-command config-w-defaults) config-w-defaults)
(restic/forget! config-w-defaults)))
(let [config-2-use (config-w-defaults config)]
(restic/unlock! config-2-use)
(i/execute! (domain/backup-role-command config-2-use) config-2-use)
(restic/forget! config-2-use)))
(defn-spec backup-db! nil?
[config ::pg-db-dump-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute! (domain/backup-db-command config-w-defaults) config-w-defaults)
(restic/forget! config-w-defaults)))
(let [config-2-use (config-w-defaults config)]
(restic/unlock! config-2-use)
(i/execute! (domain/backup-db-command config-2-use) config-2-use)
(restic/forget! config-2-use)))

23
src/dda/backup/infrastructure.clj
View file

@ -3,12 +3,29 @@
[babashka.tasks :as t]
[dda.backup.core.domain :as core]))
(defn-spec execute-out! string?
[command ::core/command
config ::core/execution]
(let [{:keys [dry-run debug]} config]
(when debug
(println command))
(when-not dry-run
(:out (t/shell {:out :string :err :string} (clojure.string/join " " command))))))
(defn-spec execute-single! string?
[command ::core/command
config ::core/execution]
(let [{:keys [dry-run debug]} config]
(when debug
(println command))
(when-not dry-run
(:out (t/shell {:err :string} (clojure.string/join " " command))))))
(defn-spec execute! nil?
[commands ::core/commands
config ::core/execution]
(let [{:keys [dry-run debug]} config]
(doseq [c commands]
(when debug
(println c))
(when-not dry-run
(apply t/shell c)))))
(execute-single! c config)))))

64
src/dda/backup/restic.clj
View file

@ -12,39 +12,77 @@
:months-to-keep 12}))
(s/def ::restic-config
(s/merge ::core/execution
(s/merge ::core/execution
(s/keys :req-un [::domain/restic-repository
::domain/backup-path]
:opt-un [::domain/certificate-file
::domain/password-file
::domain/new-password-file
::domain/days-to-keep
::domain/months-to-keep])))
(defn-spec initalized? boolean?
(s/def ::check-result #{:initialized :wrong-password :not-initialized :error})
(defn-spec check ::check-result
[restic-config ::restic-config]
(let [config-w-defaults (merge core/default restic-config)]
(try
(i/execute! (domain/check-repo-command config-w-defaults) config-w-defaults)
true
(catch Exception e false))))
:initialized
(catch Exception e
(let [data (ex-data e)
parsed-error (domain/parse-check-error (get-in data [:proc :err]))]
(cond
(= parsed-error :not-initialized) :not-initialized
(= parsed-error :wrong-password) :wrong-password
:default :error))))))
(defn-spec use-new-password? boolean?
"deprecated"
[restic-config ::restic-config]
(if (contains? restic-config :new-password-file)
(= :initialized (check (merge restic-config {:password-file (:new-password-file restic-config)})))
false))
(defn- config-w-defaults
[restic-config]
(if (use-new-password? restic-config)
(merge default restic-config {:password-file (:new-password-file restic-config)})
(merge default restic-config)))
(defn-spec initalized? boolean?
"deprecated"
[restic-config ::restic-config]
(let [config-2-use (config-w-defaults restic-config)]
(= :initialized (check config-2-use))))
(defn-spec init! nil?
[restic-config ::restic-config]
(let [config-w-defaults (merge core/default restic-config)]
(when (not (initalized? config-w-defaults))
(i/execute! (domain/init-repo-command config-w-defaults) config-w-defaults))))
(let [config-2-use (config-w-defaults restic-config)]
(when (= :not-initialized (check config-2-use))
(i/execute! (domain/init-repo-command config-2-use) config-2-use))))
(defn-spec unlock! nil?
[restic-config ::restic-config]
(let [config-w-defaults (merge core/default restic-config)]
(i/execute! (domain/unlock-repo-command config-w-defaults) config-w-defaults)))
(let [config-2-use (config-w-defaults restic-config)]
(i/execute! (domain/unlock-repo-command config-2-use) config-2-use)))
(defn-spec forget! nil?
[restic-config ::restic-config]
(let [config-w-defaults (merge core/default restic-config)]
(i/execute! (domain/forget-command config-w-defaults) config-w-defaults)))
(let [config-2-use (config-w-defaults restic-config)]
(i/execute! (domain/forget-command config-2-use) config-2-use)))
(defn-spec list-snapshots! nil?
[restic-config ::restic-config]
(let [config-w-defaults (merge core/default restic-config)]
(i/execute! (domain/list-snapshot-command config-w-defaults) config-w-defaults)))
(let [config-2-use (config-w-defaults restic-config)]
(i/execute! (domain/list-snapshot-command config-2-use) config-2-use)))
(defn-spec change-password! nil?
[restic-config ::restic-config]
(when (contains? restic-config :new-password-file)
(let [config-2-use (merge core/default restic-config)]
(when (= :initialized (check config-2-use))
(do
(i/execute! (domain/change-password-command config-2-use) config-2-use)
(when-not (= :wrong-password (check config-2-use))
(throw (Exception. "password-change did not work!"))))))))

36
src/dda/backup/restic/domain.clj
View file

@ -6,24 +6,28 @@
(s/def ::certificate-file string?)
(s/def ::password-file string?)
(s/def ::new-password-file string?)
(s/def ::restic-repository string?)
(s/def ::backup-path string?)
(s/def ::days-to-keep pos?)
(s/def ::months-to-keep pos?)
(s/def ::restic-config
(s/keys :req-un [::restic-repository
::backup-path
::days-to-keep
(s/keys :req-un [::restic-repository
::backup-path
::days-to-keep
::months-to-keep]
:opt-un [::certificate-file
::password-file
:opt-un [::certificate-file
::password-file
::new-password-file
::cd/execution-directory]))
(s/def ::check-error #{:not-initialized :wrong-password :no-password :unknown})
(defn-spec repo-command ::cd/command
[config ::restic-config
command ::cd/command]
(let [{:keys [certificate-file password-file execution-directory
(let [{:keys [certificate-file password-file execution-directory
restic-repository backup-path]} config]
(into
[]
@ -60,7 +64,23 @@
(defn-spec forget-command ::cd/commands
[config ::restic-config]
(let [{:keys [days-to-keep months-to-keep]} config]
[(repo-command config ["forget" "--group-by" ""
[(repo-command config ["forget" "--group-by" "''"
"--keep-last" "1"
"--keep-daily" (str days-to-keep)
"--keep-monthly" (str months-to-keep) "--prune"])]))
"--keep-monthly" (str months-to-keep) "--prune"])]))
(defn-spec change-password-command ::cd/command
[config ::restic-config]
(if (contains? config :new-password-file)
(let [{:keys [new-password-file]} config]
[(repo-command config ["--new-password-file" new-password-file
"key" "passwd"])])
(throw (Exception. "change-password: new password required"))))
(defn-spec parse-check-error ::check-error
[error string?]
(cond
(clojure.string/includes? error "Fatal: unable to open config file") :not-initialized
(clojure.string/includes? error "Fatal: wrong password or no key found") :wrong-password
(clojure.string/includes? error "Resolving password failed") :no-password
:default :unknown))

21
src/dda/backup/restore.clj
View file

@ -20,19 +20,26 @@
(s/merge ::pg/pg-config
(s/keys :req-un [::domain/snapshot-id])))
(defn- config-w-defaults
[config]
(if (restic/use-new-password? config)
(merge default config {:password-file (:new-password-file config)})
(merge default config)))
(defn-spec restore-file! nil?
[config ::restore-file-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(let [config-2-use (config-w-defaults config)]
(restic/unlock! config-2-use)
(i/execute!
(domain/restore-dir-command config-w-defaults)
config-w-defaults)))
(domain/restore-dir-command config-2-use)
config-2-use)))
(defn-spec restore-db! nil?
[config ::restore-db-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute! (domain/restore-db-command config-w-defaults) config-w-defaults)))
(let [config-2-use (config-w-defaults config)]
(restic/unlock! config-2-use)
(i/execute! (domain/restore-db-command config-2-use) config-2-use)))
;; function restore-roles() {

29
test/dda/backup/restic/domain_test.clj
View file

@ -64,9 +64,34 @@
(deftest should-calculate-forget-command
(is (= [["restic" "-r" "repo/dir" "-v" "forget"
"--group-by" "" "--keep-last" "1"
"--group-by" "''" "--keep-last" "1"
"--keep-daily" "39" "--keep-monthly" "3" "--prune"]]
(cut/forget-command {:restic-repository "repo"
:backup-path "dir"
:days-to-keep 39
:months-to-keep 3}))))
:months-to-keep 3}))))
(deftest should-calculate-change-password-command
(is (= [["restic"
"-r"
"repo/dir"
"-v"
"--new-password-file"
"/new-pwd"
"key"
"passwd"]]
(cut/change-password-command {:restic-repository "repo"
:new-password-file "/new-pwd"
:backup-path "dir"
:days-to-keep 39
:months-to-keep 3}))))
(deftest should-parse-check-error
(is (= :not-initialized
(cut/parse-check-error "Fatal: unable to open config file: stat /restic-repo/files/config: no such file or directory\nIs there a repository at the following location?\n/restic-repo/files" )
))
(is (= :wrong-password
(cut/parse-check-error "Fatal: wrong password or no key found\n")))
(is (= :no-password
(cut/parse-check-error "Resolving password failed: Fatal: /restic-pwd does not exist\n")))
)