diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d4dc574..f58418f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,7 @@ stages: - image .img: &img - image: "domaindrivenarchitecture/ddadevops-dind:4.13.0" + image: "domaindrivenarchitecture/ddadevops-dind:4.13.1" services: - docker:dind before_script: @@ -15,13 +15,16 @@ stages: - export IMAGE_TAG=$CI_COMMIT_TAG .clj-job: &clj - image: "domaindrivenarchitecture/ddadevops-clj:4.13.0" + image: "domaindrivenarchitecture/ddadevops-clj:4.13.1" cache: key: ${CI_COMMIT_REF_SLUG} paths: - .m2 before_script: - export RELEASE_ARTIFACT_TOKEN=$MEISSA_REPO_BUERO_RW + - curl -L -O https://github.com/clojure/brew-install/releases/latest/download/linux-install.sh + - chmod +x linux-install.sh + - ./linux-install.sh - mkdir -p /root/.lein - echo "{:auth {:repository-auth {#\"clojars\" {:username \"${CLOJARS_USER}\" :password \"${CLOJARS_TOKEN_DOMAINDRIVENARCHITECTURE}\" }}}}" > ~/.lein/profiles.clj diff --git a/deps.edn b/deps.edn index 5833359..415ad59 100644 --- a/deps.edn +++ b/deps.edn @@ -11,7 +11,9 @@ {;; Application org.clojure/clojure {:mvn/version "1.11.4"} org.clojure/spec.alpha {:mvn/version "0.5.238"} - orchestra/orchestra {:mvn/version "2021.01.01-1"}} + orchestra/orchestra {:mvn/version "2021.01.01-1"} + cheshire/cheshire {:mvn/version "5.13.0"} + com.widdindustries/cljc.java-time {:mvn/version "0.1.21"}} ;; --------------------------------------------------------- ;; --------------------------------------------------------- diff --git a/docs/CredentialRotation.md b/docs/CredentialRotation.md index fb66135..833e5df 100644 --- a/docs/CredentialRotation.md +++ b/docs/CredentialRotation.md @@ -1,5 +1,30 @@ # Credential Rotation +## change password step + +```mermaid +stateDiagram-v2 + noAction: no-pwd-change-needed + wait: wait-for-new-pwd + new: change-pwd + finished: pwd-change-finished + state configExist? <> + state valid? <> + state finished? <> + + [*] --> configExist? + configExist? --> valid?: new-password-config-exist? + configExist? --> noAction + valid? --> finished?: valid-from > now? + valid? --> wait + finished? --> finished: current > valid-from? + finished? --> new + new --> [*] + finished --> [*] + noAction --> [*] + wait --> [*] +``` + ## Example Data Default @@ -86,7 +111,7 @@ Validation: Steps to perform: - Add new password -- `restic -r key add --new-password-file ` +- `restic -r --new-password-file key passwd` #### New password has been added diff --git a/infrastructure/backup/image/Dockerfile b/infrastructure/backup/image/Dockerfile index 24e9b47..a6436eb 100644 --- a/infrastructure/backup/image/Dockerfile +++ b/infrastructure/backup/image/Dockerfile @@ -5,3 +5,7 @@ ADD resources /tmp/ RUN /tmp/install.sh ADD local/ /usr/local/lib/dda-backup RUN init-bb.bb +# ADD resources2 /tmp/ +# RUN install -m 0700 -o root -g root /tmp/test.bb /usr/local/bin/ +# RUN install -m 0700 -o root -g root /tmp/check.bb /usr/local/bin/ +# RUN test.bb diff --git a/infrastructure/backup/image/resources/install.sh b/infrastructure/backup/image/resources/install.sh index 559ad1a..d31a042 100755 --- a/infrastructure/backup/image/resources/install.sh +++ b/infrastructure/backup/image/resources/install.sh @@ -18,12 +18,12 @@ function main() { apt-get install -qqy ca-certificates curl gnupg postgresql-client-16 restic openjdk-21-jre-headless nano curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list' - upgradeSystem babashka_install } > /dev/null update-ca-certificates install -m 0700 -o root -g root /tmp/init-bb.bb /usr/local/bin/ + install -m 0600 -o root -g root /tmp/bb.edn /usr/local/bin/ cleanupDocker } diff --git a/infrastructure/backup/image/resources2/change-password.bb b/infrastructure/backup/image/resources2/change-password.bb new file mode 100755 index 0000000..ab0d700 --- /dev/null +++ b/infrastructure/backup/image/resources2/change-password.bb @@ -0,0 +1,34 @@ +#!/usr/bin/env bb + +(require '[dda.backup.cred-rot :as cr]) + +(def restic-repo {:password-file "/restic-pwd" + :restic-repository "/restic-repo" + :debug true}) + +(def file-config (merge restic-repo {:backup-path "files" + :files ["/test-backup"] + :restore-target-directory "/test-restore"})) + +(def cred-config (merge file-config + {:restic-repository "/restic-repo/files" + :new-password-config {:new-password-file "/new-pw" + :valid-from "2024-12-17 00:00:00"}})) + + +(def dry-run {:dry-run true :debug true}) + +(defn prepare! + [] + (spit "/restic-pwd" "ThePassword") + (spit "/new-pw" "newPassword")) + + +(defn change-password! + [] + (println "change-password!") + (cr/change-password! cred-config)) + + +(prepare!) +(change-password!) diff --git a/infrastructure/backup/image/resources2/check.bb b/infrastructure/backup/image/resources2/check.bb new file mode 100755 index 0000000..183feec --- /dev/null +++ b/infrastructure/backup/image/resources2/check.bb @@ -0,0 +1,23 @@ +#!/usr/bin/env bb + +(require '[dda.backup.restic :as rc]) + +(def restic-repo {:password-file "/restic-pwd" + :restic-repository "/restic-repo" + :debug true}) + +(def file-config (merge restic-repo {:backup-path "files" + :files ["/test-backup"] + :restore-target-directory "/test-restore"})) + +(def cred-config (merge file-config {:new-password-file "new-pw"})) + + +(def dry-run {:dry-run true :debug true}) + +(defn restic-repo-check + [] + (println "restic-repo-check") + (println (rc/check file-config))) + +(restic-repo-check) \ No newline at end of file diff --git a/infrastructure/backup/image/resources2/test.bb b/infrastructure/backup/image/resources2/test.bb new file mode 100755 index 0000000..fb0a550 --- /dev/null +++ b/infrastructure/backup/image/resources2/test.bb @@ -0,0 +1,62 @@ +#!/usr/bin/env bb + +(require '[babashka.tasks :as tasks] + '[dda.backup.restic :as rc] + '[dda.backup.backup :as bak] + '[dda.backup.restore :as rs]) + +(def restic-repo {:password-file "/restic-pwd" + :new-password-file "/new-restic-pwd" + :restic-repository "/restic-repo" + :debug true}) + +(def file-config (merge restic-repo {:backup-path "files" + :files ["/test-backup"] + :restore-target-directory "/test-restore"})) + +(def dry-run {:dry-run true :debug true}) + +(defn prepare! + [] + (spit "/tmp/file_password" "file-password") + + (spit "/restic-pwd" "oldPassword") + (spit "/new-restic-pwd" "newPassword") + + (tasks/shell "mkdir" "-p" "/test-backup") + (spit "/test-backup/file" "I was here") + (tasks/shell "mkdir" "-p" "/test-restore")) + +(defn restic-repo-init! + [] + (println "restic-repo-init!") + (rc/init! file-config)) + +(defn restic-backup! + [] + (println "restic-backup!") + (bak/backup-file! file-config)) + +(defn list-snapshots! + [] + (println "list-snapshots!") + (rc/list-snapshots! file-config)) + + +(defn restic-restore! + [] + (println "restic-restore!") + (rs/restore-file! file-config)) + +(defn change-password! + [] + (println "change-password!") + (rc/change-password! file-config)) + + +(prepare!) +(restic-repo-init!) +(restic-backup!) +(list-snapshots!) +(restic-restore!) +(change-password!) diff --git a/infrastructure/backup/test/resources/test.bb b/infrastructure/backup/test/resources/test.bb index 7883125..a129154 100755 --- a/infrastructure/backup/test/resources/test.bb +++ b/infrastructure/backup/test/resources/test.bb @@ -7,13 +7,14 @@ '[dda.backup.backup :as bak] '[dda.backup.restore :as rs]) -(def restic-repo {:password-file "restic-pwd" - :restic-repository "restic-repo"}) +(def restic-repo {:password-file "/restic-pwd" + :new-password-file "/new-restic-pwd" + :restic-repository "/restic-repo" + :debug true}) (def file-config (merge restic-repo {:backup-path "files" - :files ["test-backup"] - :restore-target-directory "test-restore"})) - + :files ["/test-backup"] + :restore-target-directory "/test-restore"})) (def db-config (merge restic-repo {:backup-path "db" :pg-db "mydb" @@ -25,38 +26,59 @@ (defn prepare! [] (spit "/tmp/file_password" "file-password") - (println (bc/env-or-file "FILE_PASSWORD")) - (println (bc/env-or-file "ENV_PASSWORD")) - (spit "restic-pwd" "ThePassword") - (tasks/shell "mkdir" "-p" "test-backup") - (spit "test-backup/file" "I was here") - (tasks/shell "mkdir" "-p" "test-restore") + + (spit "/restic-pwd" "oldPassword") + (spit "/new-restic-pwd" "newPassword") + + (tasks/shell "mkdir" "-p" "/test-backup") + (spit "/test-backup/file" "I was here") + (tasks/shell "mkdir" "-p" "/test-restore") (pg/create-pg-pass! db-config)) +(defn check-env-or-file + [] + (println "\ncheck-env-or-file") + (println (bc/env-or-file "FILE_PASSWORD")) + (println (bc/env-or-file "ENV_PASSWORD"))) + (defn restic-repo-init! - [] + [] + (println "\nrestic-repo-init!") (rc/init! file-config) (rc/init! (merge db-config dry-run))) (defn restic-backup! - [] + [] + (println "\nrestic-backup!") (bak/backup-file! file-config) (bak/backup-db! (merge db-config dry-run))) (defn list-snapshots! [] + (println "\nlist-snapshots!") (rc/list-snapshots! file-config) (rc/list-snapshots! (merge db-config dry-run))) (defn restic-restore! - [] + [] + (println "\nrestic-restore!") (rs/restore-file! file-config) (pg/drop-create-db! (merge db-config dry-run)) (rs/restore-db! (merge db-config dry-run))) +(defn change-password! + [] + (println "\nchange-password!") + (rc/change-password! file-config)) + (prepare!) +(check-env-or-file) (restic-repo-init!) (restic-backup!) (list-snapshots!) (restic-restore!) +(change-password!) +(restic-backup!) +(list-snapshots!) +(restic-restore!) diff --git a/src/dda/backup/backup.clj b/src/dda/backup/backup.clj index 13d5e2d..5facd07 100644 --- a/src/dda/backup/backup.clj +++ b/src/dda/backup/backup.clj @@ -26,25 +26,31 @@ (s/merge ::pg/pg-config ::restic/restic-config)) +(defn- config-w-defaults + [config] + (if (restic/use-new-password? config) + (merge default config {:password-file (:new-password-file config)}) + (merge default config))) + (defn-spec backup-file! nil? [config ::backup-file-config] - (let [config-w-defaults (merge default config)] - (restic/unlock! config-w-defaults) + (let [config-2-use (config-w-defaults config)] + (restic/unlock! config-2-use) (i/execute! - (domain/backup-files-command config-w-defaults) - config-w-defaults) - (restic/forget! config-w-defaults))) + (domain/backup-files-command config-2-use) + config-2-use) + (restic/forget! config-2-use))) (defn-spec backup-db-roles! nil? [config ::pg-role-dump-config] - (let [config-w-defaults (merge default config)] - (restic/unlock! config-w-defaults) - (i/execute! (domain/backup-role-command config-w-defaults) config-w-defaults) - (restic/forget! config-w-defaults))) + (let [config-2-use (config-w-defaults config)] + (restic/unlock! config-2-use) + (i/execute! (domain/backup-role-command config-2-use) config-2-use) + (restic/forget! config-2-use))) (defn-spec backup-db! nil? [config ::pg-db-dump-config] - (let [config-w-defaults (merge default config)] - (restic/unlock! config-w-defaults) - (i/execute! (domain/backup-db-command config-w-defaults) config-w-defaults) - (restic/forget! config-w-defaults))) \ No newline at end of file + (let [config-2-use (config-w-defaults config)] + (restic/unlock! config-2-use) + (i/execute! (domain/backup-db-command config-2-use) config-2-use) + (restic/forget! config-2-use))) \ No newline at end of file diff --git a/src/dda/backup/infrastructure.clj b/src/dda/backup/infrastructure.clj index 9a92d9c..15a3b45 100644 --- a/src/dda/backup/infrastructure.clj +++ b/src/dda/backup/infrastructure.clj @@ -3,12 +3,29 @@ [babashka.tasks :as t] [dda.backup.core.domain :as core])) +(defn-spec execute-out! string? + [command ::core/command + config ::core/execution] + (let [{:keys [dry-run debug]} config] + (when debug + (println command)) + (when-not dry-run + (:out (t/shell {:out :string :err :string} (clojure.string/join " " command)))))) + +(defn-spec execute-single! string? + [command ::core/command + config ::core/execution] + (let [{:keys [dry-run debug]} config] + (when debug + (println command)) + (when-not dry-run + (:out (t/shell {:err :string} (clojure.string/join " " command)))))) + + (defn-spec execute! nil? [commands ::core/commands config ::core/execution] (let [{:keys [dry-run debug]} config] (doseq [c commands] - (when debug - (println c)) (when-not dry-run - (apply t/shell c))))) + (execute-single! c config))))) \ No newline at end of file diff --git a/src/dda/backup/restic.clj b/src/dda/backup/restic.clj index 419a7ce..3da74cb 100644 --- a/src/dda/backup/restic.clj +++ b/src/dda/backup/restic.clj @@ -12,39 +12,77 @@ :months-to-keep 12})) (s/def ::restic-config - (s/merge ::core/execution + (s/merge ::core/execution (s/keys :req-un [::domain/restic-repository ::domain/backup-path] :opt-un [::domain/certificate-file ::domain/password-file + ::domain/new-password-file ::domain/days-to-keep ::domain/months-to-keep]))) -(defn-spec initalized? boolean? +(s/def ::check-result #{:initialized :wrong-password :not-initialized :error}) + +(defn-spec check ::check-result [restic-config ::restic-config] (let [config-w-defaults (merge core/default restic-config)] (try (i/execute! (domain/check-repo-command config-w-defaults) config-w-defaults) - true - (catch Exception e false)))) + :initialized + (catch Exception e + (let [data (ex-data e) + parsed-error (domain/parse-check-error (get-in data [:proc :err]))] + (cond + (= parsed-error :not-initialized) :not-initialized + (= parsed-error :wrong-password) :wrong-password + :default :error)))))) + +(defn-spec use-new-password? boolean? + "deprecated" + [restic-config ::restic-config] + (if (contains? restic-config :new-password-file) + (= :initialized (check (merge restic-config {:password-file (:new-password-file restic-config)}))) + false)) + +(defn- config-w-defaults + [restic-config] + (if (use-new-password? restic-config) + (merge default restic-config {:password-file (:new-password-file restic-config)}) + (merge default restic-config))) + +(defn-spec initalized? boolean? + "deprecated" + [restic-config ::restic-config] + (let [config-2-use (config-w-defaults restic-config)] + (= :initialized (check config-2-use)))) (defn-spec init! nil? [restic-config ::restic-config] - (let [config-w-defaults (merge core/default restic-config)] - (when (not (initalized? config-w-defaults)) - (i/execute! (domain/init-repo-command config-w-defaults) config-w-defaults)))) + (let [config-2-use (config-w-defaults restic-config)] + (when (= :not-initialized (check config-2-use)) + (i/execute! (domain/init-repo-command config-2-use) config-2-use)))) (defn-spec unlock! nil? [restic-config ::restic-config] - (let [config-w-defaults (merge core/default restic-config)] - (i/execute! (domain/unlock-repo-command config-w-defaults) config-w-defaults))) + (let [config-2-use (config-w-defaults restic-config)] + (i/execute! (domain/unlock-repo-command config-2-use) config-2-use))) (defn-spec forget! nil? [restic-config ::restic-config] - (let [config-w-defaults (merge core/default restic-config)] - (i/execute! (domain/forget-command config-w-defaults) config-w-defaults))) + (let [config-2-use (config-w-defaults restic-config)] + (i/execute! (domain/forget-command config-2-use) config-2-use))) (defn-spec list-snapshots! nil? [restic-config ::restic-config] - (let [config-w-defaults (merge core/default restic-config)] - (i/execute! (domain/list-snapshot-command config-w-defaults) config-w-defaults))) + (let [config-2-use (config-w-defaults restic-config)] + (i/execute! (domain/list-snapshot-command config-2-use) config-2-use))) + +(defn-spec change-password! nil? + [restic-config ::restic-config] + (when (contains? restic-config :new-password-file) + (let [config-2-use (merge core/default restic-config)] + (when (= :initialized (check config-2-use)) + (do + (i/execute! (domain/change-password-command config-2-use) config-2-use) + (when-not (= :wrong-password (check config-2-use)) + (throw (Exception. "password-change did not work!")))))))) diff --git a/src/dda/backup/restic/domain.clj b/src/dda/backup/restic/domain.clj index d06ae11..10594e5 100644 --- a/src/dda/backup/restic/domain.clj +++ b/src/dda/backup/restic/domain.clj @@ -6,24 +6,28 @@ (s/def ::certificate-file string?) (s/def ::password-file string?) +(s/def ::new-password-file string?) (s/def ::restic-repository string?) (s/def ::backup-path string?) (s/def ::days-to-keep pos?) (s/def ::months-to-keep pos?) (s/def ::restic-config - (s/keys :req-un [::restic-repository - ::backup-path - ::days-to-keep + (s/keys :req-un [::restic-repository + ::backup-path + ::days-to-keep ::months-to-keep] - :opt-un [::certificate-file - ::password-file + :opt-un [::certificate-file + ::password-file + ::new-password-file ::cd/execution-directory])) +(s/def ::check-error #{:not-initialized :wrong-password :no-password :unknown}) + (defn-spec repo-command ::cd/command [config ::restic-config command ::cd/command] - (let [{:keys [certificate-file password-file execution-directory + (let [{:keys [certificate-file password-file execution-directory restic-repository backup-path]} config] (into [] @@ -60,7 +64,23 @@ (defn-spec forget-command ::cd/commands [config ::restic-config] (let [{:keys [days-to-keep months-to-keep]} config] - [(repo-command config ["forget" "--group-by" "" + [(repo-command config ["forget" "--group-by" "''" "--keep-last" "1" "--keep-daily" (str days-to-keep) - "--keep-monthly" (str months-to-keep) "--prune"])])) \ No newline at end of file + "--keep-monthly" (str months-to-keep) "--prune"])])) + +(defn-spec change-password-command ::cd/command + [config ::restic-config] + (if (contains? config :new-password-file) + (let [{:keys [new-password-file]} config] + [(repo-command config ["--new-password-file" new-password-file + "key" "passwd"])]) + (throw (Exception. "change-password: new password required")))) + +(defn-spec parse-check-error ::check-error + [error string?] + (cond + (clojure.string/includes? error "Fatal: unable to open config file") :not-initialized + (clojure.string/includes? error "Fatal: wrong password or no key found") :wrong-password + (clojure.string/includes? error "Resolving password failed") :no-password + :default :unknown)) diff --git a/src/dda/backup/restore.clj b/src/dda/backup/restore.clj index 63054e3..57a5988 100644 --- a/src/dda/backup/restore.clj +++ b/src/dda/backup/restore.clj @@ -20,19 +20,26 @@ (s/merge ::pg/pg-config (s/keys :req-un [::domain/snapshot-id]))) +(defn- config-w-defaults + [config] + (if (restic/use-new-password? config) + (merge default config {:password-file (:new-password-file config)}) + (merge default config))) + + (defn-spec restore-file! nil? [config ::restore-file-config] - (let [config-w-defaults (merge default config)] - (restic/unlock! config-w-defaults) + (let [config-2-use (config-w-defaults config)] + (restic/unlock! config-2-use) (i/execute! - (domain/restore-dir-command config-w-defaults) - config-w-defaults))) + (domain/restore-dir-command config-2-use) + config-2-use))) (defn-spec restore-db! nil? [config ::restore-db-config] - (let [config-w-defaults (merge default config)] - (restic/unlock! config-w-defaults) - (i/execute! (domain/restore-db-command config-w-defaults) config-w-defaults))) + (let [config-2-use (config-w-defaults config)] + (restic/unlock! config-2-use) + (i/execute! (domain/restore-db-command config-2-use) config-2-use))) ;; function restore-roles() { diff --git a/test/dda/backup/restic/domain_test.clj b/test/dda/backup/restic/domain_test.clj index d4e7a4f..44453c6 100644 --- a/test/dda/backup/restic/domain_test.clj +++ b/test/dda/backup/restic/domain_test.clj @@ -64,9 +64,34 @@ (deftest should-calculate-forget-command (is (= [["restic" "-r" "repo/dir" "-v" "forget" - "--group-by" "" "--keep-last" "1" + "--group-by" "''" "--keep-last" "1" "--keep-daily" "39" "--keep-monthly" "3" "--prune"]] (cut/forget-command {:restic-repository "repo" :backup-path "dir" :days-to-keep 39 - :months-to-keep 3})))) \ No newline at end of file + :months-to-keep 3})))) + +(deftest should-calculate-change-password-command + (is (= [["restic" + "-r" + "repo/dir" + "-v" + "--new-password-file" + "/new-pwd" + "key" + "passwd"]] + (cut/change-password-command {:restic-repository "repo" + :new-password-file "/new-pwd" + :backup-path "dir" + :days-to-keep 39 + :months-to-keep 3})))) + +(deftest should-parse-check-error + (is (= :not-initialized + (cut/parse-check-error "Fatal: unable to open config file: stat /restic-repo/files/config: no such file or directory\nIs there a repository at the following location?\n/restic-repo/files" ) + )) + (is (= :wrong-password + (cut/parse-check-error "Fatal: wrong password or no key found\n"))) + (is (= :no-password + (cut/parse-check-error "Resolving password failed: Fatal: /restic-pwd does not exist\n"))) + )