From 8f69d2c1d63bc4a3a06e5211e0cec80c4d959207 Mon Sep 17 00:00:00 2001 From: Michael Jerger Date: Wed, 21 Aug 2024 08:57:08 +0200 Subject: [PATCH] db restore now might work --- infrastructure/backup/test/resources/test.bb | 6 +- src/dda/backup/backup.clj | 38 +++++++-- src/dda/backup/backup/domain.clj | 13 ++- src/dda/backup/postgresql.clj | 84 ++------------------ src/dda/backup/postgresql/domain.clj | 43 ++++++---- src/dda/backup/restore.clj | 32 +++++++- src/dda/backup/restore/domain.clj | 25 +++++- test/babashka/task.clj | 3 + test/dda/backup/backup/domain_test.clj | 31 ++++++++ test/dda/backup/postgresql/domain_test.clj | 14 ---- test/dda/backup/restore/domain_test.clj | 17 ++++ 11 files changed, 187 insertions(+), 119 deletions(-) create mode 100644 test/babashka/task.clj diff --git a/infrastructure/backup/test/resources/test.bb b/infrastructure/backup/test/resources/test.bb index 9228be7..cba7666 100755 --- a/infrastructure/backup/test/resources/test.bb +++ b/infrastructure/backup/test/resources/test.bb @@ -35,13 +35,15 @@ [] (tasks/shell "mkdir" "-p" "test-backup") (spit "test-backup/file" "I was here") - (bak/backup-file! file-config)) + (bak/backup-file! file-config) + (bak/backup-db! (merge db-config dry-run))) (defn restic-restore! [] (tasks/shell "mkdir" "-p" "test-restore") (rs/restore-file! file-config) - (pg/drop-create-db! (merge db-config dry-run))) + (pg/drop-create-db! (merge db-config dry-run)) + (rs/restore-db! (merge db-config dry-run))) (prepare!) (restic-repo-init!) diff --git a/src/dda/backup/backup.clj b/src/dda/backup/backup.clj index 89d177d..13d5e2d 100644 --- a/src/dda/backup/backup.clj +++ b/src/dda/backup/backup.clj @@ -4,19 +4,47 @@ [clojure.spec.alpha :as s] [dda.backup.backup.domain :as domain] [dda.backup.restic :as restic] + [dda.backup.postgresql :as pg] [dda.backup.infrastructure :as i])) +(def default + (merge restic/default + pg/default)) + (s/def ::backup-file-config (s/merge ::restic/restic-config (s/keys :req-un [::domain/files] :opt-un []))) +(s/def ::pg-role-dump-config + (s/merge ::pg/pg-config + ::restic/restic-config + (s/keys :req-un [::domain/pg-role-prefix] + :opt-un []))) + +(s/def ::pg-db-dump-config + (s/merge ::pg/pg-config + ::restic/restic-config)) + (defn-spec backup-file! nil? [config ::backup-file-config] - (let [config-w-defaults (merge restic/default config)] + (let [config-w-defaults (merge default config)] (restic/unlock! config-w-defaults) - (i/execute! - (domain/backup-files-command config-w-defaults) + (i/execute! + (domain/backup-files-command config-w-defaults) config-w-defaults) - (restic/forget! config-w-defaults) - )) \ No newline at end of file + (restic/forget! config-w-defaults))) + +(defn-spec backup-db-roles! nil? + [config ::pg-role-dump-config] + (let [config-w-defaults (merge default config)] + (restic/unlock! config-w-defaults) + (i/execute! (domain/backup-role-command config-w-defaults) config-w-defaults) + (restic/forget! config-w-defaults))) + +(defn-spec backup-db! nil? + [config ::pg-db-dump-config] + (let [config-w-defaults (merge default config)] + (restic/unlock! config-w-defaults) + (i/execute! (domain/backup-db-command config-w-defaults) config-w-defaults) + (restic/forget! config-w-defaults))) \ No newline at end of file diff --git a/src/dda/backup/backup/domain.clj b/src/dda/backup/backup/domain.clj index a85339b..00b678a 100644 --- a/src/dda/backup/backup/domain.clj +++ b/src/dda/backup/backup/domain.clj @@ -2,10 +2,13 @@ (:require [orchestra.core :refer [defn-spec]] [clojure.spec.alpha :as s] + [clojure.string :as st] [dda.backup.core.domain :as cd] - [dda.backup.restic.domain :as rd])) + [dda.backup.restic.domain :as rd] + [dda.backup.postgresql.domain :as pd])) (s/def ::files (s/+ string?)) +(s/def ::pg-role-prefix ::pd/pg-role-prefix) (s/def ::backup-file-config (s/merge ::rd/restic-config @@ -15,3 +18,11 @@ [config ::backup-file-config] (let [{:keys [files]} config] [(rd/repo-command config (into ["backup"] files))])) + +(defn-spec backup-role-command ::cd/commands + [config ::pd/pg-role-dump-config] + [["bash" "-c" (st/join " " (pd/pgdumpall-command config ["--roles-only"]))]]) + +(defn-spec backup-db-command ::cd/commands + [config ::pd/pg-db-dump-config] + [["bash" "-c" (st/join " " (pd/pgdump-command config ["--serializable-deferrable"]))]]) diff --git a/src/dda/backup/postgresql.clj b/src/dda/backup/postgresql.clj index 6413a20..144142d 100644 --- a/src/dda/backup/postgresql.clj +++ b/src/dda/backup/postgresql.clj @@ -6,96 +6,24 @@ [dda.backup.core :as core] [dda.backup.infrastructure :as i])) -(def default - (merge core/default +(def default + (merge core/default {:pg-host "localhost" :pg_port 5432})) -(s/def ::psql-config - (s/merge ::core/execution +(s/def ::pg-config (s/keys :req-un [::domain/pg-db ::domain/pg-user ::domain/pg-password] :opt-un [::domain/pg-host - ::domain/pg-port]))) - + ::domain/pg-port])) (defn-spec create-pg-pass! nil? - [config ::psql-config] + [config ::pg-config] (let [config-w-defaults (merge default config)] (spit "/root/.pgpass" (domain/pgpass config-w-defaults)))) - (defn-spec drop-create-db! nil? - [config ::psql-config] + [config ::pg-config] (let [config-w-defaults (merge default config)] (i/execute! (domain/db-drop-create-command config-w-defaults) config-w-defaults))) - -;; function backup-roles() { -;; local role_prefix="$1"; shift - -;; if [ -z ${CERTIFICATE_FILE} ]; -;; then -;; roles-unlock-command -;; pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \ -;; grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin -;; roles-forget-command -;; else -;; roles-unlock-command --cacert ${CERTIFICATE_FILE} -;; pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \ -;; grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin --cacert ${CERTIFICATE_FILE} -;; roles-forget-command --cacert ${CERTIFICATE_FILE} -;; fi -;; } - -;; function backup-db-dump() { - -;; if [ -z ${CERTIFICATE_FILE} ]; -;; then -;; db-unlock-command -;; pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \ -;; -U ${POSTGRES_USER} --no-password --serializable-deferrable | \ -;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin -;; db-forget-command -;; else -;; db-unlock-command --cacert ${CERTIFICATE_FILE} -;; pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \ -;; -U ${POSTGRES_USER} --no-password --serializable-deferrable | \ -;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin --cacert ${CERTIFICATE_FILE} -;; db-forget-command --cacert ${CERTIFICATE_FILE} -;; fi -;; } - -;; function restore-roles() { -;; local snapshot_id="${1:-latest}"; shift - -;; if [ -z ${CERTIFICATE_FILE} ]; -;; then -;; roles-unlock-command -;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \ -;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \ -;; --no-password -;; else -;; roles-unlock-command --cacert ${CERTIFICATE_FILE} -;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \ -;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \ -;; --no-password -;; fi -;; } - -;; function restore-db() { -;; local snapshot_id="${1:-latest}"; shift - -;; if [ -z ${CERTIFICATE_FILE} ]; -;; then -;; db-unlock-command -;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin | \ -;; psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \ -;; --no-password -;; else -;; db-unlock-command --cacert ${CERTIFICATE_FILE} -;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \ -;; psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \ -;; --no-password -;; fi -;; } \ No newline at end of file diff --git a/src/dda/backup/postgresql/domain.clj b/src/dda/backup/postgresql/domain.clj index 7b32321..a047ded 100644 --- a/src/dda/backup/postgresql/domain.clj +++ b/src/dda/backup/postgresql/domain.clj @@ -2,7 +2,6 @@ (:require [orchestra.core :refer [defn-spec]] [clojure.spec.alpha :as s] - [clojure.string :as st] [dda.backup.core.domain :as cd] [dda.backup.restic.domain :as rd])) @@ -21,23 +20,27 @@ ::pg-user] :opt-un [])) -(s/def ::pg-role-config +(s/def ::pg-role-dump-config (s/merge ::pg-config ::rd/restic-config (s/keys :req-un [::pg-role-prefix] :opt-un []))) +(s/def ::pg-db-dump-config + (s/merge ::pg-config + ::rd/restic-config)) + (defn-spec psql-command ::cd/command [config ::pg-config command ::cd/command] - (let [{:keys [pg-host pg-port pg-user]} config] + (let [{:keys [pg-host pg-port pg-db pg-user]} config] (into - ["psql" "-d" "template1" "-h" pg-host "-p" (str pg-port) "-U" pg-user - "--no-password" "-c"] + ["psql" "-d" pg-db "-h" pg-host "-p" (str pg-port) "-U" pg-user + "--no-password"] command))) (defn-spec pgdumpall-command ::cd/command - [config ::pg-role-config + [config ::pg-role-dump-config command ::cd/command] (let [{:keys [pg-host pg-port pg-user pg-role-prefix]} config] (into @@ -47,8 +50,20 @@ "--no-password"] command ["|" "grep" pg-role-prefix "|"] - (rd/repo-command config ["backup" "--stdin"]) - )))) + (rd/repo-command config ["backup" "--stdin"]))))) + +(defn-spec pgdump-command ::cd/command + [config ::pg-db-dump-config + command ::cd/command] + (let [{:keys [pg-host pg-port pg-db pg-user]} config] + (into + [] + (concat + ["pg_dump" "-h" pg-host "-p" (str pg-port) "-d" pg-db "-U" pg-user + "--no-password"] + command + ["|"] + (rd/repo-command config ["backup" "--stdin"]))))) (defn-spec pgpass string? [config ::pg-config] @@ -57,11 +72,7 @@ (defn-spec db-drop-create-command ::cd/commands [config ::pg-config] - (let [{:keys [pg-db]} config] - [(psql-command config [(str "\"DROP DATABASE \\\"" pg-db "\\\";\"")]) - (psql-command config [(str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])])) - -(defn-spec backup-role-command ::cd/commands - [config ::pg-role-config] - (let [{:keys [pg-db]} config] - ["bash" "-c" (st/join " " (pgdumpall-command config ["--roles-only"]))])) + (let [{:keys [pg-db]} config + config-w-template (merge config {:pg-db "template1"})] + [(psql-command config-w-template ["-c" (str "\"DROP DATABASE \\\"" pg-db "\\\";\"")]) + (psql-command config-w-template ["-c" (str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])])) diff --git a/src/dda/backup/restore.clj b/src/dda/backup/restore.clj index 7951ad0..648ed07 100644 --- a/src/dda/backup/restore.clj +++ b/src/dda/backup/restore.clj @@ -4,9 +4,11 @@ [clojure.spec.alpha :as s] [dda.backup.restore.domain :as domain] [dda.backup.restic :as restic] + [dda.backup.postgresql :as pg] [dda.backup.infrastructure :as i])) (def default (merge restic/default + pg/default {:snapshot-id "latest"})) (s/def ::restore-file-config @@ -14,10 +16,38 @@ (s/keys :req-un [::domain/target-directory] :opt-un [::domain/snapshot-id]))) +(s/def ::restore-db-config + (s/merge ::pg/pg-config + (s/keys :req-un [::domain/snapshot-id]))) + (defn-spec restore-file! nil? [config ::restore-file-config] (let [config-w-defaults (merge default config)] (restic/unlock! config-w-defaults) (i/execute! (domain/restore-dir-command config-w-defaults) - config-w-defaults))) \ No newline at end of file + config-w-defaults))) + +(defn-spec restore-db! nil? + [config ::restore-db-config] + (let [config-w-defaults (merge default config)] + (restic/unlock! config-w-defaults) + (i/execute! (domain/restore-db-command config-w-defaults) config-w-defaults))) + + +;; function restore-roles() { +;; local snapshot_id="${1:-latest}"; shift + +;; if [ -z ${CERTIFICATE_FILE} ]; +;; then +;; roles-unlock-command +;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \ +;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \ +;; --no-password +;; else +;; roles-unlock-command --cacert ${CERTIFICATE_FILE} +;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \ +;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \ +;; --no-password +;; fi +;; } diff --git a/src/dda/backup/restore/domain.clj b/src/dda/backup/restore/domain.clj index 20f930e..b8dd0b4 100644 --- a/src/dda/backup/restore/domain.clj +++ b/src/dda/backup/restore/domain.clj @@ -2,19 +2,40 @@ (:require [orchestra.core :refer [defn-spec]] [clojure.spec.alpha :as s] + [clojure.string :as st] [dda.backup.core.domain :as cd] - [dda.backup.restic.domain :as rd])) + [dda.backup.restic.domain :as rd] + [dda.backup.postgresql.domain :as pd])) (s/def ::target-directory string?) (s/def ::snapshot-id string?) (s/def ::restore-file-config - (s/merge ::rd/config + (s/merge ::rd/restic-config (s/keys :req-un [::target-directory ::snapshot-id]))) +(s/def ::restore-db-config + (s/merge ::pd/pg-config + (s/keys :req-un [::snapshot-id]))) + + (defn-spec restore-dir-command ::cd/commands [config ::restore-file-config] (let [{:keys [target-directory snapshot-id]} config] [["rm" "-rf" target-directory] (rd/repo-command config ["restore" snapshot-id "--target" target-directory])])) + +(defn-spec restore-db-command ::cd/commands + [config ::restore-db-config] + (let [{:keys [snapshot-id]} config] + [["bash" "-c" + (st/join + " " + (into + [] + (concat + (rd/repo-command config ["dump" snapshot-id "stdin"]) + ["|"] + (pd/psql-command config []))))]])) + diff --git a/test/babashka/task.clj b/test/babashka/task.clj new file mode 100644 index 0000000..f893cd7 --- /dev/null +++ b/test/babashka/task.clj @@ -0,0 +1,3 @@ +(ns babashka.tasks) + +(defn shell [a &]) \ No newline at end of file diff --git a/test/dda/backup/backup/domain_test.clj b/test/dda/backup/backup/domain_test.clj index 199ae72..1e7a9d5 100644 --- a/test/dda/backup/backup/domain_test.clj +++ b/test/dda/backup/backup/domain_test.clj @@ -5,6 +5,8 @@ [dda.backup.backup.domain :as cut])) (st/instrument `cut/backup-files-command) +(st/instrument `cut/backup-role-command) +(st/instrument `cut/backup-db-command) (deftest should-calculate-backup-files-command (is (= [[{:dir "dir-to-backup"} @@ -21,3 +23,32 @@ :days-to-keep 39 :months-to-keep 3 :files ["file2" "file2"]})))) + +(deftest should-calculate-backup-role-command + (is (= [["bash" "-c" (str "pg_dumpall -h localhost -p 5432 -U user --no-password --roles-only | " + "grep prefix | " + "restic -r repo/dir-at-repo -v backup --stdin")]] + (cut/backup-role-command {:restic-repository "repo" + :backup-path "dir-at-repo" + :days-to-keep 39 + :months-to-keep 3 + :pg-host "localhost" + :pg-port 5432 + :pg-db "mydb" + :pg-role-prefix "prefix" + :pg-user "user" + :pg-password "password"})))) + +(deftest should-calculate-backup-db-command + (is (= [["bash" "-c" (str "pg_dump -h localhost -p 5432 -d mydb -U user --no-password " + "--serializable-deferrable | " + "restic -r repo/dir-at-repo -v backup --stdin")]] + (cut/backup-db-command {:restic-repository "repo" + :backup-path "dir-at-repo" + :days-to-keep 39 + :months-to-keep 3 + :pg-host "localhost" + :pg-port 5432 + :pg-db "mydb" + :pg-user "user" + :pg-password "password"})))) diff --git a/test/dda/backup/postgresql/domain_test.clj b/test/dda/backup/postgresql/domain_test.clj index 1da7fe7..eb62e17 100644 --- a/test/dda/backup/postgresql/domain_test.clj +++ b/test/dda/backup/postgresql/domain_test.clj @@ -6,7 +6,6 @@ (st/instrument `cut/pgpass) (st/instrument `cut/db-drop-create-command) -(st/instrument `cut/backup-role-command) (deftest should-calculate-pgpass (is (= "localhost:mydb:user:password" @@ -30,16 +29,3 @@ :pg-db "mydb" :pg-user "user" :pg-password "password"})))) - -(deftest should-calculate-backup-role-command - (is (= [["bash" "-c" "pg_dumpall -h localhost -p 5432 -U user --no-password --roles-only | grep prefix | restic -r repo/dir-at-repo -v backup --stdin"]] - (cut/backup-role-command {:restic-repository "repo" - :backup-path "dir-at-repo" - :days-to-keep 39 - :months-to-keep 3 - :pg-host "localhost" - :pg-port 5432 - :pg-db "mydb" - :pg-role-prefix "prefix" - :pg-user "user" - :pg-password "password"})))) diff --git a/test/dda/backup/restore/domain_test.clj b/test/dda/backup/restore/domain_test.clj index ec761ca..b1c7214 100644 --- a/test/dda/backup/restore/domain_test.clj +++ b/test/dda/backup/restore/domain_test.clj @@ -5,6 +5,7 @@ [dda.backup.restore.domain :as cut])) (st/instrument `cut/restore-dir-command) +(st/instrument `cut/restore-db-command) (deftest should-calculate-restore-dir (is (= [["rm" "-rf" "dir-to-backup"] @@ -22,3 +23,19 @@ :days-to-keep 39 :months-to-keep 3 :snapshot-id "latest"})))) + +(deftest should-calculate-restore-db + (is (= [["bash" + "-c" + (str "restic -r repo/dir-at-repo -v dump latest stdin | " + "psql -d mydb -h localhost -p 5432 -U user --no-password")]] + (cut/restore-db-command {:restic-repository "repo" + :backup-path "dir-at-repo" + :pg-host "localhost" + :pg-port 5432 + :pg-db "mydb" + :pg-user "user" + :pg-password "password" + :days-to-keep 39 + :months-to-keep 3 + :snapshot-id "latest"}))))