meissa/dda-backup
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

db restore now might work

Browse source
This commit is contained in:
Michael Jerger 2024-08-21 08:57:08 +02:00
parent dccc20a5f9
commit 8f69d2c1d6
11 changed files with 187 additions and 119 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
infrastructure/backup/test/resources/test.bb
View file

@ -35,13 +35,15 @@
[]
(tasks/shell "mkdir" "-p" "test-backup")
(spit "test-backup/file" "I was here")
(bak/backup-file! file-config))
(bak/backup-file! file-config)
(bak/backup-db! (merge db-config dry-run)))
(defn restic-restore!
[]
(tasks/shell "mkdir" "-p" "test-restore")
(rs/restore-file! file-config)
(pg/drop-create-db! (merge db-config dry-run)))
(pg/drop-create-db! (merge db-config dry-run))
(rs/restore-db! (merge db-config dry-run)))
(prepare!)
(restic-repo-init!)

38
src/dda/backup/backup.clj
View file

@ -4,19 +4,47 @@
[clojure.spec.alpha :as s]
[dda.backup.backup.domain :as domain]
[dda.backup.restic :as restic]
[dda.backup.postgresql :as pg]
[dda.backup.infrastructure :as i]))
(def default
(merge restic/default
pg/default))
(s/def ::backup-file-config
(s/merge ::restic/restic-config
(s/keys :req-un [::domain/files]
:opt-un [])))
(s/def ::pg-role-dump-config
(s/merge ::pg/pg-config
::restic/restic-config
(s/keys :req-un [::domain/pg-role-prefix]
:opt-un [])))
(s/def ::pg-db-dump-config
(s/merge ::pg/pg-config
::restic/restic-config))
(defn-spec backup-file! nil?
[config ::backup-file-config]
(let [config-w-defaults (merge restic/default config)]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute!
(domain/backup-files-command config-w-defaults)
(i/execute!
(domain/backup-files-command config-w-defaults)
config-w-defaults)
(restic/forget! config-w-defaults)
))
(restic/forget! config-w-defaults)))
(defn-spec backup-db-roles! nil?
[config ::pg-role-dump-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute! (domain/backup-role-command config-w-defaults) config-w-defaults)
(restic/forget! config-w-defaults)))
(defn-spec backup-db! nil?
[config ::pg-db-dump-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute! (domain/backup-db-command config-w-defaults) config-w-defaults)
(restic/forget! config-w-defaults)))

13
src/dda/backup/backup/domain.clj
View file

@ -2,10 +2,13 @@
(:require
[orchestra.core :refer [defn-spec]]
[clojure.spec.alpha :as s]
[clojure.string :as st]
[dda.backup.core.domain :as cd]
[dda.backup.restic.domain :as rd]))
[dda.backup.restic.domain :as rd]
[dda.backup.postgresql.domain :as pd]))
(s/def ::files (s/+ string?))
(s/def ::pg-role-prefix ::pd/pg-role-prefix)
(s/def ::backup-file-config
(s/merge ::rd/restic-config
@ -15,3 +18,11 @@
[config ::backup-file-config]
(let [{:keys [files]} config]
[(rd/repo-command config (into ["backup"] files))]))
(defn-spec backup-role-command ::cd/commands
[config ::pd/pg-role-dump-config]
[["bash" "-c" (st/join " " (pd/pgdumpall-command config ["--roles-only"]))]])
(defn-spec backup-db-command ::cd/commands
[config ::pd/pg-db-dump-config]
[["bash" "-c" (st/join " " (pd/pgdump-command config ["--serializable-deferrable"]))]])

84
src/dda/backup/postgresql.clj
View file

@ -6,96 +6,24 @@
[dda.backup.core :as core]
[dda.backup.infrastructure :as i]))
(def default
(merge core/default
(def default
(merge core/default
{:pg-host "localhost"
:pg_port 5432}))
(s/def ::psql-config
(s/merge ::core/execution
(s/def ::pg-config
(s/keys :req-un [::domain/pg-db
::domain/pg-user
::domain/pg-password]
:opt-un [::domain/pg-host
::domain/pg-port])))
::domain/pg-port]))
(defn-spec create-pg-pass! nil?
[config ::psql-config]
[config ::pg-config]
(let [config-w-defaults (merge default config)]
(spit "/root/.pgpass" (domain/pgpass config-w-defaults))))
(defn-spec drop-create-db! nil?
[config ::psql-config]
[config ::pg-config]
(let [config-w-defaults (merge default config)]
(i/execute! (domain/db-drop-create-command config-w-defaults) config-w-defaults)))
;; function backup-roles() {
;; local role_prefix="$1"; shift
;; if [ -z ${CERTIFICATE_FILE} ];
;; then
;; roles-unlock-command
;; pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
;; grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin
;; roles-forget-command
;; else
;; roles-unlock-command --cacert ${CERTIFICATE_FILE}
;; pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
;; grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin --cacert ${CERTIFICATE_FILE}
;; roles-forget-command --cacert ${CERTIFICATE_FILE}
;; fi
;; }
;; function backup-db-dump() {
;; if [ -z ${CERTIFICATE_FILE} ];
;; then
;; db-unlock-command
;; pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
;; -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin
;; db-forget-command
;; else
;; db-unlock-command --cacert ${CERTIFICATE_FILE}
;; pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
;; -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin --cacert ${CERTIFICATE_FILE}
;; db-forget-command --cacert ${CERTIFICATE_FILE}
;; fi
;; }
;; function restore-roles() {
;; local snapshot_id="${1:-latest}"; shift
;; if [ -z ${CERTIFICATE_FILE} ];
;; then
;; roles-unlock-command
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
;; --no-password
;; else
;; roles-unlock-command --cacert ${CERTIFICATE_FILE}
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
;; --no-password
;; fi
;; }
;; function restore-db() {
;; local snapshot_id="${1:-latest}"; shift
;; if [ -z ${CERTIFICATE_FILE} ];
;; then
;; db-unlock-command
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin | \
;; psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
;; --no-password
;; else
;; db-unlock-command --cacert ${CERTIFICATE_FILE}
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
;; psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
;; --no-password
;; fi
;; }

43
src/dda/backup/postgresql/domain.clj
View file

@ -2,7 +2,6 @@
(:require
[orchestra.core :refer [defn-spec]]
[clojure.spec.alpha :as s]
[clojure.string :as st]
[dda.backup.core.domain :as cd]
[dda.backup.restic.domain :as rd]))
@ -21,23 +20,27 @@
::pg-user]
:opt-un []))
(s/def ::pg-role-config
(s/def ::pg-role-dump-config
(s/merge ::pg-config
::rd/restic-config
(s/keys :req-un [::pg-role-prefix]
:opt-un [])))
(s/def ::pg-db-dump-config
(s/merge ::pg-config
::rd/restic-config))
(defn-spec psql-command ::cd/command
[config ::pg-config
command ::cd/command]
(let [{:keys [pg-host pg-port pg-user]} config]
(let [{:keys [pg-host pg-port pg-db pg-user]} config]
(into
["psql" "-d" "template1" "-h" pg-host "-p" (str pg-port) "-U" pg-user
"--no-password" "-c"]
["psql" "-d" pg-db "-h" pg-host "-p" (str pg-port) "-U" pg-user
"--no-password"]
command)))
(defn-spec pgdumpall-command ::cd/command
[config ::pg-role-config
[config ::pg-role-dump-config
command ::cd/command]
(let [{:keys [pg-host pg-port pg-user pg-role-prefix]} config]
(into
@ -47,8 +50,20 @@
"--no-password"]
command
["|" "grep" pg-role-prefix "|"]
(rd/repo-command config ["backup" "--stdin"])
))))
(rd/repo-command config ["backup" "--stdin"])))))
(defn-spec pgdump-command ::cd/command
[config ::pg-db-dump-config
command ::cd/command]
(let [{:keys [pg-host pg-port pg-db pg-user]} config]
(into
[]
(concat
["pg_dump" "-h" pg-host "-p" (str pg-port) "-d" pg-db "-U" pg-user
"--no-password"]
command
["|"]
(rd/repo-command config ["backup" "--stdin"])))))
(defn-spec pgpass string?
[config ::pg-config]
@ -57,11 +72,7 @@
(defn-spec db-drop-create-command ::cd/commands
[config ::pg-config]
(let [{:keys [pg-db]} config]
[(psql-command config [(str "\"DROP DATABASE \\\"" pg-db "\\\";\"")])
(psql-command config [(str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])]))
(defn-spec backup-role-command ::cd/commands
[config ::pg-role-config]
(let [{:keys [pg-db]} config]
["bash" "-c" (st/join " " (pgdumpall-command config ["--roles-only"]))]))
(let [{:keys [pg-db]} config
config-w-template (merge config {:pg-db "template1"})]
[(psql-command config-w-template ["-c" (str "\"DROP DATABASE \\\"" pg-db "\\\";\"")])
(psql-command config-w-template ["-c" (str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])]))

32
src/dda/backup/restore.clj
View file

@ -4,9 +4,11 @@
[clojure.spec.alpha :as s]
[dda.backup.restore.domain :as domain]
[dda.backup.restic :as restic]
[dda.backup.postgresql :as pg]
[dda.backup.infrastructure :as i]))
(def default (merge restic/default
pg/default
{:snapshot-id "latest"}))
(s/def ::restore-file-config
@ -14,10 +16,38 @@
(s/keys :req-un [::domain/target-directory]
:opt-un [::domain/snapshot-id])))
(s/def ::restore-db-config
(s/merge ::pg/pg-config
(s/keys :req-un [::domain/snapshot-id])))
(defn-spec restore-file! nil?
[config ::restore-file-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute!
(domain/restore-dir-command config-w-defaults)
config-w-defaults)))
config-w-defaults)))
(defn-spec restore-db! nil?
[config ::restore-db-config]
(let [config-w-defaults (merge default config)]
(restic/unlock! config-w-defaults)
(i/execute! (domain/restore-db-command config-w-defaults) config-w-defaults)))
;; function restore-roles() {
;; local snapshot_id="${1:-latest}"; shift
;; if [ -z ${CERTIFICATE_FILE} ];
;; then
;; roles-unlock-command
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
;; --no-password
;; else
;; roles-unlock-command --cacert ${CERTIFICATE_FILE}
;; restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
;; psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
;; --no-password
;; fi
;; }

25
src/dda/backup/restore/domain.clj
View file

@ -2,19 +2,40 @@
(:require
[orchestra.core :refer [defn-spec]]
[clojure.spec.alpha :as s]
[clojure.string :as st]
[dda.backup.core.domain :as cd]
[dda.backup.restic.domain :as rd]))
[dda.backup.restic.domain :as rd]
[dda.backup.postgresql.domain :as pd]))
(s/def ::target-directory string?)
(s/def ::snapshot-id string?)
(s/def ::restore-file-config
(s/merge ::rd/config
(s/merge ::rd/restic-config
(s/keys :req-un [::target-directory
::snapshot-id])))
(s/def ::restore-db-config
(s/merge ::pd/pg-config
(s/keys :req-un [::snapshot-id])))
(defn-spec restore-dir-command ::cd/commands
[config ::restore-file-config]
(let [{:keys [target-directory snapshot-id]} config]
[["rm" "-rf" target-directory]
(rd/repo-command config ["restore" snapshot-id "--target" target-directory])]))
(defn-spec restore-db-command ::cd/commands
[config ::restore-db-config]
(let [{:keys [snapshot-id]} config]
[["bash" "-c"
(st/join
" "
(into
[]
(concat
(rd/repo-command config ["dump" snapshot-id "stdin"])
["|"]
(pd/psql-command config []))))]]))

3
test/babashka/task.clj Normal file
View file

@ -0,0 +1,3 @@
(ns babashka.tasks)
(defn shell [a &])

31
test/dda/backup/backup/domain_test.clj
View file

@ -5,6 +5,8 @@
[dda.backup.backup.domain :as cut]))
(st/instrument `cut/backup-files-command)
(st/instrument `cut/backup-role-command)
(st/instrument `cut/backup-db-command)
(deftest should-calculate-backup-files-command
(is (= [[{:dir "dir-to-backup"}
@ -21,3 +23,32 @@
:days-to-keep 39
:months-to-keep 3
:files ["file2" "file2"]}))))
(deftest should-calculate-backup-role-command
(is (= [["bash" "-c" (str "pg_dumpall -h localhost -p 5432 -U user --no-password --roles-only | "
"grep prefix | "
"restic -r repo/dir-at-repo -v backup --stdin")]]
(cut/backup-role-command {:restic-repository "repo"
:backup-path "dir-at-repo"
:days-to-keep 39
:months-to-keep 3
:pg-host "localhost"
:pg-port 5432
:pg-db "mydb"
:pg-role-prefix "prefix"
:pg-user "user"
:pg-password "password"}))))
(deftest should-calculate-backup-db-command
(is (= [["bash" "-c" (str "pg_dump -h localhost -p 5432 -d mydb -U user --no-password "
"--serializable-deferrable | "
"restic -r repo/dir-at-repo -v backup --stdin")]]
(cut/backup-db-command {:restic-repository "repo"
:backup-path "dir-at-repo"
:days-to-keep 39
:months-to-keep 3
:pg-host "localhost"
:pg-port 5432
:pg-db "mydb"
:pg-user "user"
:pg-password "password"}))))

14
test/dda/backup/postgresql/domain_test.clj
View file

@ -6,7 +6,6 @@
(st/instrument `cut/pgpass)
(st/instrument `cut/db-drop-create-command)
(st/instrument `cut/backup-role-command)
(deftest should-calculate-pgpass
(is (= "localhost:mydb:user:password"
@ -30,16 +29,3 @@
:pg-db "mydb"
:pg-user "user"
:pg-password "password"}))))
(deftest should-calculate-backup-role-command
(is (= [["bash" "-c" "pg_dumpall -h localhost -p 5432 -U user --no-password --roles-only | grep prefix | restic -r repo/dir-at-repo -v backup --stdin"]]
(cut/backup-role-command {:restic-repository "repo"
:backup-path "dir-at-repo"
:days-to-keep 39
:months-to-keep 3
:pg-host "localhost"
:pg-port 5432
:pg-db "mydb"
:pg-role-prefix "prefix"
:pg-user "user"
:pg-password "password"}))))

17
test/dda/backup/restore/domain_test.clj
View file

@ -5,6 +5,7 @@
[dda.backup.restore.domain :as cut]))
(st/instrument `cut/restore-dir-command)
(st/instrument `cut/restore-db-command)
(deftest should-calculate-restore-dir
(is (= [["rm" "-rf" "dir-to-backup"]
@ -22,3 +23,19 @@
:days-to-keep 39
:months-to-keep 3
:snapshot-id "latest"}))))
(deftest should-calculate-restore-db
(is (= [["bash"
"-c"
(str "restic -r repo/dir-at-repo -v dump latest stdin | "
"psql -d mydb -h localhost -p 5432 -U user --no-password")]]
(cut/restore-db-command {:restic-repository "repo"
:backup-path "dir-at-repo"
:pg-host "localhost"
:pg-port 5432
:pg-db "mydb"
:pg-user "user"
:pg-password "password"
:days-to-keep 39
:months-to-keep 3
:snapshot-id "latest"}))))