db restore now might work

infrastructure/backup/test/resources/test.bb

@@ -35,13 +35,15 @@
   []
   (tasks/shell "mkdir" "-p" "test-backup")
   (spit "test-backup/file" "I was here")
-  (bak/backup-file! file-config))
+  (bak/backup-file! file-config)
+  (bak/backup-db! (merge db-config dry-run)))
 
 (defn restic-restore!
   []
   (tasks/shell "mkdir" "-p" "test-restore")
   (rs/restore-file! file-config)
-  (pg/drop-create-db! (merge db-config dry-run)))
+  (pg/drop-create-db! (merge db-config dry-run))
+  (rs/restore-db! (merge db-config dry-run)))
 
 (prepare!)
 (restic-repo-init!)

src/dda/backup/backup.clj

@@ -4,19 +4,47 @@
    [clojure.spec.alpha :as s]
    [dda.backup.backup.domain :as domain]
    [dda.backup.restic :as restic]
+   [dda.backup.postgresql :as pg]
    [dda.backup.infrastructure :as i]))
 
+(def default
+  (merge restic/default
+         pg/default))
+
 (s/def ::backup-file-config
   (s/merge ::restic/restic-config
            (s/keys :req-un [::domain/files]
                    :opt-un [])))
 
+(s/def ::pg-role-dump-config
+  (s/merge ::pg/pg-config
+           ::restic/restic-config
+           (s/keys :req-un [::domain/pg-role-prefix]
+                   :opt-un [])))
+
+(s/def ::pg-db-dump-config
+  (s/merge ::pg/pg-config
+           ::restic/restic-config))
+
 (defn-spec backup-file! nil?
   [config ::backup-file-config]
-  (let [config-w-defaults (merge restic/default config)]
+  (let [config-w-defaults (merge default config)]
     (restic/unlock! config-w-defaults)
     (i/execute!
      (domain/backup-files-command config-w-defaults)
      config-w-defaults)
-    (restic/forget! config-w-defaults)
+    (restic/forget! config-w-defaults)))
-    ))
+
+(defn-spec backup-db-roles! nil?
+  [config ::pg-role-dump-config]
+  (let [config-w-defaults (merge default config)]
+    (restic/unlock! config-w-defaults)
+    (i/execute! (domain/backup-role-command config-w-defaults) config-w-defaults)
+    (restic/forget! config-w-defaults)))
+
+(defn-spec backup-db! nil?
+  [config ::pg-db-dump-config]
+  (let [config-w-defaults (merge default config)]
+    (restic/unlock! config-w-defaults)
+    (i/execute! (domain/backup-db-command config-w-defaults) config-w-defaults)
+    (restic/forget! config-w-defaults)))

src/dda/backup/backup/domain.clj

@@ -2,10 +2,13 @@
   (:require
    [orchestra.core :refer [defn-spec]]
    [clojure.spec.alpha :as s]
+   [clojure.string :as st]
    [dda.backup.core.domain :as cd]
-   [dda.backup.restic.domain :as rd]))
+   [dda.backup.restic.domain :as rd]
+   [dda.backup.postgresql.domain :as pd]))
 
 (s/def ::files (s/+ string?))
+(s/def ::pg-role-prefix ::pd/pg-role-prefix)
 
 (s/def ::backup-file-config
   (s/merge ::rd/restic-config
@@ -15,3 +18,11 @@
   [config ::backup-file-config]
   (let [{:keys [files]} config]
     [(rd/repo-command config (into ["backup"] files))]))
+
+(defn-spec backup-role-command ::cd/commands
+  [config ::pd/pg-role-dump-config]
+  [["bash" "-c" (st/join " " (pd/pgdumpall-command config ["--roles-only"]))]])
+
+(defn-spec backup-db-command ::cd/commands
+  [config ::pd/pg-db-dump-config]
+  [["bash" "-c" (st/join " " (pd/pgdump-command config ["--serializable-deferrable"]))]])

src/dda/backup/postgresql.clj

@@ -11,91 +11,19 @@
          {:pg-host "localhost"
           :pg_port 5432}))
 
-(s/def ::psql-config
+(s/def ::pg-config
-  (s/merge ::core/execution 
   (s/keys :req-un [::domain/pg-db
                    ::domain/pg-user
                    ::domain/pg-password]
           :opt-un [::domain/pg-host
-                   ::domain/pg-port])))
+                   ::domain/pg-port]))
-
 
 (defn-spec create-pg-pass! nil?
-  [config ::psql-config]
+  [config ::pg-config]
   (let [config-w-defaults (merge default config)]
     (spit "/root/.pgpass" (domain/pgpass config-w-defaults))))
 
-
 (defn-spec drop-create-db! nil?
-  [config ::psql-config]
+  [config ::pg-config]
   (let [config-w-defaults (merge default config)]
     (i/execute! (domain/db-drop-create-command config-w-defaults) config-w-defaults)))
-
-;; function backup-roles() {
-;;   local role_prefix="$1"; shift
-
-;;   if [ -z ${CERTIFICATE_FILE} ];
-;;   then
-;;     roles-unlock-command
-;;     pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
-;;       grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin
-;;     roles-forget-command
-;;   else
-;;     roles-unlock-command --cacert ${CERTIFICATE_FILE}
-;;     pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
-;;       grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin --cacert ${CERTIFICATE_FILE}
-;;     roles-forget-command --cacert ${CERTIFICATE_FILE}
-;;   fi
-;; }
-
-;; function backup-db-dump() {
-
-;;   if [ -z ${CERTIFICATE_FILE} ];
-;;   then
-;;     db-unlock-command
-;;     pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
-;;       -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
-;;       restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin
-;;     db-forget-command
-;;   else
-;;     db-unlock-command --cacert ${CERTIFICATE_FILE}
-;;     pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
-;;       -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
-;;       restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin --cacert ${CERTIFICATE_FILE}
-;;     db-forget-command --cacert ${CERTIFICATE_FILE}
-;;   fi
-;; }
-
-;; function restore-roles() {
-;;   local snapshot_id="${1:-latest}"; shift
-
-;;   if [ -z ${CERTIFICATE_FILE} ];
-;;   then
-;;     roles-unlock-command
-;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
-;;       psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
-;;       --no-password
-;;   else
-;;     roles-unlock-command --cacert ${CERTIFICATE_FILE}
-;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin  --cacert ${CERTIFICATE_FILE} | \
-;;       psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
-;;       --no-password
-;;   fi
-;; }
-
-;; function restore-db() {
-;;   local snapshot_id="${1:-latest}"; shift
-  
-;;   if [ -z ${CERTIFICATE_FILE} ];
-;;   then
-;;     db-unlock-command
-;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin | \
-;;       psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
-;;       --no-password
-;;   else
-;;     db-unlock-command --cacert ${CERTIFICATE_FILE}
-;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
-;;       psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
-;;       --no-password
-;;   fi
-;; }

src/dda/backup/postgresql/domain.clj

@@ -2,7 +2,6 @@
   (:require
    [orchestra.core :refer [defn-spec]]
    [clojure.spec.alpha :as s]
-   [clojure.string :as st]
    [dda.backup.core.domain :as cd]
    [dda.backup.restic.domain :as rd]))
 
@@ -21,23 +20,27 @@
                    ::pg-user]
           :opt-un []))
 
-(s/def ::pg-role-config
+(s/def ::pg-role-dump-config
   (s/merge ::pg-config
            ::rd/restic-config
            (s/keys :req-un [::pg-role-prefix]
                    :opt-un [])))
 
+(s/def ::pg-db-dump-config
+  (s/merge ::pg-config
+           ::rd/restic-config))
+
 (defn-spec psql-command ::cd/command
   [config ::pg-config
    command ::cd/command]
-  (let [{:keys [pg-host pg-port pg-user]} config]
+  (let [{:keys [pg-host pg-port pg-db pg-user]} config]
     (into
-     ["psql" "-d" "template1" "-h" pg-host "-p" (str pg-port) "-U" pg-user
+     ["psql" "-d" pg-db "-h" pg-host "-p" (str pg-port) "-U" pg-user
-      "--no-password" "-c"]
+      "--no-password"]
      command)))
 
 (defn-spec pgdumpall-command ::cd/command
-  [config ::pg-role-config
+  [config ::pg-role-dump-config
    command ::cd/command]
   (let [{:keys [pg-host pg-port pg-user pg-role-prefix]} config]
     (into
@@ -47,8 +50,20 @@
        "--no-password"]
       command
       ["|" "grep" pg-role-prefix "|"]
-      (rd/repo-command config ["backup" "--stdin"])
+      (rd/repo-command config ["backup" "--stdin"])))))
-      ))))
+
+(defn-spec pgdump-command ::cd/command
+  [config ::pg-db-dump-config
+   command ::cd/command]
+  (let [{:keys [pg-host pg-port pg-db pg-user]} config]
+    (into
+     []
+     (concat
+      ["pg_dump" "-h" pg-host "-p" (str pg-port) "-d" pg-db  "-U" pg-user
+       "--no-password"]
+      command
+      ["|"]
+      (rd/repo-command config ["backup" "--stdin"])))))
 
 (defn-spec pgpass string?
   [config ::pg-config]
@@ -57,11 +72,7 @@
 
 (defn-spec db-drop-create-command ::cd/commands
   [config ::pg-config]
-  (let [{:keys [pg-db]} config]
+  (let [{:keys [pg-db]} config
-    [(psql-command config [(str "\"DROP DATABASE \\\"" pg-db "\\\";\"")])
+        config-w-template (merge config  {:pg-db "template1"})]
-     (psql-command config [(str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])]))
+    [(psql-command config-w-template ["-c" (str "\"DROP DATABASE \\\"" pg-db "\\\";\"")])
-
+     (psql-command config-w-template ["-c" (str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])]))
-(defn-spec backup-role-command ::cd/commands
-  [config ::pg-role-config]
-  (let [{:keys [pg-db]} config]
-    ["bash" "-c" (st/join " " (pgdumpall-command config ["--roles-only"]))]))

src/dda/backup/restore.clj

@@ -4,9 +4,11 @@
    [clojure.spec.alpha :as s]
    [dda.backup.restore.domain :as domain]
    [dda.backup.restic :as restic]
+   [dda.backup.postgresql :as pg]
    [dda.backup.infrastructure :as i]))
 
 (def default (merge restic/default 
+                    pg/default
                     {:snapshot-id "latest"}))
 
 (s/def ::restore-file-config
@@ -14,6 +16,10 @@
            (s/keys :req-un [::domain/target-directory]
                    :opt-un [::domain/snapshot-id])))
 
+(s/def ::restore-db-config
+  (s/merge ::pg/pg-config
+           (s/keys :req-un [::domain/snapshot-id])))
+
 (defn-spec restore-file! nil?
   [config ::restore-file-config]
   (let [config-w-defaults (merge default config)]
@@ -21,3 +27,27 @@
     (i/execute!
      (domain/restore-dir-command config-w-defaults)
      config-w-defaults)))
+
+(defn-spec restore-db! nil?
+  [config ::restore-db-config]
+  (let [config-w-defaults (merge default config)]
+    (restic/unlock! config-w-defaults)
+    (i/execute! (domain/restore-db-command config-w-defaults) config-w-defaults)))
+
+
+;; function restore-roles() {
+;;   local snapshot_id="${1:-latest}"; shift
+
+;;   if [ -z ${CERTIFICATE_FILE} ];
+;;   then
+;;     roles-unlock-command
+;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
+;;       psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password
+;;   else
+;;     roles-unlock-command --cacert ${CERTIFICATE_FILE}
+;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin  --cacert ${CERTIFICATE_FILE} | \
+;;       psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password
+;;   fi
+;; }

src/dda/backup/restore/domain.clj

@@ -2,19 +2,40 @@
   (:require
    [orchestra.core :refer [defn-spec]]
    [clojure.spec.alpha :as s]
+   [clojure.string :as st]
    [dda.backup.core.domain :as cd]
-   [dda.backup.restic.domain :as rd]))
+   [dda.backup.restic.domain :as rd]
+   [dda.backup.postgresql.domain :as pd]))
 
 (s/def ::target-directory string?)
 (s/def ::snapshot-id string?)
 
 (s/def ::restore-file-config
-  (s/merge ::rd/config
+  (s/merge ::rd/restic-config
            (s/keys :req-un [::target-directory
                             ::snapshot-id])))
 
+(s/def ::restore-db-config
+  (s/merge ::pd/pg-config
+           (s/keys :req-un [::snapshot-id])))
+
+
 (defn-spec restore-dir-command ::cd/commands
   [config ::restore-file-config]
   (let [{:keys [target-directory snapshot-id]} config]
     [["rm" "-rf" target-directory]
      (rd/repo-command config ["restore" snapshot-id "--target" target-directory])]))
+
+(defn-spec restore-db-command ::cd/commands
+  [config ::restore-db-config]
+  (let [{:keys [snapshot-id]} config]
+    [["bash" "-c"
+      (st/join
+       " "
+       (into
+        []
+        (concat
+         (rd/repo-command config ["dump" snapshot-id "stdin"])
+         ["|"]
+         (pd/psql-command config []))))]]))
+          

test/babashka/task.clj

@@ -0,0 +1,3 @@
+(ns babashka.tasks)
+
+(defn shell [a &])

test/dda/backup/backup/domain_test.clj

@@ -5,6 +5,8 @@
    [dda.backup.backup.domain :as cut]))
 
 (st/instrument `cut/backup-files-command)
+(st/instrument `cut/backup-role-command)
+(st/instrument `cut/backup-db-command)
 
 (deftest should-calculate-backup-files-command
   (is (= [[{:dir "dir-to-backup"}
@@ -21,3 +23,32 @@
                                     :days-to-keep 39
                                     :months-to-keep 3
                                     :files ["file2" "file2"]}))))
+
+(deftest should-calculate-backup-role-command
+  (is (= [["bash" "-c" (str "pg_dumpall -h localhost -p 5432 -U user --no-password --roles-only | "
+                            "grep prefix | "
+                            "restic -r repo/dir-at-repo -v backup --stdin")]]
+         (cut/backup-role-command {:restic-repository "repo"
+                                   :backup-path "dir-at-repo"
+                                   :days-to-keep 39
+                                   :months-to-keep 3
+                                   :pg-host "localhost"
+                                   :pg-port 5432
+                                   :pg-db "mydb"
+                                   :pg-role-prefix "prefix"
+                                   :pg-user "user"
+                                   :pg-password "password"}))))
+
+(deftest should-calculate-backup-db-command
+  (is (= [["bash" "-c" (str "pg_dump -h localhost -p 5432 -d mydb -U user --no-password "
+                            "--serializable-deferrable | "
+                            "restic -r repo/dir-at-repo -v backup --stdin")]]
+         (cut/backup-db-command {:restic-repository "repo"
+                                 :backup-path "dir-at-repo"
+                                 :days-to-keep 39
+                                 :months-to-keep 3
+                                 :pg-host "localhost"
+                                 :pg-port 5432
+                                 :pg-db "mydb"
+                                 :pg-user "user"
+                                 :pg-password "password"}))))

test/dda/backup/postgresql/domain_test.clj

@@ -6,7 +6,6 @@
 
 (st/instrument `cut/pgpass)
 (st/instrument `cut/db-drop-create-command)
-(st/instrument `cut/backup-role-command)
 
 (deftest should-calculate-pgpass
   (is (= "localhost:mydb:user:password"
@@ -30,16 +29,3 @@
                                       :pg-db "mydb"
                                       :pg-user "user"
                                       :pg-password "password"}))))
-
-(deftest should-calculate-backup-role-command
-  (is (= [["bash" "-c" "pg_dumpall -h localhost -p 5432 -U user --no-password --roles-only | grep prefix | restic -r repo/dir-at-repo -v backup --stdin"]]
-         (cut/backup-role-command {:restic-repository "repo"
-                                   :backup-path "dir-at-repo"
-                                   :days-to-keep 39
-                                   :months-to-keep 3
-                                   :pg-host "localhost"
-                                   :pg-port 5432
-                                   :pg-db "mydb"
-                                   :pg-role-prefix "prefix"
-                                   :pg-user "user"
-                                   :pg-password "password"}))))

test/dda/backup/restore/domain_test.clj

@@ -5,6 +5,7 @@
    [dda.backup.restore.domain :as cut]))
 
 (st/instrument `cut/restore-dir-command)
+(st/instrument `cut/restore-db-command)
 
 (deftest should-calculate-restore-dir
   (is (= [["rm" "-rf" "dir-to-backup"]
@@ -22,3 +23,19 @@
                                    :days-to-keep 39
                                    :months-to-keep 3
                                    :snapshot-id "latest"}))))
+
+(deftest should-calculate-restore-db
+  (is (= [["bash"
+           "-c"
+           (str "restic -r repo/dir-at-repo -v dump latest stdin | "
+                "psql -d mydb -h localhost -p 5432 -U user --no-password")]]
+         (cut/restore-db-command {:restic-repository "repo"
+                                  :backup-path "dir-at-repo"
+                                  :pg-host "localhost"
+                                  :pg-port 5432
+                                  :pg-db "mydb"
+                                  :pg-user "user"
+                                  :pg-password "password"
+                                  :days-to-keep 39
+                                  :months-to-keep 3
+                                  :snapshot-id "latest"}))))