accept fail of drop-db

.gitlab-ci.yml

@@ -41,7 +41,7 @@ package-jar:
   <<: *clj
   stage: package
   script:
-    - pyb package_jar
+    - pyb package
   artifacts:
     paths:
       - target/org.domaindrivenarchitecture
@@ -51,7 +51,7 @@ release-to-clojars:
   <<: *tag_only
   stage: upload
   script:
-    - pyb upload_clj
+    - pyb upload
 
 dda-backup-image-publish:
   <<: *img

build.py

@@ -35,17 +35,17 @@ def initialize(project):
 
 
 @task
-def test_clj(project):
+def test(project):
     run("make test", shell=True, check=True)
 
 
 @task
-def package_jar(project):
+def package(project):
     run("make build-jar", shell=True, check=True)
 
 
 @task
-def upload_clj(project):
+def upload(project):
     run("make deploy", shell=True, check=True)
 
 

infrastructure/backup/image/Dockerfile

@@ -4,3 +4,4 @@ FROM ubuntu:24.04
 ADD resources /tmp/
 RUN /tmp/install.sh
 ADD local/ /usr/local/lib/dda-backup
+RUN init-bb.bb

infrastructure/backup/image/resources/bb.edn

@@ -0,0 +1,3 @@
+{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
+        orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

infrastructure/backup/image/resources/init-bb.bb

@@ -0,0 +1,3 @@
+#!/usr/bin/env bb
+
+(println "initialized")

infrastructure/backup/image/resources/install.sh

@@ -15,15 +15,15 @@ function babashka_install() {
 function main() {
     {
         upgradeSystem
-        apt-get install -qqy ca-certificates curl gnupg postgresql-client-16 restic
+        apt-get install -qqy ca-certificates curl gnupg postgresql-client-16 restic openjdk-21-jre-headless nano
         curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg
         sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
         upgradeSystem
-        babashka_install        
+        babashka_install      
     } > /dev/null
 
     update-ca-certificates
-
+    install -m 0700 -o root -g root /tmp/init-bb.bb /usr/local/bin/
     cleanupDocker
 }
 

infrastructure/backup/test/Dockerfile

@@ -1,7 +1,6 @@
-FROM c4k-forgejo-backup:latest
+FROM dda-backup:latest
 
 # install it
-RUN apt update && apt install -qqy openjdk-17-jre-headless
+#ADD local/ /usr/local/lib/dda-backup
-ADD local/ /usr/local/lib/dda-backup
 ADD resources /tmp/
-RUN ENV_PASSWORD=env-password /tmp/test.bb
+RUN ENV_PASSWORD=env-password FILE_PASSWORD_FILE=/tmp/file_password /tmp/test.bb

infrastructure/backup/test/resources/install-test.bb

@@ -26,6 +26,4 @@
  "https://repo.prod.meissa.de/attachments/0a1da41e-aa5b-4a3e-a3b1-215cf2d5b021"
  "https://repo.prod.meissa.de/attachments/f227cf65-cb0f-46a7-a6cd-28f46917412a")
 (install! "provs-syspec.jar")
-(tasks/shell "apt" "update")
-(tasks/shell "apt" "install" "-qqy" "openjdk-17-jre-headless")
 (tasks/shell "java" "-jar" "/usr/local/bin/provs-syspec.jar" "local" "-c" "/tmp/spec.yml" )

infrastructure/backup/test/resources/test.bb

@@ -24,9 +24,9 @@
 
 (defn prepare!
   []
-  (spit "file_password" "file-password")
+  (spit "/tmp/file_password" "file-password")
-  (println (bc/env-or-file "file_password"))
+  (println (bc/env-or-file "FILE_PASSWORD"))
-  (println (bc/env-or-file "env_password"))
+  (println (bc/env-or-file "ENV_PASSWORD"))
   (spit "restic-pwd" "ThePassword")
   (tasks/shell "mkdir" "-p" "test-backup")
   (spit "test-backup/file" "I was here")
@@ -43,6 +43,12 @@
   (bak/backup-file! file-config)
   (bak/backup-db! (merge db-config dry-run)))
 
+(defn list-snapshots!
+  []
+  (rc/list-snapshots! file-config)
+  (rc/list-snapshots! (merge db-config dry-run)))
+
+
 (defn restic-restore!
   []  
   (rs/restore-file! file-config)
@@ -52,4 +58,5 @@
 (prepare!)
 (restic-repo-init!)
 (restic-backup!)
+(list-snapshots!)
 (restic-restore!)

src/dda/backup/core.clj

@@ -2,7 +2,7 @@
   (:require
    [orchestra.core :refer [defn-spec]]
    [clojure.spec.alpha :as s]
-   [clojure.string :as st]))
+   [dda.backup.infrastructure :as i]))
 
 (def default {:dry-run false
               :debug false})
@@ -13,11 +13,29 @@
                    ::debug
                    ::execution-directory]))
 
+(s/def ::aws-access-key-id string?)
+(s/def ::aws-secret-access-key string?)
+(s/def ::aws-config
+  (s/keys :req-un [::aws-access-key-id
+                   ::aws-secret-access-key]))
+
 (defn-spec env-or-file string?
   [name string?]
-  (let [name-upper (st/upper-case name)
+  (let [from-env (System/getenv name)
-        name-lower (st/lower-case name)
+        name-from-file (System/getenv (str name "_FILE"))]
-        from-env (System/getenv name-upper)]
+    (cond 
-    (if (some? from-env)
+      (some? from-env) from-env
-      from-env
+      (some? name-from-file) (slurp name-from-file)
-      (slurp name-lower))))
+      :else (throw ( RuntimeException. 
+                    (str "Environment: [" name "," name-from-file "] was missing." ))))))
+
+
+(defn-spec create-aws-credentials! nil?
+  [config ::aws-config]
+  (let [{:keys [aws-access-key-id aws-secret-access-key]} config]
+    (i/execute! [["mkdir" "-p" "/root/.aws"]] {})
+    (spit "/root/.aws/credentials"
+          (str "[default]\n"
+               "aws_access_key_id=" aws-access-key-id "\n"
+               "aws_secret_access_key=" aws-secret-access-key "\n"))
+    (i/execute! [["chmod" "0600" "/root/.aws/credentials"]] {})))

src/dda/backup/infrastructure.clj

@@ -4,7 +4,7 @@
             [dda.backup.core.domain :as core]))
 
 (defn-spec execute! nil?
-  [commands ::core/command
+  [commands ::core/commands
    config ::core/execution]
   (let [{:keys [dry-run debug]} config]
     (doseq [c commands]

src/dda/backup/postgresql.clj

@@ -21,9 +21,12 @@
 (defn-spec create-pg-pass! nil?
   [config ::pg-config]
   (let [config-w-defaults (merge default config)]
-    (spit "/root/.pgpass" (domain/pgpass config-w-defaults))))
+    (spit "/root/.pgpass" (domain/pgpass config-w-defaults))
+    (i/execute! [["chmod" "0600" "/root/.pgpass"]] config)))
 
 (defn-spec drop-create-db! nil?
   [config ::pg-config]
   (let [config-w-defaults (merge default config)]
-    (i/execute! (domain/db-drop-create-command config-w-defaults) config-w-defaults)))
+    (try (i/execute! (domain/db-drop-command config-w-defaults) config-w-defaults)
+         (catch Exception e (println (.getMessage e))))
+    (i/execute! (domain/db-create-command config-w-defaults) config-w-defaults)))

src/dda/backup/postgresql/domain.clj

@@ -67,12 +67,18 @@
 
 (defn-spec pgpass string?
   [config ::pg-config]
-  (let [{:keys [pg-host pg-db pg-user pg-password]} config]
+  (let [{:keys [pg-host pg-port pg-db pg-user pg-password]} config]
-    (str pg-host ":" pg-db ":" pg-user ":" pg-password)))
+    (str pg-host ":" pg-port ":" pg-db ":" pg-user ":" pg-password "\n" 
+         pg-host ":" pg-port ":template1:" pg-user ":" pg-password "\n")))
 
-(defn-spec db-drop-create-command ::cd/commands
+(defn-spec db-drop-command ::cd/commands
   [config ::pg-config]
   (let [{:keys [pg-db]} config
         config-w-template (merge config  {:pg-db "template1"})]
-    [(psql-command config-w-template ["-c" (str "\"DROP DATABASE \\\"" pg-db "\\\";\"")])
+    [(psql-command config-w-template ["-c" (str "DROP DATABASE \"" pg-db "\";")])]))
-     (psql-command config-w-template ["-c" (str "\"CREATE DATABASE \\\"" pg-db "\\\";\"")])]))
+
+(defn-spec db-create-command ::cd/commands
+  [config ::pg-config]
+  (let [{:keys [pg-db]} config
+        config-w-template (merge config  {:pg-db "template1"})]
+    [(psql-command config-w-template ["-c" (str "CREATE DATABASE \"" pg-db "\";")])]))

src/dda/backup/restic.clj

@@ -42,4 +42,9 @@
 (defn-spec forget! nil?
   [restic-config ::restic-config]
   (let [config-w-defaults (merge core/default restic-config)]
-    (i/execute! (domain/forget-command config-w-defaults) config-w-defaults)))
+    (i/execute! (domain/forget-command config-w-defaults) config-w-defaults)))
+
+(defn-spec list-snapshots! nil?
+  [restic-config ::restic-config]
+  (let [config-w-defaults (merge core/default restic-config)]
+    (i/execute! (domain/list-snapshot-command config-w-defaults) config-w-defaults)))

test/dda/backup/postgresql/domain_test.clj

@@ -5,10 +5,11 @@
    [dda.backup.postgresql.domain :as cut]))
 
 (st/instrument `cut/pgpass)
-(st/instrument `cut/db-drop-create-command)
+(st/instrument `cut/db-drop-command)
+(st/instrument `cut/db-create-command)
 
 (deftest should-calculate-pgpass
-  (is (= "localhost:mydb:user:password"
+  (is (= "localhost:5432:mydb:user:password\nlocalhost:5432:template1:user:password\n"
          (cut/pgpass {:restic-repository "repo"
                       :backup-path "dir-at-repo"
                       :pg-host "localhost"
@@ -17,12 +18,21 @@
                       :pg-user "user"
                       :pg-password "password"}))))
 
-(deftest should-calculate-db-drop-create-command
+(deftest should-calculate-db-drop-command
   (is (= [["psql" "-d" "template1" "-h" "localhost" "-p" "5432" "-U" "user"
-           "--no-password" "-c" "\"DROP DATABASE \\\"mydb\\\";\""]
+           "--no-password" "-c" "DROP DATABASE \"mydb\";"]]
-          ["psql" "-d" "template1" "-h" "localhost" "-p" "5432" "-U" "user" 
+         (cut/db-drop-command {:restic-repository "repo"
-           "--no-password" "-c" "\"CREATE DATABASE \\\"mydb\\\";\""]]
+                                      :backup-path "dir-at-repo"
-         (cut/db-drop-create-command {:restic-repository "repo"
+                                      :pg-host "localhost"
+                                      :pg-port 5432
+                                      :pg-db "mydb"
+                                      :pg-user "user"
+                                      :pg-password "password"}))))
+
+(deftest should-calculate-db-create-command
+  (is (= [["psql" "-d" "template1" "-h" "localhost" "-p" "5432" "-U" "user"
+           "--no-password" "-c" "CREATE DATABASE \"mydb\";"]]
+         (cut/db-create-command {:restic-repository "repo"
                                       :backup-path "dir-at-repo"
                                       :pg-host "localhost"
                                       :pg-port 5432