add postgre

.gitignore

@@ -26,6 +26,12 @@
 !src/
 !test/
 
+# ------------------------
+# Include backup container
+!infrastructure/
+infrastructure/backup/__pycache__
+infrastructure/backup/.pybuilder
+
 # ------------------------
 # Include Clojure tools
 !.cljstyle

infrastructure/backup/build.py

@@ -0,0 +1,65 @@
+from os import environ
+from datetime import datetime
+from pybuilder.core import task, init
+from ddadevops import *
+from ddadevops.infrastructure import FileApi, ExecutionApi
+import logging
+
+name = 'dda-backup'
+MODULE = 'NOT_SET'
+PROJECT_ROOT_PATH = '../..'
+version = "5.0.0-dev"
+
+
+@init
+def initialize(project):
+    image_tag = version
+    if "dev" in image_tag:
+        image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+
+    input = {
+        "name": name,
+        "module": MODULE,
+        "stage": "notused",
+        "project_root_path": PROJECT_ROOT_PATH,
+        "build_types": ["IMAGE"],
+        "mixin_types": [],
+        "image_naming": "NAME_ONLY",
+        "image_tag": f"{image_tag}",
+    }
+
+    project.build_depends_on("ddadevops>=4.7.0")
+
+    build = DevopsImageBuild(project, input)
+    build.initialize_build_dir()
+    file_api = FileApi()
+    e_api = ExecutionApi()
+    e_api.execute(f"mkdir {build.build_path()}/image/local/")
+    file_api.cp("../../deps.edn", f"{build.build_path()}/image/local/")
+    file_api.cp_recursive("../../src", f"{build.build_path()}/image/local/src")
+    e_api.execute(f"mkdir {build.build_path()}/test/local/")
+    file_api.cp("../../deps.edn", f"{build.build_path()}/test/local/")
+    file_api.cp_recursive("../../src", f"{build.build_path()}/test/local/src")
+
+
+@task
+def image(project):
+    build = get_devops_build(project)
+    build.image()
+
+@task
+def test(project):
+    build = get_devops_build(project)
+    build.test()
+
+@task
+def drun(project):
+    build = get_devops_build(project)
+    build.drun()
+
+
+@task
+def publish(project):
+    build = get_devops_build(project)
+    build.dockerhub_login()
+    build.dockerhub_publish()

infrastructure/backup/doc/backup_dev_notes.md

@@ -0,0 +1,79 @@
+## Init Statemachine
+
+### Inputs
+1. `restic-password: ""`
+2. `restic-password-to-rotate: ""`
+
+### Manual init the restic repository for the first time
+
+1. apply backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=1`
+2. exec into pod and execute restore pod (press tab to get your exact pod name)   
+   `kubectl exec -it backup-restore-... -- /usr/local/bin/init.sh`
+3. remove backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=0`
+
+### Password Rotation
+
+1. apply backup-and-restore pod:   
+   `kubectl scale deployment backup-restore --replicas=1`
+2. add new password to restic repository   
+    `restic key add ....`   
+    => Trigger ::   
+    field (1) credential current   
+    filed (2) credential new
+3. replace field (1) with (2) & clear (2)
+4. remove old key - ???  
+`restic remove ....`
+
+
+```mermaid
+stateDiagram-v2
+    [*] --> init
+    init --> backup_ready: trigger, restic-password !empty
+    backup_ready --> new_password_added: restic-password !empty && restic-password-to-rotate !empty 
+    new_password_added --> backup_ready: restic-password !empty && restic-password-to-rotate empty
+```
+
+### First Steps
+
+1. Cloud Testserver hochfahren
+2. Dort backup-restore deployment (leeres Secret mgl.?), neues Secret "rotation-credential-secret" als Daten
+3. mounten von angelegtem Secret in Pod backup-restore
+4. ba*bash*ka Skript in pod starten -> liest Secret ?leer
+5. Micha cons. 
+
+```mermaid
+sequenceDiagram
+    participant k8s
+    participant e as entrypoint.sh
+    participant rm as restic-management.clj
+
+    k8s ->> e: cronjob calls
+    e ->> rm: start-file
+    rm ->> rm: rotate
+    activate rm
+        rm ->> rm: read-backup-repository-state (state)
+        rm ->> rm: read-secret (backup-secret/restic-password,  rotation-credential-secret/rotation-credential)
+        rm ->> rm: switch
+        activate rm
+        rm ->> rm: if init && restic-password != null
+        activate rm
+        rm ->> rm: init.sh
+        rm ->> rm: state init -> backup-ready
+        deactivate rm
+        rm ->> rm: if backup-ready && rotation-credential != null
+        activate rm
+        rm ->> rm: add-new-password-to-restic-repository.sh
+        rm ->> rm: state backup-ready -> new-password-added
+        deactivate rm
+        rm ->> rm: if new-password-added && rotation-credential == null
+        activate rm
+        rm ->> rm: remove-old-password-from-restic-repository.sh
+        rm ->> rm: state new-password-added -> backup-ready
+        deactivate rm
+        deactivate rm
+
+        rm ->> rm: store-repository-state (state)
+    deactivate rm
+```

infrastructure/backup/image/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:jammy
+
+# install it
+ADD resources /tmp/
+RUN /tmp/install.sh
+ADD local/ /usr/local/lib/dda-backup

infrastructure/backup/image/resources/install.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -exo pipefail
+
+function babashka_install() {
+    babashka_version="1.3.189"
+    curl -SsLo /tmp/babashka-${babashka_version}-linux-amd64.tar.gz https://github.com/babashka/babashka/releases/download/v${babashka_version}/babashka-${babashka_version}-linux-amd64.tar.gz
+    curl -SsLo /tmp/checksum https://github.com/babashka/babashka/releases/download/v${babashka_version}/babashka-${babashka_version}-linux-amd64.tar.gz.sha256
+    echo "  /tmp/babashka-$babashka_version-linux-amd64.tar.gz"|tee -a /tmp/checksum
+    sha256sum -c --status /tmp/checksum
+    tar -C /tmp -xzf /tmp/babashka-${babashka_version}-linux-amd64.tar.gz 
+    install -m 0700 -o root -g root /tmp/bb /usr/local/bin/
+}
+
+function main() {
+    {
+        upgradeSystem
+        apt-get install -qqy ca-certificates curl gnupg postgresql-client-14 restic
+        curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg
+        sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        upgradeSystem
+        babashka_install        
+    } > /dev/null
+
+    update-ca-certificates
+
+    cleanupDocker
+}
+
+source /tmp/install_functions_debian.sh
+DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes main

infrastructure/backup/test/Dockerfile

@@ -0,0 +1,7 @@
+FROM c4k-forgejo-backup:latest
+
+# install it
+RUN apt update && apt install -qqy openjdk-17-jre-headless
+ADD local/ /usr/local/lib/dda-backup
+ADD resources /tmp/
+RUN /tmp/test.bb

infrastructure/backup/test/resources/bb.edn

@@ -0,0 +1,3 @@
+{:deps {org.clojure/spec.alpha {:mvn/version "0.4.233"}
+        orchestra/orchestra {:mvn/version "2021.01.01-1"}
+        org.domaindrivenarchitecture/dda-backup {:local/root "/usr/local/lib/dda-backup"}}}

infrastructure/backup/test/resources/install-test.bb

@@ -0,0 +1,31 @@
+#!/usr/bin/env bb
+
+(require '[babashka.tasks :as tasks])
+
+(defn curl-and-check!
+  [filename artifact-url sha256-url]
+  (let [filepath (str "/tmp/" filename)]
+    (tasks/shell "curl" "-SsLo" filepath artifact-url)
+    (tasks/shell "curl" "-SsLo" "/tmp/checksum" sha256-url)
+    (tasks/shell "bash" "-c" (str "echo \" " filepath "\"|tee -a /tmp/checksum"))
+    ;(tasks/shell "sha256sum" "-c" "--status" "/tmp/checksum")
+    ))
+
+(defn tar-install!
+  [filename binname]
+  (let [filepath (str "/tmp/" filename)]
+    (tasks/shell "tar" "-C" "/tmp" "-xzf" filepath)
+    (tasks/shell "install" "-m" "0700" "-o" "root" "-g" "root" (str "/tmp/" binname) "/usr/local/bin/")))
+
+(defn install!
+  [filename]
+  (tasks/shell "install" "-m" "0700" "-o" "root" "-g" "root" (str "/tmp/" filename) "/usr/local/bin/"))
+
+(curl-and-check!
+ "provs-syspec.jar"
+ "https://repo.prod.meissa.de/attachments/0a1da41e-aa5b-4a3e-a3b1-215cf2d5b021"
+ "https://repo.prod.meissa.de/attachments/f227cf65-cb0f-46a7-a6cd-28f46917412a")
+(install! "provs-syspec.jar")
+(tasks/shell "apt" "update")
+(tasks/shell "apt" "install" "-qqy" "openjdk-17-jre-headless")
+(tasks/shell "java" "-jar" "/usr/local/bin/provs-syspec.jar" "local" "-c" "/tmp/spec.yml" )

infrastructure/backup/test/resources/spec.yml

@@ -0,0 +1,7 @@
+package:
+  - name: "restic"
+
+command:
+  - command: "bb -h"
+  - command: "/tmp/test.bb"
+  

infrastructure/backup/test/resources/test.bb

@@ -0,0 +1,31 @@
+#!/usr/bin/env bb
+
+(require '[babashka.tasks :as tasks]
+         '[dda.backup.management :as mgm]
+         '[dda.backup.backup :as bak]
+         '[dda.backup.restore :as rs])
+
+(defn restic-repo-init!
+  []
+  (spit "restic-pwd" "ThePassword")
+  (mgm/init! {:password-file "restic-pwd"
+              :restic-repository "restic-repo"}))
+
+(defn restic-backup!
+  []
+  (tasks/shell "mkdir" "-p" "test-backup")
+  (spit "test-backup/file" "I was here")
+  (bak/backup! {:password-file "restic-pwd"
+                :restic-repository "restic-repo"
+                :files ["test-backup"]}))
+
+(defn restic-restore!
+  []
+  (tasks/shell "mkdir" "-p" "test-restore")
+  (rs/restore! {:password-file "restic-pwd"
+                :restic-repository "restic-repo"
+                :target-directory "test-restore"}))
+
+(restic-repo-init!)
+(restic-backup!)
+(restic-restore!)

src/dda/backup/postgresql.clj

@@ -0,0 +1,92 @@
+(ns dda.backup.postgres
+  (:require
+   [orchestra.core :refer [defn-spec]]
+   [clojure.spec.alpha :as s]
+   [dda.backup.core.domain :as cd]
+   [dda.backup.management.domain :as domain]
+   [dda.backup.core :as core]
+   [dda.backup.infrastructure :as i]))
+
+;; function drop-create-db() {
+;;   psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password -c "DROP DATABASE \"${POSTGRES_DB}\";"
+;;   psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password -c "CREATE DATABASE \"${POSTGRES_DB}\";"
+;; }
+
+;; function create-pg-pass() {
+;;   local pg_host=${POSTGRES_HOST:-localhost}
+
+;;   echo "${pg_host}:${POSTGRES_DB}:${POSTGRES_USER}:${POSTGRES_PASSWORD}" > /root/.pgpass
+;; 	echo "${POSTGRES_HOST}:template1:${POSTGRES_USER}:${POSTGRES_PASSWORD}" >> /root/.pgpass
+;; 	chmod 0600 /root/.pgpass
+;; }
+
+;; function backup-roles() {
+;;   local role_prefix="$1"; shift
+
+;;   if [ -z ${CERTIFICATE_FILE} ];
+;;   then
+;;     roles-unlock-command
+;;     pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
+;;       grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin
+;;     roles-forget-command
+;;   else
+;;     roles-unlock-command --cacert ${CERTIFICATE_FILE}
+;;     pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
+;;       grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin --cacert ${CERTIFICATE_FILE}
+;;     roles-forget-command --cacert ${CERTIFICATE_FILE}
+;;   fi
+;; }
+
+;; function backup-db-dump() {
+
+;;   if [ -z ${CERTIFICATE_FILE} ];
+;;   then
+;;     db-unlock-command
+;;     pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
+;;       -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
+;;       restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin
+;;     db-forget-command
+;;   else
+;;     db-unlock-command --cacert ${CERTIFICATE_FILE}
+;;     pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
+;;       -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
+;;       restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin --cacert ${CERTIFICATE_FILE}
+;;     db-forget-command --cacert ${CERTIFICATE_FILE}
+;;   fi
+;; }
+
+;; function restore-roles() {
+;;   local snapshot_id="${1:-latest}"; shift
+
+;;   if [ -z ${CERTIFICATE_FILE} ];
+;;   then
+;;     roles-unlock-command
+;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
+;;       psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password
+;;   else
+;;     roles-unlock-command --cacert ${CERTIFICATE_FILE}
+;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin  --cacert ${CERTIFICATE_FILE} | \
+;;       psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password
+;;   fi
+;; }
+
+;; function restore-db() {
+;;   local snapshot_id="${1:-latest}"; shift
+  
+;;   if [ -z ${CERTIFICATE_FILE} ];
+;;   then
+;;     db-unlock-command
+;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin | \
+;;       psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password
+;;   else
+;;     db-unlock-command --cacert ${CERTIFICATE_FILE}
+;;     restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
+;;       psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+;;       --no-password
+;;   fi
+;; }