move backup image to devops-build

.gitlab-ci.yml

@@ -87,3 +87,10 @@ kotlin-image-publish:
   stage: image
   script:
     - cd infrastructure/kotlin && pyb image publish
+
+backup-image-publish:
+  <<: *img
+  <<: *tag_only
+  stage: image
+  script:
+    - cd infrastructure/backup && pyb image publish

build.py

@@ -97,6 +97,7 @@ def initialize(project):
         "mixin_types": ["RELEASE"],
         "release_primary_build_file": "build.py",
         "release_secondary_build_files": [
+            "infrastructure/backup/build.py",
             "infrastructure/python/build.py",
             "infrastructure/dind/build.py",
             "infrastructure/ddadevops/build.py",

infrastructure/backup/build.py

@@ -0,0 +1,51 @@
+from os import environ
+from datetime import datetime
+from pybuilder.core import task, init
+from ddadevops import *
+import logging
+
+name = 'dda-backup'
+MODULE = 'NOT_SET'
+PROJECT_ROOT_PATH = '../..'
+version = "4.11.8-dev"
+
+
+@init
+def initialize(project):
+    image_tag = version
+    if "dev" in image_tag:
+        image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+
+    input = {
+        "name": name,
+        "module": MODULE,
+        "stage": "notused",
+        "project_root_path": PROJECT_ROOT_PATH,
+        "build_types": ["IMAGE"],
+        "mixin_types": [],
+        "image_naming": "NAME_ONLY",
+        "image_tag": f"{image_tag}",
+    }
+
+    project.build_depends_on("ddadevops>=4.7.0")
+
+    build = DevopsImageBuild(project, input)
+    build.initialize_build_dir()
+
+
+@task
+def image(project):
+    build = get_devops_build(project)
+    build.image()
+
+@task
+def drun(project):
+    build = get_devops_build(project)
+    build.drun()
+
+
+@task
+def publish(project):
+    build = get_devops_build(project)
+    build.dockerhub_login()
+    build.dockerhub_publish()

infrastructure/backup/image/Dockerfile

@@ -0,0 +1,5 @@
+FROM ubuntu:jammy
+
+# install it
+ADD resources /tmp/
+RUN /tmp/install.sh

infrastructure/backup/image/resources/file-functions.sh

@@ -0,0 +1,69 @@
+backup_file_path='files'
+
+function init-file-repo() {
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -r ${RESTIC_REPOSITORY}/${backup_file_path} -v init
+  else
+    restic -r ${RESTIC_REPOSITORY}/${backup_file_path} -v init --cacert ${CERTIFICATE_FILE}
+  fi
+}
+
+# First arg is the directory, second is optional for the path to a certificate file
+function backup-directory() {
+  local directory="$1"; shift
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache
+    cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup .
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune
+  else
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache --cacert ${CERTIFICATE_FILE}
+    cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup . --cacert ${CERTIFICATE_FILE}
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune --cacert ${CERTIFICATE_FILE}
+  fi
+}
+
+# First arg is the directory, the remaining args are the sub-directories (relative to the first directory) to backup.
+function backup-fs-from-directory() {
+  local directory="$1"; shift
+  
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache
+    cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup $@
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune
+  else
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache --cacert ${CERTIFICATE_FILE}
+    cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup $@ --cacert ${CERTIFICATE_FILE}
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune --cacert ${CERTIFICATE_FILE}
+  fi
+  
+}
+
+function restore-directory() {
+  local directory="$1"; shift
+  local snapshot_id="${1:-latest}"; shift
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache
+    rm -rf ${directory}*
+    restic -v -r $RESTIC_REPOSITORY/${backup_file_path} restore ${snapshot_id} --target ${directory}
+  else
+    restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache --cacert ${CERTIFICATE_FILE}
+    rm -rf ${directory}*
+    restic -v -r $RESTIC_REPOSITORY/${backup_file_path} restore ${snapshot_id} --target ${directory} --cacert ${CERTIFICATE_FILE}
+  fi
+  
+}
+
+function list-snapshot-files() {
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots
+  else
+    restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots --cacert ${CERTIFICATE_FILE}
+  fi
+}

infrastructure/backup/image/resources/functions.sh

@@ -0,0 +1,21 @@
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+function file_env() {
+  local var="$1"
+  local fileVar="${var}_FILE"
+  local def="${2:-}"
+  if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+    echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+    exit 1
+  fi
+  local val="$def"
+  if [ "${!var:-}" ]; then
+    val="${!var}"
+  elif [ "${!fileVar:-}" ]; then
+    val="$(< "${!fileVar}")"
+  fi
+  export "$var"="$val"
+  unset "$fileVar"
+}

infrastructure/backup/image/resources/install.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -exo pipefail
+
+function main() {
+    {
+        upgradeSystem
+        apt-get install -qqy ca-certificates curl gnupg postgresql-client-14
+        curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg
+        sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        upgradeSystem
+    } > /dev/null
+
+    update-ca-certificates
+
+    install -m 0400 /tmp/functions.sh /usr/local/lib/
+    install -m 0400 /tmp/pg-functions.sh /usr/local/lib/
+    install -m 0400 /tmp/file-functions.sh /usr/local/lib/
+
+    cleanupDocker
+}
+
+source /tmp/install_functions_debian.sh
+DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes main

infrastructure/backup/image/resources/pg-functions.sh

@@ -0,0 +1,149 @@
+backup_pg_role_path='pg-role'
+backup_pg_database_path='pg-database'
+
+function init-command() {
+  restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} -v init $@
+}
+
+function init-role-repo() {
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    init-command
+  else
+    init-command --cacert ${CERTIFICATE_FILE}
+  fi
+  
+}
+
+function init-database-command() {
+  restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} -v init $@
+}
+
+function init-database-repo() {
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    init-database-command 
+  else
+    init-database-command --cacert ${CERTIFICATE_FILE}
+  fi
+}
+
+function drop-create-db() {
+  psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+      --no-password -c "DROP DATABASE \"${POSTGRES_DB}\";"
+  psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+      --no-password -c "CREATE DATABASE \"${POSTGRES_DB}\";"
+}
+
+function create-pg-pass() {
+  local pg_host=${POSTGRES_HOST:-localhost}
+
+  echo "${pg_host}:${POSTGRES_DB}:${POSTGRES_USER}:${POSTGRES_PASSWORD}" > /root/.pgpass
+	echo "${POSTGRES_HOST}:template1:${POSTGRES_USER}:${POSTGRES_PASSWORD}" >> /root/.pgpass
+	chmod 0600 /root/.pgpass
+}
+
+function roles-unlock-command() {
+  restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} unlock --cleanup-cache $@
+}
+
+function roles-forget-command() {
+  restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune $@
+}
+
+function backup-roles() {
+  local role_prefix="$1"; shift
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    roles-unlock-command
+    pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
+      grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin
+    roles-forget-command
+  else
+    roles-unlock-command --cacert ${CERTIFICATE_FILE}
+    pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
+      grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin --cacert ${CERTIFICATE_FILE}
+    roles-forget-command --cacert ${CERTIFICATE_FILE}
+  fi
+}
+
+function db-unlock-command() {
+  restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} unlock --cleanup-cache $@
+}
+
+function db-forget-command() {
+  restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune $@
+}
+
+function backup-db-dump() {
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    db-unlock-command
+    pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
+      -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
+      restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin
+    db-forget-command
+  else
+    db-unlock-command --cacert ${CERTIFICATE_FILE}
+    pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
+      -U ${POSTGRES_USER} --no-password --serializable-deferrable | \
+      restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin --cacert ${CERTIFICATE_FILE}
+    db-forget-command --cacert ${CERTIFICATE_FILE}
+  fi
+}
+
+function restore-roles() {
+  local snapshot_id="${1:-latest}"; shift
+
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    roles-unlock-command
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
+      psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+      --no-password
+  else
+    roles-unlock-command --cacert ${CERTIFICATE_FILE}
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin  --cacert ${CERTIFICATE_FILE} | \
+      psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+      --no-password
+  fi
+}
+
+function restore-db() {
+  local snapshot_id="${1:-latest}"; shift
+  
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    db-unlock-command
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin | \
+      psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+      --no-password
+  else
+    db-unlock-command --cacert ${CERTIFICATE_FILE}
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
+      psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
+      --no-password
+  fi
+}
+
+function list-snapshot-roles() {
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} snapshots
+  else
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} snapshots --cacert ${CERTIFICATE_FILE}
+  fi
+}
+
+function list-snapshot-db() {
+  if [ -z ${CERTIFICATE_FILE} ];
+  then
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} snapshots
+  else
+    restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} snapshots --cacert ${CERTIFICATE_FILE}
+  fi
+}