move backup image to devops-build
parent
4034b0022b
commit
e96581754c
@ -0,0 +1,51 @@
|
||||
from os import environ
|
||||
from datetime import datetime
|
||||
from pybuilder.core import task, init
|
||||
from ddadevops import *
|
||||
import logging
|
||||
|
||||
name = 'dda-backup'
|
||||
MODULE = 'NOT_SET'
|
||||
PROJECT_ROOT_PATH = '../..'
|
||||
version = "4.11.8-dev"
|
||||
|
||||
|
||||
@init
|
||||
def initialize(project):
|
||||
image_tag = version
|
||||
if "dev" in image_tag:
|
||||
image_tag += datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
|
||||
|
||||
input = {
|
||||
"name": name,
|
||||
"module": MODULE,
|
||||
"stage": "notused",
|
||||
"project_root_path": PROJECT_ROOT_PATH,
|
||||
"build_types": ["IMAGE"],
|
||||
"mixin_types": [],
|
||||
"image_naming": "NAME_ONLY",
|
||||
"image_tag": f"{image_tag}",
|
||||
}
|
||||
|
||||
project.build_depends_on("ddadevops>=4.7.0")
|
||||
|
||||
build = DevopsImageBuild(project, input)
|
||||
build.initialize_build_dir()
|
||||
|
||||
|
||||
@task
|
||||
def image(project):
|
||||
build = get_devops_build(project)
|
||||
build.image()
|
||||
|
||||
@task
|
||||
def drun(project):
|
||||
build = get_devops_build(project)
|
||||
build.drun()
|
||||
|
||||
|
||||
@task
|
||||
def publish(project):
|
||||
build = get_devops_build(project)
|
||||
build.dockerhub_login()
|
||||
build.dockerhub_publish()
|
||||
@ -0,0 +1,5 @@
|
||||
FROM ubuntu:jammy
|
||||
|
||||
# install it
|
||||
ADD resources /tmp/
|
||||
RUN /tmp/install.sh
|
||||
@ -0,0 +1,69 @@
|
||||
backup_file_path='files'
|
||||
|
||||
function init-file-repo() {
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} -v init
|
||||
else
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} -v init --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
|
||||
# First arg is the directory, second is optional for the path to a certificate file
|
||||
function backup-directory() {
|
||||
local directory="$1"; shift
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache
|
||||
cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup .
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune
|
||||
else
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache --cacert ${CERTIFICATE_FILE}
|
||||
cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup . --cacert ${CERTIFICATE_FILE}
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
|
||||
# First arg is the directory, the remaining args are the sub-directories (relative to the first directory) to backup.
|
||||
function backup-fs-from-directory() {
|
||||
local directory="$1"; shift
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache
|
||||
cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup $@
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune
|
||||
else
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache --cacert ${CERTIFICATE_FILE}
|
||||
cd ${directory} && restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} backup $@ --cacert ${CERTIFICATE_FILE}
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function restore-directory() {
|
||||
local directory="$1"; shift
|
||||
local snapshot_id="${1:-latest}"; shift
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache
|
||||
rm -rf ${directory}*
|
||||
restic -v -r $RESTIC_REPOSITORY/${backup_file_path} restore ${snapshot_id} --target ${directory}
|
||||
else
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_file_path} unlock --cleanup-cache --cacert ${CERTIFICATE_FILE}
|
||||
rm -rf ${directory}*
|
||||
restic -v -r $RESTIC_REPOSITORY/${backup_file_path} restore ${snapshot_id} --target ${directory} --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function list-snapshot-files() {
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots
|
||||
else
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_file_path} snapshots --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
# usage: file_env VAR [DEFAULT]
|
||||
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
|
||||
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
|
||||
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
|
||||
function file_env() {
|
||||
local var="$1"
|
||||
local fileVar="${var}_FILE"
|
||||
local def="${2:-}"
|
||||
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
|
||||
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
|
||||
exit 1
|
||||
fi
|
||||
local val="$def"
|
||||
if [ "${!var:-}" ]; then
|
||||
val="${!var}"
|
||||
elif [ "${!fileVar:-}" ]; then
|
||||
val="$(< "${!fileVar}")"
|
||||
fi
|
||||
export "$var"="$val"
|
||||
unset "$fileVar"
|
||||
}
|
||||
@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -exo pipefail
|
||||
|
||||
function main() {
|
||||
{
|
||||
upgradeSystem
|
||||
apt-get install -qqy ca-certificates curl gnupg postgresql-client-14
|
||||
curl -Ss --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg
|
||||
sh -c 'echo "deb [signed-by=/etc/apt/trusted.gpg.d/postgresql-common_pgdg_archive_keyring.gpg] https://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
|
||||
upgradeSystem
|
||||
} > /dev/null
|
||||
|
||||
update-ca-certificates
|
||||
|
||||
install -m 0400 /tmp/functions.sh /usr/local/lib/
|
||||
install -m 0400 /tmp/pg-functions.sh /usr/local/lib/
|
||||
install -m 0400 /tmp/file-functions.sh /usr/local/lib/
|
||||
|
||||
cleanupDocker
|
||||
}
|
||||
|
||||
source /tmp/install_functions_debian.sh
|
||||
DEBIAN_FRONTEND=noninteractive DEBCONF_NOWARNINGS=yes main
|
||||
@ -0,0 +1,149 @@
|
||||
backup_pg_role_path='pg-role'
|
||||
backup_pg_database_path='pg-database'
|
||||
|
||||
function init-command() {
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} -v init $@
|
||||
}
|
||||
|
||||
function init-role-repo() {
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
init-command
|
||||
else
|
||||
init-command --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function init-database-command() {
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} -v init $@
|
||||
}
|
||||
|
||||
function init-database-repo() {
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
init-database-command
|
||||
else
|
||||
init-database-command --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
|
||||
function drop-create-db() {
|
||||
psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
--no-password -c "DROP DATABASE \"${POSTGRES_DB}\";"
|
||||
psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
--no-password -c "CREATE DATABASE \"${POSTGRES_DB}\";"
|
||||
}
|
||||
|
||||
function create-pg-pass() {
|
||||
local pg_host=${POSTGRES_HOST:-localhost}
|
||||
|
||||
echo "${pg_host}:${POSTGRES_DB}:${POSTGRES_USER}:${POSTGRES_PASSWORD}" > /root/.pgpass
|
||||
echo "${POSTGRES_HOST}:template1:${POSTGRES_USER}:${POSTGRES_PASSWORD}" >> /root/.pgpass
|
||||
chmod 0600 /root/.pgpass
|
||||
}
|
||||
|
||||
function roles-unlock-command() {
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} unlock --cleanup-cache $@
|
||||
}
|
||||
|
||||
function roles-forget-command() {
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune $@
|
||||
}
|
||||
|
||||
function backup-roles() {
|
||||
local role_prefix="$1"; shift
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
roles-unlock-command
|
||||
pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
|
||||
grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin
|
||||
roles-forget-command
|
||||
else
|
||||
roles-unlock-command --cacert ${CERTIFICATE_FILE}
|
||||
pg_dumpall -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U${POSTGRES_USER} --no-password --roles-only | \
|
||||
grep ${role_prefix} | restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} backup --stdin --cacert ${CERTIFICATE_FILE}
|
||||
roles-forget-command --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
|
||||
function db-unlock-command() {
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} unlock --cleanup-cache $@
|
||||
}
|
||||
|
||||
function db-forget-command() {
|
||||
restic -v -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} forget --group-by '' --keep-last 1 --keep-daily ${RESTIC_DAYS_TO_KEEP} --keep-monthly ${RESTIC_MONTHS_TO_KEEP} --prune $@
|
||||
}
|
||||
|
||||
function backup-db-dump() {
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
db-unlock-command
|
||||
pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
|
||||
-U ${POSTGRES_USER} --no-password --serializable-deferrable | \
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin
|
||||
db-forget-command
|
||||
else
|
||||
db-unlock-command --cacert ${CERTIFICATE_FILE}
|
||||
pg_dump -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} \
|
||||
-U ${POSTGRES_USER} --no-password --serializable-deferrable | \
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} backup --stdin --cacert ${CERTIFICATE_FILE}
|
||||
db-forget-command --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
|
||||
function restore-roles() {
|
||||
local snapshot_id="${1:-latest}"; shift
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
roles-unlock-command
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin | \
|
||||
psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
--no-password
|
||||
else
|
||||
roles-unlock-command --cacert ${CERTIFICATE_FILE}
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
|
||||
psql -d template1 -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
--no-password
|
||||
fi
|
||||
}
|
||||
|
||||
function restore-db() {
|
||||
local snapshot_id="${1:-latest}"; shift
|
||||
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
db-unlock-command
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin | \
|
||||
psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
--no-password
|
||||
else
|
||||
db-unlock-command --cacert ${CERTIFICATE_FILE}
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} dump ${snapshot_id} stdin --cacert ${CERTIFICATE_FILE} | \
|
||||
psql -d ${POSTGRES_DB} -h ${POSTGRES_SERVICE} -p ${POSTGRES_PORT} -U ${POSTGRES_USER} \
|
||||
--no-password
|
||||
fi
|
||||
}
|
||||
|
||||
function list-snapshot-roles() {
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_role_path} snapshots
|
||||
else
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} snapshots --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
|
||||
function list-snapshot-db() {
|
||||
if [ -z ${CERTIFICATE_FILE} ];
|
||||
then
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} snapshots
|
||||
else
|
||||
restic -r ${RESTIC_REPOSITORY}/${backup_pg_database_path} snapshots --cacert ${CERTIFICATE_FILE}
|
||||
fi
|
||||
}
|
||||
Loading…
Reference in New Issue