Merge branch 'main' into password-rotation
This commit is contained in:
commit
9c1bb12c53
5 changed files with 83 additions and 57 deletions
src
main/cljc/dda/c4k_forgejo
test/cljc/dda/c4k_forgejo
|
@ -1,6 +1,8 @@
|
||||||
(ns dda.c4k-forgejo.backup
|
(ns dda.c4k-forgejo.backup
|
||||||
(:require
|
(:require
|
||||||
[clojure.spec.alpha :as s]
|
[clojure.spec.alpha :as s]
|
||||||
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
[dda.c4k-common.yaml :as yaml]
|
[dda.c4k-common.yaml :as yaml]
|
||||||
[dda.c4k-common.base64 :as b64]
|
[dda.c4k-common.base64 :as b64]
|
||||||
[dda.c4k-common.common :as cm]
|
[dda.c4k-common.common :as cm]
|
||||||
|
@ -12,26 +14,32 @@
|
||||||
(s/def ::restic-password p/bash-env-string?)
|
(s/def ::restic-password p/bash-env-string?)
|
||||||
(s/def ::restic-repository p/bash-env-string?)
|
(s/def ::restic-repository p/bash-env-string?)
|
||||||
|
|
||||||
|
(s/def ::config (s/keys :req-un [::restic-repository]))
|
||||||
|
|
||||||
|
(s/def ::auth (s/keys :req-un [::restic-password ::aws-access-key-id ::aws-secret-access-key]
|
||||||
|
:opt-un [::restic-new-password]))
|
||||||
|
|
||||||
#?(:cljs
|
#?(:cljs
|
||||||
(defmethod yaml/load-resource :backup [resource-name]
|
(defmethod yaml/load-resource :backup [resource-name]
|
||||||
(get (inline-resources "backup") resource-name)))
|
(get (inline-resources "backup") resource-name)))
|
||||||
|
|
||||||
(defn generate-config [my-conf]
|
(defn-spec generate-config p/map-or-seq?
|
||||||
|
[my-conf ::config]
|
||||||
(let [{:keys [restic-repository]} my-conf]
|
(let [{:keys [restic-repository]} my-conf]
|
||||||
(->
|
(->
|
||||||
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
|
(yaml/from-string (yaml/load-resource "backup/config.yaml"))
|
||||||
(cm/replace-key-value :restic-repository restic-repository))))
|
(cm/replace-key-value :restic-repository restic-repository))))
|
||||||
|
|
||||||
(defn generate-cron []
|
(defn-spec generate-cron p/map-or-seq?
|
||||||
|
[]
|
||||||
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
|
(yaml/from-string (yaml/load-resource "backup/cron.yaml")))
|
||||||
|
|
||||||
(defn generate-backup-restore-deployment [my-conf]
|
(defn-spec generate-backup-restore-deployment p/map-or-seq?
|
||||||
(let [backup-restore-yaml (yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml"))]
|
[my-conf ::config]
|
||||||
(if (and (contains? my-conf :local-integration-test) (= true (:local-integration-test my-conf)))
|
(yaml/from-string (yaml/load-resource "backup/backup-restore-deployment.yaml")))
|
||||||
(cm/replace-named-value backup-restore-yaml "CERTIFICATE_FILE" "/var/run/secrets/localstack-secrets/ca.crt")
|
|
||||||
backup-restore-yaml)))
|
|
||||||
|
|
||||||
(defn generate-secret [my-auth]
|
(defn-spec generate-secret p/map-or-seq?
|
||||||
|
[my-auth ::auth]
|
||||||
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
|
(let [{:keys [aws-access-key-id aws-secret-access-key restic-password]} my-auth]
|
||||||
(->
|
(->
|
||||||
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
|
(yaml/from-string (yaml/load-resource "backup/secret.yaml"))
|
||||||
|
|
|
@ -1,8 +1,11 @@
|
||||||
(ns dda.c4k-forgejo.core
|
(ns dda.c4k-forgejo.core
|
||||||
(:require
|
(:require
|
||||||
[clojure.spec.alpha :as s]
|
[clojure.spec.alpha :as s]
|
||||||
|
#?(:clj [orchestra.core :refer [defn-spec]]
|
||||||
|
:cljs [orchestra.core :refer-macros [defn-spec]])
|
||||||
[dda.c4k-common.yaml :as yaml]
|
[dda.c4k-common.yaml :as yaml]
|
||||||
[dda.c4k-common.common :as cm]
|
[dda.c4k-common.common :as cm]
|
||||||
|
[dda.c4k-common.predicate :as p]
|
||||||
[dda.c4k-common.monitoring :as mon]
|
[dda.c4k-common.monitoring :as mon]
|
||||||
[dda.c4k-forgejo.forgejo :as forgejo]
|
[dda.c4k-forgejo.forgejo :as forgejo]
|
||||||
[dda.c4k-forgejo.backup :as backup]
|
[dda.c4k-forgejo.backup :as backup]
|
||||||
|
@ -18,8 +21,10 @@
|
||||||
:pvc-storage-class-name ""
|
:pvc-storage-class-name ""
|
||||||
:postgres-image "postgres:14"
|
:postgres-image "postgres:14"
|
||||||
:postgres-size :2gb})
|
:postgres-size :2gb})
|
||||||
|
|
||||||
(def rate-limit-defaults {:max-rate 10, :max-concurrent-requests 5})
|
(def rate-limit-defaults {:max-rate 10, :max-concurrent-requests 5})
|
||||||
|
|
||||||
|
|
||||||
(def config? (s/keys :req-un [::forgejo/fqdn
|
(def config? (s/keys :req-un [::forgejo/fqdn
|
||||||
::forgejo/mailer-from
|
::forgejo/mailer-from
|
||||||
::forgejo/mailer-host
|
::forgejo/mailer-host
|
||||||
|
@ -28,7 +33,7 @@
|
||||||
:opt-un [::forgejo/issuer
|
:opt-un [::forgejo/issuer
|
||||||
::forgejo/deploy-federated
|
::forgejo/deploy-federated
|
||||||
::forgejo/federation-enabled
|
::forgejo/federation-enabled
|
||||||
::forgejo/default-app-name
|
::forgejo/default-app-name
|
||||||
::forgejo/service-domain-whitelist
|
::forgejo/service-domain-whitelist
|
||||||
::forgejo/forgejo-image-version-overwrite
|
::forgejo/forgejo-image-version-overwrite
|
||||||
::backup/restic-repository
|
::backup/restic-repository
|
||||||
|
@ -37,12 +42,11 @@
|
||||||
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
|
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
|
||||||
::forgejo/mailer-user ::forgejo/mailer-pw
|
::forgejo/mailer-user ::forgejo/mailer-pw
|
||||||
::backup/aws-access-key-id ::backup/aws-secret-access-key]
|
::backup/aws-access-key-id ::backup/aws-secret-access-key]
|
||||||
:opt-un [::backup/restic-password ; TODO gec: Is restic password opt or req?
|
:opt-un [::backup/restic-password
|
||||||
::mon/mon-cfg]))
|
::mon/mon-auth]))
|
||||||
|
|
||||||
(def vol? (s/keys :req-un [::forgejo/volume-total-storage-size]))
|
(defn-spec config-objects p/map-or-seq?
|
||||||
|
[config config?]
|
||||||
(defn config-objects [config] ; ToDo: ADR for generate functions - vector or no vector?
|
|
||||||
(let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
|
(let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
|
||||||
(map yaml/to-string
|
(map yaml/to-string
|
||||||
(filter #(not (nil? %))
|
(filter #(not (nil? %))
|
||||||
|
@ -67,7 +71,9 @@
|
||||||
(when (contains? config :mon-cfg)
|
(when (contains? config :mon-cfg)
|
||||||
(mon/generate-config)))))))
|
(mon/generate-config)))))))
|
||||||
|
|
||||||
(defn auth-objects [config auth]
|
(defn-spec auth-objects p/map-or-seq?
|
||||||
|
[config config?
|
||||||
|
auth auth?]
|
||||||
(map yaml/to-string
|
(map yaml/to-string
|
||||||
(filter #(not (nil? %))
|
(filter #(not (nil? %))
|
||||||
(cm/concat-vec
|
(cm/concat-vec
|
||||||
|
|
|
@ -44,27 +44,25 @@
|
||||||
(s/def ::mailer-pw pred/bash-env-string?)
|
(s/def ::mailer-pw pred/bash-env-string?)
|
||||||
(s/def ::issuer pred/letsencrypt-issuer?)
|
(s/def ::issuer pred/letsencrypt-issuer?)
|
||||||
(s/def ::volume-total-storage-size (partial pred/int-gt-n? 5))
|
(s/def ::volume-total-storage-size (partial pred/int-gt-n? 5))
|
||||||
(s/def ::max-rate int?)
|
(s/def ::max-rate int?)
|
||||||
(s/def ::max-concurrent-requests int?)
|
(s/def ::max-concurrent-requests int?)
|
||||||
|
|
||||||
(def config? (s/keys :req-un [::fqdn
|
(s/def ::config (s/keys :req-un [::fqdn
|
||||||
::mailer-from
|
::mailer-from
|
||||||
::mailer-host
|
::mailer-host
|
||||||
::mailer-port
|
::mailer-port
|
||||||
::service-noreply-address]
|
::service-noreply-address
|
||||||
:opt-un [::issuer
|
::volume-total-storage-size
|
||||||
::deploy-federated
|
::max-rate
|
||||||
::federation-enabled
|
::max-concurrent-requests]
|
||||||
::default-app-name
|
:opt-un [::issuer
|
||||||
::service-domain-whitelist
|
::deploy-federated
|
||||||
::forgejo-image-version-overwrite]))
|
::federation-enabled
|
||||||
|
::default-app-name
|
||||||
|
::service-domain-whitelist
|
||||||
|
::forgejo-image-version-overwrite]))
|
||||||
|
|
||||||
(def rate-limit-config? (s/keys :req-un [::max-rate
|
(s/def ::auth (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw]))
|
||||||
::max-concurrent-requests]))
|
|
||||||
|
|
||||||
(def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw]))
|
|
||||||
|
|
||||||
(def vol? (s/keys :req-un [::volume-total-storage-size]))
|
|
||||||
|
|
||||||
(defn data-storage-by-volume-size
|
(defn data-storage-by-volume-size
|
||||||
[total]
|
[total]
|
||||||
|
@ -76,7 +74,7 @@
|
||||||
(def non-federated-image-version "8.0.3")
|
(def non-federated-image-version "8.0.3")
|
||||||
|
|
||||||
(defn-spec generate-image-str string?
|
(defn-spec generate-image-str string?
|
||||||
[config config?]
|
[config ::config]
|
||||||
(let [{:keys [deploy-federated forgejo-image-version-overwrite]} config
|
(let [{:keys [deploy-federated forgejo-image-version-overwrite]} config
|
||||||
deploy-federated-bool (boolean-from-string deploy-federated)]
|
deploy-federated-bool (boolean-from-string deploy-federated)]
|
||||||
(if deploy-federated-bool
|
(if deploy-federated-bool
|
||||||
|
@ -110,16 +108,16 @@
|
||||||
(cm/replace-all-matching-values-by-new-value "MAILERPORT" mailer-port)
|
(cm/replace-all-matching-values-by-new-value "MAILERPORT" mailer-port)
|
||||||
(cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
|
(cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
|
||||||
(cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address)
|
(cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address)
|
||||||
(cm/replace-all-matching-values-by-new-value "IS_FEDERATED"
|
(cm/replace-all-matching-values-by-new-value "IS_FEDERATED"
|
||||||
(if federation-enabled-bool
|
(if federation-enabled-bool
|
||||||
"true"
|
"true"
|
||||||
"false")))))
|
"false")))))
|
||||||
|
|
||||||
(defn generate-secrets
|
(defn-spec generate-secrets pred/map-or-seq?
|
||||||
[auth]
|
[auth ::auth]
|
||||||
(let [{:keys [postgres-db-user
|
(let [{:keys [postgres-db-user
|
||||||
postgres-db-password
|
postgres-db-password
|
||||||
mailer-user
|
mailer-user
|
||||||
mailer-pw]} auth]
|
mailer-pw]} auth]
|
||||||
(->
|
(->
|
||||||
(yaml/load-as-edn "forgejo/secrets.yaml")
|
(yaml/load-as-edn "forgejo/secrets.yaml")
|
||||||
|
@ -128,8 +126,8 @@
|
||||||
(cm/replace-all-matching "MAILERUSER" (b64/encode mailer-user))
|
(cm/replace-all-matching "MAILERUSER" (b64/encode mailer-user))
|
||||||
(cm/replace-all-matching "MAILERPW" (b64/encode mailer-pw)))))
|
(cm/replace-all-matching "MAILERPW" (b64/encode mailer-pw)))))
|
||||||
|
|
||||||
(defn-spec generate-ratelimit-ingress-and-cert seq?
|
(defn-spec generate-ratelimit-ingress-and-cert pred/map-or-seq?
|
||||||
[config config?]
|
[config ::config]
|
||||||
(let [{:keys [fqdn max-rate max-concurrent-requests namespace]} config]
|
(let [{:keys [fqdn max-rate max-concurrent-requests namespace]} config]
|
||||||
(ing/generate-simple-ingress (merge
|
(ing/generate-simple-ingress (merge
|
||||||
{:service-name "forgejo-service"
|
{:service-name "forgejo-service"
|
||||||
|
@ -141,18 +139,18 @@
|
||||||
config))))
|
config))))
|
||||||
|
|
||||||
(defn-spec generate-data-volume pred/map-or-seq?
|
(defn-spec generate-data-volume pred/map-or-seq?
|
||||||
[config vol?]
|
[config ::config]
|
||||||
(let [{:keys [volume-total-storage-size]} config
|
(let [{:keys [volume-total-storage-size]} config
|
||||||
data-storage-size (data-storage-by-volume-size volume-total-storage-size)]
|
data-storage-size (data-storage-by-volume-size volume-total-storage-size)]
|
||||||
(->
|
(->
|
||||||
(yaml/load-as-edn "forgejo/datavolume.yaml")
|
(yaml/load-as-edn "forgejo/datavolume.yaml")
|
||||||
(cm/replace-all-matching "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
|
(cm/replace-all-matching "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
|
||||||
|
|
||||||
(defn-spec generate-deployment pred/map-or-seq?
|
(defn-spec generate-deployment pred/map-or-seq?
|
||||||
[config config?]
|
[config ::config]
|
||||||
(->
|
(->
|
||||||
(yaml/load-as-edn "forgejo/deployment.yaml")
|
(yaml/load-as-edn "forgejo/deployment.yaml")
|
||||||
(cm/replace-all-matching "IMAGE_NAME" (generate-image-str config))))
|
(cm/replace-all-matching "IMAGE_NAME" (generate-image-str config))))
|
||||||
|
|
||||||
(defn generate-service
|
(defn generate-service
|
||||||
[]
|
[]
|
||||||
|
|
|
@ -5,8 +5,22 @@
|
||||||
[clojure.spec.test.alpha :as st]
|
[clojure.spec.test.alpha :as st]
|
||||||
[dda.c4k-forgejo.backup :as cut]))
|
[dda.c4k-forgejo.backup :as cut]))
|
||||||
|
|
||||||
|
(st/instrument `cut/generate-secret)
|
||||||
(st/instrument `cut/generate-config)
|
(st/instrument `cut/generate-config)
|
||||||
|
|
||||||
|
(deftest should-generate-secret
|
||||||
|
(is (= {:apiVersion "v1",
|
||||||
|
:kind "Secret",
|
||||||
|
:metadata {:name "backup-secret", :namespace "forgejo"},
|
||||||
|
:type "Opaque",
|
||||||
|
:data
|
||||||
|
{:aws-access-key-id "YXdzLWlk",
|
||||||
|
:aws-secret-access-key "YXdzLXNlY3JldA==",
|
||||||
|
:restic-password "cmVzdGljLXB3"}}
|
||||||
|
(cut/generate-secret {:aws-access-key-id "aws-id"
|
||||||
|
:aws-secret-access-key "aws-secret"
|
||||||
|
:restic-password "restic-pw"}))))
|
||||||
|
|
||||||
(deftest should-generate-backup-config
|
(deftest should-generate-backup-config
|
||||||
(testing "federated"
|
(testing "federated"
|
||||||
(is (= {:apiVersion "v1",
|
(is (= {:apiVersion "v1",
|
||||||
|
|
|
@ -111,7 +111,10 @@
|
||||||
:mailer-host "m.t.de"
|
:mailer-host "m.t.de"
|
||||||
:mailer-port "123"
|
:mailer-port "123"
|
||||||
:service-domain-whitelist "adb.de"
|
:service-domain-whitelist "adb.de"
|
||||||
:service-noreply-address ""}))))
|
:service-noreply-address ""
|
||||||
|
:volume-total-storage-size 10
|
||||||
|
:max-rate 10
|
||||||
|
:max-concurrent-requests 1}))))
|
||||||
(testing "federated-deployment"
|
(testing "federated-deployment"
|
||||||
(is (= {:apiVersion "apps/v1",
|
(is (= {:apiVersion "apps/v1",
|
||||||
:kind "Deployment",
|
:kind "Deployment",
|
||||||
|
@ -138,7 +141,10 @@
|
||||||
:mailer-host "m.t.de"
|
:mailer-host "m.t.de"
|
||||||
:mailer-port "123"
|
:mailer-port "123"
|
||||||
:service-domain-whitelist "adb.de"
|
:service-domain-whitelist "adb.de"
|
||||||
:service-noreply-address ""})))))
|
:service-noreply-address ""
|
||||||
|
:volume-total-storage-size 10
|
||||||
|
:max-rate 10
|
||||||
|
:max-concurrent-requests 1})))))
|
||||||
|
|
||||||
(deftest should-generate-secret
|
(deftest should-generate-secret
|
||||||
(is (= {:FORGEJO__database__USER-c1 "",
|
(is (= {:FORGEJO__database__USER-c1 "",
|
||||||
|
@ -157,9 +163,3 @@
|
||||||
:postgres-db-password "pg-pw"
|
:postgres-db-password "pg-pw"
|
||||||
:mailer-user "maileruser"
|
:mailer-user "maileruser"
|
||||||
:mailer-pw "mailerpw"})))))
|
:mailer-pw "mailerpw"})))))
|
||||||
|
|
||||||
(deftest should-generate-data-volume
|
|
||||||
(is (= {:storage-c1 "1Gi",
|
|
||||||
:storage-c2 "15Gi"}
|
|
||||||
(th/map-diff (cut/generate-data-volume {:volume-total-storage-size 1})
|
|
||||||
(cut/generate-data-volume {:volume-total-storage-size 15})))))
|
|
||||||
|
|
Loading…
Reference in a new issue