fix inst

.gitignore

@@ -25,8 +25,5 @@ public/js/
 *.iml
 .idea/
 
-auth.edn
-config.edn
-
 .eastwood
 

README.md

@@ -12,9 +12,9 @@ c4k-forgejo provides a k8s deployment file for forgejo containing:
 * ingress having a letsencrypt managed certificate
 * postgres database
 * encrypted backup on S3 & restore
-* monitoring on graphana-cloud
+* monitoring on grafana-cloud
 
-c4k-forgejo is an example how to create efficient k8s one shot deployments with https://repo.prod.meissa.de/meissa/c4k-common.
+c4k-forgejo is an example how to create efficient k8s one-shot deployments with https://repo.prod.meissa.de/meissa/c4k-common.
 
 ## Try out
 
@@ -32,7 +32,7 @@ After having deployed the yaml-file generated by the c4k-forgejo module you need
 * Adjust the settings according to your needs
 * Add the administrator's data (name, password and email) and submit the page.
 * The required database will be created and the forgejo setup will be completed.
-* The SSH-URL for a repo has the format: "ssh://git@domain:2222/[username]/[repo].git
+* The SSH-URL for a repo has the format: "ssh://git@domain:2222/[username]/[repo].git"
   Example: "git clone ssh://git@repo.test.meissa.de:2222/myuser/c4k-forgejo.git"
 
 ### Add Impressum

build.py

@@ -150,12 +150,12 @@ def inst(project):
         shell=True,
         check=True,
     )
-    package_native(project)
     run(
         "sudo install -m=755 target/uberjar/" + project.name + "-standalone.jar /usr/local/bin/" + project.name + "-standalone.jar",
         shell=True,
         check=True,
     )
+    package_native(project)
     run(
         "sudo install -m=755 target/graalvm/" + project.name + " /usr/local/bin/" + project.name + "",
         shell=True,

doc/BackupAndRestore.md

@@ -1,73 +1,8 @@
 # Backup Architecture details
 
-![](backup.svg)
+Use process documented at https://repo.prod.meissa.de/meissa/dda-backup/src/branch/main/docs/Backup.md
 
-* we use restic to produce small & encrypted backups
-* backup is scheduled at `schedule: "10 23 * * *"`
-* Forgejo stores files in `/data/gitea` and `/data/git/repositories`, these files are backed up. 
-* The postgres db is also backed up
+Parameters are:
 
-## Manual backup
-
-1. Scale down forgejo deployment:   
-   `kubectl -n forgejo scale deployment forgejo --replicas=0`
-2. apply backup-and-restore pod:   
-  `kubectl -n forgejo scale deployment backup-restore --replicas=1`
-3. exec into pod and execute backup pod (press tab to get your exact pod name)   
-   `kubectl -n forgejo exec -it backup-restore-... -- /usr/local/bin/backup.bb`
-4. remove backup-and-restore pod:   
-   `kubectl -n forgejo scale deployment backup-restore --replicas=0`
-5. Scale up forgejo deployment:   
-   `kubectl -n forgejo scale deployment forgejo --replicas=1`
-
-## Manual restore
-
-1. Scale down forgejo deployment:
-   `kubectl -n forgejo scale deployment forgejo --replicas=0`
-2. apply backup-and-restore pod:   
-  `kubectl -n forgejo scale deployment backup-restore --replicas=1`
-3. exec into pod and execute restore pod (press tab to get your exact pod name)   
-   `kubectl -n forgejo exec -it backup-restore-... -- /usr/local/bin/restore.bb`
-4. Start forgejo again:
-   `kubectl -n forgejo scale deployment forgejo --replicas=1`
-5. remove backup-and-restore pod:   
-   `kubectl -n forgejo scale deployment backup-restore --replicas=0`
-
-## Change Password
-
-1. Check restic-new-password env is set in backup deployment   
-   ```
-   kind: Deployment
-   metadata:
-     name: backup-restore
-   spec:
-       spec:
-         containers:
-         - name: backup-app
-           env:
-           - name: RESTIC_NEW_PASSWORD_FILE
-             value: /var/run/secrets/backup-secrets/restic-new-password
-   ```
-2. Add restic-new-password to secret   
-   ```
-   kind: Secret
-   metadata:
-     name: backup-secret
-   data:
-     restic-password: old
-     restic-new-password: new
-   ```
-3. Scale backup-restore deployment up:   
-   `kubectl -n nextcloud scale deployment backup-restore --replicas=1`
-4. exec into pod and execute restore pod   
-   `kubectl -n nextcloud exec -it backup-restore -- change-password.bb`
-5. Scale backup-restore deployment down:   
-  `kubectl -n nextcloud scale deployment backup-restore --replicas=0`
-6. Replace restic-password with restic-new-password in secret   
-   ```
-   kind: Secret
-   metadata:
-     name: backup-secret
-   data:
-     restic-password: new
-   ```
+1. **deployment-name**: forgejo
+2. **deployment-namespace**: forgejo

infrastructure/backup/build.py

@@ -6,7 +6,7 @@ from ddadevops import *
 name = "c4k-forgejo"
 MODULE = "backup"
 PROJECT_ROOT_PATH = "../.."
-version = "4.1.0"
+version = "4.1.3-dev"
 
 
 @init

infrastructure/backup/image/resources/config.edn

@@ -0,0 +1,21 @@
+{:restic-repo {:password-file #env-or-file "RESTIC_PASSWORD_FILE"
+               :restic-repository #env-or-file "RESTIC_REPOSITORY"}
+
+ :file-config #merge [#ref [:restic-repo]
+                      {:backup-path "files"
+                       :execution-directory "/var/backups/"
+                       :files ["gitea/" "git/repositories/"]}]
+
+
+ :db-config #merge [#ref [:restic-repo]
+                    {:backup-path "pg-database"
+                     :pg-host #env-or-file "POSTGRES_SERVICE"
+                     :pg-port #env-or-file "POSTGRES_PORT"
+                     :pg-db #env-or-file "POSTGRES_DB"
+                     :pg-user #env-or-file "POSTGRES_USER"
+                     :pg-password #env-or-file "POSTGRES_PASSWORD"}]
+
+ :aws-config {:aws-access-key-id #env-or-file "AWS_ACCESS_KEY_ID"
+              :aws-secret-access-key #env-or-file "AWS_SECRET_ACCESS_KEY"}
+
+ :dry-run {:dry-run true :debug true}}

infrastructure/federated/build.py

@@ -6,7 +6,7 @@ from ddadevops import *
 name = 'c4k-forgejo'
 MODULE = 'federated'
 PROJECT_ROOT_PATH = '../..'
-version = "4.1.0"
+version = "4.1.3-dev"
 
 @init
 def initialize(project):

package.json

@@ -2,7 +2,7 @@
     "name": "c4k-forgejo",
     "description": "Generate c4k yaml for a forgejo deployment.",
     "author": "meissa GmbH",
-    "version": "4.1.0",
+    "version": "4.1.3-SNAPSHOT",
     "homepage": "https://gitlab.com/domaindrivenarchitecture/c4k-forgejo#readme",
     "repository": "https://www.npmjs.com/package/c4k-forgejo",
     "license": "APACHE2",

project.clj

@@ -1,4 +1,4 @@
-(defproject org.domaindrivenarchitecture/c4k-forgejo "4.1.0"
+(defproject org.domaindrivenarchitecture/c4k-forgejo "4.1.3-SNAPSHOT"
   :description "forgejo c4k-installation package"
   :url "https://domaindrivenarchitecture.org"
   :license {:name "Apache License, Version 2.0"

src/main/clj/dda/c4k_forgejo/uberjar.clj

@@ -1,8 +1,8 @@
 (ns dda.c4k-forgejo.uberjar
   (:gen-class)
   (:require
-   [dda.c4k-forgejo.core :as core]
-   [dda.c4k-common.uberjar :as uberjar]))
+   [dda.c4k-common.uberjar :as uberjar]
+   [dda.c4k-forgejo.core :as core]))
 
 (set! *warn-on-reflection* true)
 

src/main/cljc/dda/c4k_forgejo/core.cljc

@@ -20,10 +20,9 @@
                       :pv-storage-size-gb 5
                       :pvc-storage-class-name ""
                       :postgres-image "postgres:14"
-                      :postgres-size :2gb})
-
-(def rate-limit-defaults {:max-rate 10, :max-concurrent-requests 5})
-
+                      :postgres-size :2gb
+                      :max-rate 10, 
+                      :max-concurrent-requests 5})
 
 (def config? (s/keys :req-un [::forgejo/fqdn
                               ::forgejo/mailer-from
@@ -40,48 +39,51 @@
                               ::mon/mon-cfg]))
 
 (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password
-                            ::forgejo/mailer-user ::forgejo/mailer-pw
-                            ::backup/aws-access-key-id ::backup/aws-secret-access-key]
+                            ::forgejo/mailer-user ::forgejo/mailer-pw]
                    :opt-un [::backup/restic-password
                             ::backup/restic-new-password
+                            ::backup/aws-access-key-id 
+                            ::backup/aws-secret-access-key
                             ::mon/mon-auth]))
 
 (defn-spec config-objects p/map-or-seq?
   [config config?]
-  (let [storage-class (if (contains? config :postgres-data-volume-path) :manual :local-path)]
+  (let [resolved-config (merge config-defaults config)
+        storage-class (if (contains? resolved-config :postgres-data-volume-path) :manual :local-path)]
     (map yaml/to-string
          (filter #(not (nil? %))
                  (cm/concat-vec
-                  (ns/generate config)
-                  [(postgres/generate-configmap config)
-                   (when (contains? config :postgres-data-volume-path)
-                     (postgres/generate-persistent-volume (select-keys config [:postgres-data-volume-path :pv-storage-size-gb])))
-                   (postgres/generate-pvc (merge config {:pvc-storage-class-name storage-class}))
-                   (postgres/generate-deployment config)
-                   (postgres/generate-service config)
-                   (forgejo/generate-deployment config)
+                  (ns/generate resolved-config)
+                  [(postgres/generate-configmap resolved-config)
+                   (when (contains? resolved-config :postgres-data-volume-path)
+                     (postgres/generate-persistent-volume (select-keys resolved-config [:postgres-data-volume-path :pv-storage-size-gb])))
+                   (postgres/generate-pvc (merge resolved-config {:pvc-storage-class-name storage-class}))
+                   (postgres/generate-deployment resolved-config)
+                   (postgres/generate-service resolved-config)
+                   (forgejo/generate-deployment resolved-config)
                    (forgejo/generate-service)
                    (forgejo/generate-service-ssh)
-                   (forgejo/generate-data-volume config)
-                   (forgejo/generate-appini-env config)]
-                  (forgejo/generate-ratelimit-ingress-and-cert config) ; this function has a vector as output
-                  (when (contains? config :restic-repository)
-                    [(backup/generate-config config)
+                   (forgejo/generate-data-volume resolved-config)
+                   (forgejo/generate-appini-env resolved-config)]
+                  (forgejo/generate-ratelimit-ingress-and-cert resolved-config) ; this function has a vector as output
+                  (when (contains? resolved-config :restic-repository)
+                    [(backup/generate-config resolved-config)
                      (backup/generate-cron)
-                     (backup/generate-backup-restore-deployment config)])
-                  (when (contains? config :mon-cfg)
+                     (backup/generate-backup-restore-deployment resolved-config)])
+                  (when (contains? resolved-config :mon-cfg)
                     (mon/generate-config)))))))
 
 (defn-spec auth-objects p/map-or-seq?
   [config config?
    auth auth?]
-  (map yaml/to-string
-       (filter #(not (nil? %))
-               (cm/concat-vec
-                (ns/generate config)
-                [(postgres/generate-secret config auth)
-                 (forgejo/generate-secrets auth)]
-                (when (contains? config :restic-repository)
-                  [(backup/generate-secret auth)])
-                (when (contains? config :mon-cfg)
-                  (mon/generate-auth (:mon-cfg config) (:mon-auth auth)))))))
+  (let [resolved-config (merge config-defaults config)]
+    (map yaml/to-string
+         (filter #(not (nil? %))
+                 (cm/concat-vec
+                  (ns/generate resolved-config)
+                  [(postgres/generate-secret resolved-config auth)
+                   (forgejo/generate-secrets auth)]
+                  (when (contains? resolved-config :restic-repository)
+                    [(backup/generate-secret auth)])
+                  (when (contains? resolved-config :mon-cfg)
+                    (mon/generate-auth (:mon-cfg resolved-config) (:mon-auth auth))))))))

src/main/cljc/dda/c4k_forgejo/forgejo.cljc

@@ -100,18 +100,18 @@
         federation-enabled-bool (boolean-from-string federation-enabled)]
     (->
      (yaml/load-as-edn "forgejo/appini-env-configmap.yaml")
-     (cm/replace-all-matching-values-by-new-value "APPNAME" default-app-name)
-     (cm/replace-all-matching-values-by-new-value "FQDN" fqdn)
-     (cm/replace-all-matching-values-by-new-value "URL" (str "https://" fqdn))
-     (cm/replace-all-matching-values-by-new-value "FROM" mailer-from)
-     (cm/replace-all-matching-values-by-new-value "MAILERHOST" mailer-host)
-     (cm/replace-all-matching-values-by-new-value "MAILERPORT" mailer-port)
-     (cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
-     (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address)
-     (cm/replace-all-matching-values-by-new-value "IS_FEDERATED"
-                                                  (if federation-enabled-bool
-                                                    "true"
-                                                    "false")))))
+     (cm/replace-all-matching "APPNAME" default-app-name)
+     (cm/replace-all-matching "FQDN" fqdn)
+     (cm/replace-all-matching "URL" (str "https://" fqdn))
+     (cm/replace-all-matching "FROM" mailer-from)
+     (cm/replace-all-matching "MAILERHOST" mailer-host)
+     (cm/replace-all-matching "MAILERPORT" mailer-port)
+     (cm/replace-all-matching "WHITELISTDOMAINS" service-domain-whitelist)
+     (cm/replace-all-matching "NOREPLY" service-noreply-address)
+     (cm/replace-all-matching "IS_FEDERATED"
+                              (if federation-enabled-bool
+                                "true"
+                                "false")))))
 
 (defn-spec generate-secrets pred/map-or-seq?
   [auth ::auth]

src/main/cljs/dda/c4k_forgejo/browser.cljs

@@ -1,11 +1,9 @@
 (ns dda.c4k-forgejo.browser
   (:require
-   [clojure.string :as st]
    [clojure.tools.reader.edn :as edn]
-   [dda.c4k-forgejo.core :as core]
-   [dda.c4k-forgejo.forgejo :as forgejo]
    [dda.c4k-common.browser :as br]
-   [dda.c4k-common.common :as cm]))
+   [dda.c4k-common.common :as cm]
+   [dda.c4k-forgejo.core :as core]))
 
 (defn generate-group
   [name
@@ -26,31 +24,32 @@
    [(assoc
      (br/generate-needs-validation) :content
      (cm/concat-vec
+      (br/generate-group
+       "config"
+       (br/generate-text-area
+        "config" "Your config.edn:"
+        "{:fqdn \"forgejo.your.domain\"
+ :mailer-from \"test@test.de\"
+ :mailer-host \"test.de\"
+ :mailer-port \"25\"                                                                                  
+ :deploy-federated \"false\"                                                          
+ :service-noreply-address \"no-reply@test.de\"
+ :volume-total-storage-size \"20\"
+ :mon-cfg {:cluster-name \"forgejo\"
+                 :cluster-stage \"test\"
+                 :grafana-cloud-url \"https://prometheus-prod-01-eu-west-0.grafana.net/api/prom/push\"}}"
+        "10"))
       (generate-group
-       "domain"
-       (cm/concat-vec
-        (br/generate-input-field "fqdn" "Your fqdn:" "repo.test.de")
-        (br/generate-input-field "mailer-from" "Your mailer email address:" "test@test.de")
-        (br/generate-input-field "mailer-host" "Your mailer host:" "test.de")
-        (br/generate-input-field "mailer-port" "Your mailer port:" "123")
-        (br/generate-input-field "service-noreply-address" "Your noreply domain:" "test.de")
-        (br/generate-input-field "deploy-federated" "(Optional) Deploy a federated version of forgejo:" "")
-        (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")
-        (br/generate-input-field "app-name" "(Optional) Your app name:" "")
-        (br/generate-input-field "domain-whitelist" "(Optional) Domain whitelist for registration email-addresses:" "")))
-      (generate-group
-       "provider"
-       (cm/concat-vec
-        (br/generate-input-field "volume-total-storage-size" "Your forgejo volume-total-storage-size:" "20")))
-      (generate-group
-       "credentials"
+       "auth"
        (br/generate-text-area
         "auth" "Your auth.edn:"
         "{:postgres-db-user \"forgejo\"
-         :postgres-db-password \"forgejo-db-password\"
-         :mailer-user \"test@test.de\"
-         :mailer-pw \"mail-test-password\"}"
-        "5"))
+:postgres-db-password \"forgejo-db-password\"
+:mailer-user \"test@test.de\"
+:mailer-pw \"mail-test-password\"
+:mon-auth {:grafana-cloud-user \"your-user-id\"
+                  :grafana-cloud-password \"your-cloud-password\"}}"
+        "6"))
       [(br/generate-br)]
       (br/generate-button "generate-button" "Generate c4k yaml")))]
    (br/generate-output "c4k-forgejo-output" "Your c4k deployment.yaml:" "25")))
@@ -62,44 +61,15 @@
    :content
    (generate-content)})
 
-(defn config-from-document []
-  (let [issuer (br/get-content-from-element "issuer" :optional true)
-        app-name (br/get-content-from-element "app-name" :optional true)
-        domain-whitelist (br/get-content-from-element "domain-whitelist" :optional true)]
-    (merge
-     {:fqdn (br/get-content-from-element "fqdn")
-      :deploy-federated (br/get-content-from-element "deploy-federated")
-      :mailer-from (br/get-content-from-element "mailer-from")
-      :mailer-host (br/get-content-from-element "mailer-host")
-      :mailer-port (br/get-content-from-element "mailer-port")
-      :service-noreply-address (br/get-content-from-element "service-noreply-address")
-      :volume-total-storage-size (br/get-content-from-element "volume-total-storage-size" :deserializer js/parseInt)}
-     (when (not (st/blank? issuer))
-       {:issuer issuer})
-     (when (not (st/blank? app-name))
-       {:default-app-name app-name})
-     (when (not (st/blank? domain-whitelist))
-       {:service-domain-whitelist domain-whitelist}))))
-
 (defn validate-all! []
-  (br/validate! "fqdn" ::forgejo/fqdn)
-  (br/validate! "mailer-from" ::forgejo/mailer-from)
-  (br/validate! "mailer-host" ::forgejo/mailer-host)
-  (br/validate! "mailer-port" ::forgejo/mailer-port)
-  (br/validate! "service-noreply-address" ::forgejo/service-noreply-address)
-  (br/validate! "deploy-federated" ::forgejo/deploy-federated :optional true)
-  (br/validate! "issuer" ::forgejo/issuer :optional true)
-  (br/validate! "app-name" ::forgejo/default-app-name :optional true)
-  (br/validate! "domain-whitelist" ::forgejo/service-domain-whitelist :optional true)
-  (br/validate! "volume-total-storage-size" ::forgejo/volume-total-storage-size :deserializer js/parseInt)
-  (br/validate! "auth" forgejo/auth? :deserializer edn/read-string)
+  (br/validate! "config" core/config? :deserializer edn/read-string)
+  (br/validate! "auth" core/auth? :deserializer edn/read-string)
   (br/set-form-validated!))
 
 (defn add-validate-listener [name]
   (-> (br/get-element-by-id name)
       (.addEventListener "blur" #(do (validate-all!)))))
 
-
 (defn init []
   (br/append-hickory (generate-content-div))
   (let [config-only false
@@ -109,7 +79,7 @@
         (.addEventListener "click"
                            #(do (validate-all!)
                                 (-> (cm/generate-cm
-                                     (config-from-document)
+                                     (br/get-content-from-element "config" :deserializer edn/read-string)
                                      (br/get-content-from-element "auth" :deserializer edn/read-string)
                                      core/config-defaults
                                      core/config-objects
@@ -117,14 +87,5 @@
                                      config-only
                                      auth-only)
                                     (br/set-output!))))))
-  (add-validate-listener "fqdn")
-  (add-validate-listener "deploy-federated")
-  (add-validate-listener "mailer-from")
-  (add-validate-listener "mailer-host")
-  (add-validate-listener "mailer-port")
-  (add-validate-listener "service-noreply-address")
-  (add-validate-listener "app-name")
-  (add-validate-listener "domain-whitelist")
-  (add-validate-listener "volume-total-storage-size")
-  (add-validate-listener "issuer")
+  (add-validate-listener "config")
   (add-validate-listener "auth"))

src/main/resources/backup/backup-restore-deployment.yaml

@@ -59,8 +59,6 @@ spec:
           value: /var/run/secrets/backup-secrets/restic-password
         - name: RESTIC_NEW_PASSWORD_FILE
           value: /var/run/secrets/backup-secrets/restic-new-password
-        - name: CERTIFICATE_FILE
-          value: ""
         volumeMounts:
         - name: forgejo-data-volume
           mountPath: /var/backups

src/test/cljc/dda/c4k_forgejo/core_test.cljc

@@ -1,19 +1,31 @@
 (ns dda.c4k-forgejo.core-test
   (:require
-   #?(:cljs [shadow.resource :as rc])
    #?(:clj [clojure.test :refer [deftest is are testing run-tests]]
       :cljs [cljs.test :refer-macros [deftest is are testing run-tests]])
    [clojure.spec.alpha :as s]
+   [clojure.spec.test.alpha :as st]
    [dda.c4k-common.yaml :as yaml]
-   [dda.c4k-forgejo.core :as cut]))
+   [dda.c4k-forgejo.core :as cut]
+    #?(:cljs [dda.c4k-common.macros :refer-macros [inline-resources]])))
+
+(st/instrument `cut/onfig-objects)
+(st/instrument `cut/auth-objects)
 
 #?(:cljs
    (defmethod yaml/load-resource :forgejo-test [resource-name]
-     (case resource-name
-       "forgejo-test/valid-auth.yaml"   (rc/inline "forgejo-test/valid-auth.yaml")
-       "forgejo-test/valid-config.yaml" (rc/inline "forgejo-test/valid-config.yaml")
-       (throw (js/Error. "Undefined Resource!")))))
+     (get (inline-resources "forgejo-test") resource-name)))
 
 (deftest validate-valid-resources
   (is (s/valid? cut/config? (yaml/load-as-edn "forgejo-test/valid-config.yaml")))
-  (is (s/valid? cut/auth? (yaml/load-as-edn "forgejo-test/valid-auth.yaml"))))
+  (is (s/valid? cut/auth? (yaml/load-as-edn "forgejo-test/valid-auth.yaml"))))
+
+(deftest test-whole-generation
+  (is (= 32
+         (count
+          (cut/config-objects
+           (yaml/load-as-edn "forgejo-test/valid-config.yaml")))))
+  (is (= 5
+         (count
+          (cut/auth-objects
+           (yaml/load-as-edn "forgejo-test/valid-config.yaml")
+           (yaml/load-as-edn "forgejo-test/valid-auth.yaml"))))))