Merge branch 'optional-forgejo-fed' into 'main'

Added optional support for installation of forge-fed branch

See merge request domaindrivenarchitecture/c4k-forgejo!4

.gitlab-ci.yml

@@ -114,3 +114,11 @@ release:
         --assets-link "{\"name\":\"c4k-forgejo.js\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-forgejo/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-forgejo.js\"}" \
         --assets-link "{\"name\":\"c4k-forgejo.js.sha256\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-forgejo/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-forgejo.js.sha256\"}" \
         --assets-link "{\"name\":\"c4k-forgejo.js.sha512\",\"url\":\"https://gitlab.com/domaindrivenarchitecture/c4k-forgejo/-/jobs/${CI_JOB_ID}/artifacts/file/target/frontend-build/c4k-forgejo.js.sha512\"}" \
+
+forgejo-image-test-publish:
+  image: domaindrivenarchitecture/devops-build:latest
+  stage: image
+  rules:
+    - if: '$CI_COMMIT_TAG != null'
+  script:
+    - cd infrastructure/docker-federated && pyb image test publish

build-and-move-frontend.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# set things
+set -o nounset
+set -o xtrace
+set -o errexit
+set -eo pipefail
+
+# dirs
+srcDir="/home/$USER/repo/c4k/c4k-forgejo/public/js/"
+srcName="main.js"
+targetDir="/home/$USER/repo/website/dda-io/content/templates/js/"
+targetName="c4k-gitea.js"
+
+echo "build test"
+shadow-cljs compile test
+
+echo "test"
+node target/node-tests.js
+
+echo "build frontend"
+shadow-cljs compile frontend
+
+echo "move and rename file"
+cp $srcDir$srcName $targetDir$targetName
+
+echo "run"
+(cd $targetDir; lein ring server)

copy-and-build-dda-io.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-# set things
-set -o nounset
-set -o xtrace
-set -o errexit
-set -eo pipefail
-
-# dirs
-srcDir="/home/$USER/"
-srcName="main.js"
-targetDir="/home/$USER/"
-targetName="c4k-forgejo.js"
-
-echo "build"
-shadow-cljs compile frontend
-
-echo "move and rename file"
-cp $srcDir$srcName $targetDir$targetName
-
-echo "build"
-(cd $targetDir; lein ring server)

infrastructure/docker-federated/build.py

@@ -0,0 +1,51 @@
+from os import environ
+from pybuilder.core import task, init
+from ddadevops import *
+import logging
+
+name = 'c4k-forgejo-fed'
+MODULE = 'docker'
+PROJECT_ROOT_PATH = '../..'
+
+class MyBuild(DevopsDockerBuild):
+    pass
+
+@init
+def initialize(project):
+    project.build_depends_on('ddadevops>=0.15.5')
+    stage = 'prod'
+    dockerhub_user = environ.get('DOCKERHUB_USER')
+    if not dockerhub_user:
+        dockerhub_user = gopass_field_from_path('meissa/web/docker.com', 'login')
+    dockerhub_password = environ.get('DOCKERHUB_PASSWORD')
+    if not dockerhub_password:
+        dockerhub_password = gopass_password_from_path('meissa/web/docker.com')
+    tag = environ.get('CI_COMMIT_TAG')
+    if not tag:
+        tag = get_tag_from_latest_commit()
+    config = create_devops_docker_build_config(
+        stage, PROJECT_ROOT_PATH, MODULE, dockerhub_user, dockerhub_password, docker_publish_tag=tag)
+    build = MyBuild(project, config)
+    build.initialize_build_dir()
+
+
+@task
+def image(project):
+    build = get_devops_build(project)
+    build.image()
+
+@task
+def drun(project):
+    build = get_devops_build(project)
+    build.drun()
+
+@task
+def publish(project):
+    build = get_devops_build(project)
+    build.dockerhub_login()
+    build.dockerhub_publish()
+
+@task
+def test(project):
+    build = get_devops_build(project)
+    build.test()

infrastructure/docker-federated/image/Dockerfile

@@ -0,0 +1,74 @@
+#Build stage
+FROM docker.io/library/golang:1.20.4-alpine3.18 AS build-env
+
+ARG GOPROXY
+ENV GOPROXY ${GOPROXY:-direct}
+
+#ARG GITEA_VERSION
+ARG TAGS="sqlite sqlite_unlock_notify"
+ENV TAGS "bindata timetzdata $TAGS"
+ARG CGO_EXTRA_CFLAGS
+
+ENV FORGEJO_GIT_URL "https://codeberg.org/forgejo/forgejo.git"
+ENV FORGEJO_BRANCH "forgejo-federation"
+
+#Build deps
+RUN apk --no-cache add build-base git nodejs npm
+
+#Setup repo
+RUN git clone --single-branch --branch ${FORGEJO_BRANCH} ${FORGEJO_GIT_URL} ${GOPATH}/src/code.gitea.io/gitea 
+
+WORKDIR ${GOPATH}/src/code.gitea.io/gitea
+
+#Checkout version if set
+RUN make clean-all build
+
+# Begin env-to-ini build
+RUN go build contrib/environment-to-ini/environment-to-ini.go
+
+# Run stage
+FROM docker.io/library/alpine:3.18
+LABEL maintainer="contact@forgejo.org"
+
+EXPOSE 22 3000
+
+RUN apk --no-cache add \
+    bash \
+    ca-certificates \
+    curl \
+    gettext \
+    git \
+    linux-pam \
+    openssh \
+    s6 \
+    sqlite \
+    su-exec \
+    gnupg
+
+RUN addgroup \
+    -S -g 1000 \
+    git && \
+  adduser \
+    -S -H -D \
+    -h /data/git \
+    -s /bin/bash \
+    -u 1000 \
+    -G git \
+    git && \
+  echo "git:*" | chpasswd -e
+
+ENV USER git
+ENV GITEA_CUSTOM /data/gitea
+
+VOLUME ["/data"]
+
+ENTRYPOINT ["/usr/bin/entrypoint"]
+CMD ["/bin/s6-svscan", "/etc/s6"]
+
+COPY --from=build-env /go/src/code.gitea.io/gitea/docker/root /
+COPY --from=build-env /go/src/code.gitea.io/gitea/gitea /app/gitea/gitea
+COPY --from=build-env /go/src/code.gitea.io/gitea/environment-to-ini /usr/local/bin/environment-to-ini
+COPY --from=build-env /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete /etc/profile.d/gitea_bash_autocomplete.sh
+RUN chmod 755 /usr/bin/entrypoint /app/gitea/gitea /usr/local/bin/gitea /usr/local/bin/environment-to-ini
+RUN chmod 755 /etc/s6/gitea/* /etc/s6/openssh/* /etc/s6/.s6-svscan/*
+RUN chmod 644 /etc/profile.d/gitea_bash_autocomplete.sh

infrastructure/docker-federated/test/Dockerfile

@@ -0,0 +1,10 @@
+FROM c4k-forgejo-fed
+
+RUN apk --no-cache add openjdk11-jre-headless
+
+RUN curl -L -o /tmp/serverspec.jar \
+    https://github.com/DomainDrivenArchitecture/dda-serverspec-crate/releases/download/2.0.0/dda-serverspec-standalone.jar
+
+COPY serverspec.edn /tmp/serverspec.edn
+
+RUN java -jar /tmp/serverspec.jar /tmp/serverspec.edn -v

infrastructure/docker-federated/test/serverspec.edn

@@ -0,0 +1,8 @@
+{:file [{:path "/usr/bin/entrypoint" :mod "755"}
+        {:path "/app/gitea/gitea" :mod "755"}
+        {:path "/usr/local/bin/gitea" :mod "755"}
+        {:path "/usr/local/bin/environment-to-ini" :mod "755"}
+        {:path "/etc/s6/gitea/" :mod "755"}
+        {:path "/etc/s6/openssh/" :mod "755"}
+        {:path "/etc/s6/.s6-svscan/" :mod "755"}
+        {:path "/etc/profile.d/gitea_bash_autocomplete.sh" :mod "644"}]}

src/main/cljc/dda/c4k_forgejo/core.cljc

@@ -11,6 +11,7 @@
 (def config-defaults {:issuer "staging"})
 
 (def config? (s/keys :req-un [::forgejo/fqdn 
+                              ::forgejo/deploy-federated
                               ::forgejo/mailer-from 
                               ::forgejo/mailer-host 
                               ::forgejo/mailer-port
@@ -43,7 +44,7 @@
                    (postgres/generate-deployment {:postgres-image "postgres:14"
                                                   :postgres-size :2gb})
                    (postgres/generate-service)
-                   (forgejo/generate-deployment)
+                   (forgejo/generate-deployment config)
                    (forgejo/generate-service)
                    (forgejo/generate-service-ssh)                   
                    (forgejo/generate-data-volume config)

src/main/cljc/dda/c4k_forgejo/forgejo.cljc

@@ -20,8 +20,21 @@
    (st/blank? input)
    (pred/string-of-separated-by? pred/fqdn-string? #"," input)))
 
+(defn boolean-from-string [input]
+  (cond
+    (= input "true") true
+    (= input "false") false
+    :else nil))
+
+(defn boolean-string?
+  [input]
+  (and
+   (string? input)
+   (boolean? (boolean-from-string input))))
+
 (s/def ::default-app-name string?)
 (s/def ::fqdn pred/fqdn-string?)
+(s/def ::deploy-federated boolean-string?)
 (s/def ::mailer-from pred/bash-env-string?)
 (s/def ::mailer-host pred/bash-env-string?)
 (s/def ::mailer-port pred/bash-env-string?)
@@ -34,13 +47,14 @@
 
 (def config-defaults {:issuer "staging"})
 
-(def config? (s/keys :req-un [::fqdn 
+(def config? (s/keys :req-un [::fqdn
-                              ::mailer-from 
+                              ::deploy-federated
+                              ::mailer-from
                               ::mailer-host
                               ::mailer-port
                               ::service-noreply-address]
-                     :opt-un [::issuer 
+                     :opt-un [::issuer
-                              ::default-app-name 
+                              ::default-app-name
                               ::service-domain-whitelist]))
 
 (def auth? (s/keys :req-un [::postgres/postgres-db-user ::postgres/postgres-db-password ::mailer-user ::mailer-pw]))
@@ -51,6 +65,8 @@
   [total]
   total)
 
+(def federated-image-name "codeberg.org/meissa/forgejo:federated-latest")
+(def non-federated-image-name "codeberg.org/forgejo/forgejo:1.19")
 
 #?(:cljs
    (defmethod yaml/load-resource :forgejo [resource-name]
@@ -66,6 +82,7 @@
 (defn generate-appini-env
   [config]
   (let [{:keys [default-app-name
+                deploy-federated
                 fqdn
                 mailer-from
                 mailer-host
@@ -73,8 +90,8 @@
                 service-domain-whitelist
                 service-noreply-address]
          :or {default-app-name "forgejo instance"
-              service-domain-whitelist fqdn}}
+              service-domain-whitelist fqdn}} config
-        config]
+        deploy-federated-bool (boolean-from-string deploy-federated)]
     (->
      (yaml/load-as-edn "forgejo/appini-env-configmap.yaml")
      (cm/replace-all-matching-values-by-new-value "APPNAME" default-app-name)
@@ -84,7 +101,11 @@
      (cm/replace-all-matching-values-by-new-value "MAILERHOST" mailer-host)
      (cm/replace-all-matching-values-by-new-value "MAILERPORT" mailer-port)
      (cm/replace-all-matching-values-by-new-value "WHITELISTDOMAINS" service-domain-whitelist)
-     (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address))))
+     (cm/replace-all-matching-values-by-new-value "NOREPLY" service-noreply-address)
+     (cm/replace-all-matching-values-by-new-value "IS_FEDERATED" 
+                                                  (if deploy-federated-bool
+                                                    "true"
+                                                    "false")))))
 
 (defn generate-secrets
   [auth]
@@ -117,9 +138,16 @@
      (yaml/load-as-edn "forgejo/datavolume.yaml")
      (cm/replace-all-matching-values-by-new-value "DATASTORAGESIZE" (str (str data-storage-size) "Gi")))))
 
-(defn generate-deployment
+(defn-spec generate-deployment pred/map-or-seq?
-  []
+  [config config?]
-  (yaml/load-as-edn "forgejo/deployment.yaml"))
+  (let [{:keys [deploy-federated]} config
+        deploy-federated-bool (boolean-from-string deploy-federated)]
+    (->
+     (yaml/load-as-edn "forgejo/deployment.yaml")
+     (cm/replace-all-matching-values-by-new-value "IMAGE_NAME" 
+                                                  (if deploy-federated-bool
+                                                    federated-image-name
+                                                    non-federated-image-name)))))
 
 (defn generate-service
   []

src/main/cljs/dda/c4k_forgejo/browser.cljs

@@ -30,8 +30,10 @@
        "domain"
        (cm/concat-vec
         (br/generate-input-field "fqdn" "Your fqdn:" "repo.test.de")
+        (br/generate-input-field "deploy-federated" "Deploy a federated version of forgejo:" "false")
         (br/generate-input-field "mailer-from" "Your mailer email address:" "test@test.de")
-        (br/generate-input-field "mailer-host-port" "Your mailer host with port:" "test.de:123")
+        (br/generate-input-field "mailer-host" "Your mailer host:" "test.de")
+        (br/generate-input-field "mailer-port" "Your mailer port:" "123")
         (br/generate-input-field "service-noreply-address" "Your noreply domain:" "test.de")
         (br/generate-input-field "issuer" "(Optional) Your issuer prod/staging:" "")
         (br/generate-input-field "app-name" "(Optional) Your app name:" "")
@@ -66,8 +68,10 @@
         domain-whitelist (br/get-content-from-element "domain-whitelist" :optional true)]
     (merge
      {:fqdn (br/get-content-from-element "fqdn")
+      :deploy-federated (br/get-content-from-element "deploy-federated")
       :mailer-from (br/get-content-from-element "mailer-from")
-      :mailer-host-port (br/get-content-from-element "mailer-host-port")
+      :mailer-host (br/get-content-from-element "mailer-host")
+      :mailer-port (br/get-content-from-element "mailer-port")
       :service-noreply-address (br/get-content-from-element "service-noreply-address")
       :volume-total-storage-size (br/get-content-from-element "volume-total-storage-size" :deserializer js/parseInt)}     
      (when (not (st/blank? issuer))
@@ -80,8 +84,10 @@
 
 (defn validate-all! []
   (br/validate! "fqdn" ::forgejo/fqdn)
+  (br/validate! "deploy-federated" ::forgejo/deploy-federated)
   (br/validate! "mailer-from" ::forgejo/mailer-from)
-  (br/validate! "mailer-host-port" ::forgejo/mailer-host-port)
+  (br/validate! "mailer-host" ::forgejo/mailer-host)
+  (br/validate! "mailer-port" ::forgejo/mailer-port)
   (br/validate! "service-noreply-address" ::forgejo/service-noreply-address)
   (br/validate! "issuer" ::forgejo/issuer :optional true)
   (br/validate! "app-name" ::forgejo/default-app-name :optional true)
@@ -108,8 +114,10 @@
                                    core/k8s-objects)
                                (br/set-output!)))))
   (add-validate-listener "fqdn")
+  (add-validate-listener "deploy-federated")
   (add-validate-listener "mailer-from")
-  (add-validate-listener "mailer-host-port")
+  (add-validate-listener "mailer-host")
+  (add-validate-listener "mailer-port")
   (add-validate-listener "service-noreply-address")
   (add-validate-listener "app-name")
   (add-validate-listener "domain-whitelist")  

src/main/resources/forgejo/appini-env-configmap.yaml

@@ -24,7 +24,7 @@ data:
   RUN_USER: git
 
   #[federation]
-  FORGEJO__federation__ENABLED: "false"
+  FORGEJO__federation__ENABLED: IS_FEDERATED
   
   #[indexer]
   FORGEJO__indexer__ISSUE_INDEXER_PATH: /data/gitea/indexers/issues.bleve

src/main/resources/forgejo/deployment.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       containers:
         - name: forgejo
-          image: codeberg.org/forgejo/forgejo:1.19
+          image: IMAGE_NAME
           imagePullPolicy: IfNotPresent
           # config settings
           envFrom:

src/test/cljc/dda/c4k_forgejo/forgejo_test.cljc

@@ -7,6 +7,7 @@
    [dda.c4k-common.base64 :as b64]
    [dda.c4k-forgejo.forgejo :as cut]))
 
+(st/instrument `cut/generate-deployment)
 (st/instrument `cut/generate-appini-env)
 (st/instrument `cut/generate-ingress)
 (st/instrument `cut/generate-secrets)
@@ -14,6 +15,8 @@
 (deftest should-generate-appini-env
   (is (= {:APP_NAME-c1 "",
           :APP_NAME-c2 "test forgejo",
+          :FORGEJO__federation__ENABLED-c1 "false",
+          :FORGEJO__federation__ENABLED-c2 "true",
           :FORGEJO__mailer__FROM-c1 "",
           :FORGEJO__mailer__FROM-c2 "test@test.com",
           :FORGEJO__mailer__SMTP_ADDR-c1 "m.t.de",
@@ -31,6 +34,7 @@
           :FORGEJO__service__NO_REPLY_ADDRESS-c1 "",
           :FORGEJO__service__NO_REPLY_ADDRESS-c2 "noreply@test.com"}
          (th/map-diff (cut/generate-appini-env {:default-app-name ""
+                                                :deploy-federated "false"
                                                 :fqdn "test.de"                                                
                                                 :mailer-from ""
                                                 :mailer-host "m.t.de"
@@ -39,13 +43,70 @@
                                                 :service-noreply-address ""
                                                 })
                       (cut/generate-appini-env {:default-app-name "test forgejo"
-                                                :fqdn "test.com"                                                 
+                                                :deploy-federated "true"
+                                                :fqdn "test.com"
                                                 :mailer-from "test@test.com"
                                                 :mailer-host "mail.test.com"
                                                 :mailer-port "456"
                                                 :service-domain-whitelist "test.com,test.net"
-                                                :service-noreply-address "noreply@test.com"
+                                                :service-noreply-address "noreply@test.com"})))))
-                                                })))))
+
+(deftest should-generate-non-federated-deployment
+  (is (= {:apiVersion "apps/v1",
+          :kind "Deployment",
+          :metadata {:name "forgejo", :namespace "default", :labels {:app "forgejo"}},
+          :spec
+          {:replicas 1,
+           :selector {:matchLabels {:app "forgejo"}},
+           :template
+           {:metadata {:name "forgejo", :labels {:app "forgejo"}},
+            :spec
+            {:containers
+             [{:name "forgejo",
+               :image "codeberg.org/forgejo/forgejo:1.19",
+               :imagePullPolicy "IfNotPresent",
+               :envFrom [{:configMapRef {:name "forgejo-env"}} {:secretRef {:name "forgejo-secrets"}}],
+               :volumeMounts [{:name "forgejo-data-volume", :mountPath "/data"}],
+               :ports [{:containerPort 22, :name "git-ssh"} {:containerPort 3000, :name "forgejo"}]}],
+             :volumes [{:name "forgejo-data-volume", :persistentVolumeClaim {:claimName "forgejo-data-pvc"}}]}}}}
+       (cut/generate-deployment 
+        {:default-app-name ""
+         :deploy-federated "false"
+         :fqdn "test.de"
+         :mailer-from ""
+         :mailer-host "m.t.de"
+         :mailer-port "123"
+         :service-domain-whitelist "adb.de"
+         :service-noreply-address ""}
+        ))))
+
+(deftest should-generate-federated-deployment
+  (is (= {:apiVersion "apps/v1",
+          :kind "Deployment",
+          :metadata {:name "forgejo", :namespace "default", :labels {:app "forgejo"}},
+          :spec
+          {:replicas 1,
+           :selector {:matchLabels {:app "forgejo"}},
+           :template
+           {:metadata {:name "forgejo", :labels {:app "forgejo"}},
+            :spec
+            {:containers
+             [{:name "forgejo",
+               :image "codeberg.org/meissa/forgejo:federated-latest",
+               :imagePullPolicy "IfNotPresent",
+               :envFrom [{:configMapRef {:name "forgejo-env"}} {:secretRef {:name "forgejo-secrets"}}],
+               :volumeMounts [{:name "forgejo-data-volume", :mountPath "/data"}],
+               :ports [{:containerPort 22, :name "git-ssh"} {:containerPort 3000, :name "forgejo"}]}],
+             :volumes [{:name "forgejo-data-volume", :persistentVolumeClaim {:claimName "forgejo-data-pvc"}}]}}}}
+         (cut/generate-deployment
+          {:default-app-name ""
+           :deploy-federated "true"
+           :fqdn "test.de"
+           :mailer-from ""
+           :mailer-host "m.t.de"
+           :mailer-port "123"
+           :service-domain-whitelist "adb.de"
+           :service-noreply-address ""}))))
 
 (deftest should-generate-secret
   (is (= {:FORGEJO__database__USER-c1 "",
@@ -69,4 +130,4 @@
   (is (= {:storage-c1 "1Gi",
           :storage-c2 "15Gi"}
          (th/map-diff (cut/generate-data-volume {:volume-total-storage-size 1})
-                      (cut/generate-data-volume {:volume-total-storage-size 15})))))
+                      (cut/generate-data-volume {:volume-total-storage-size 15})))))

src/test/resources/forgejo-test/valid-config.yaml

@@ -8,6 +8,7 @@ service-whitelist-domains: "test.de"
 service-noreply-address: "noreply@test.de"
 volume-total-storage-size: 6
 restic-repository: "repo-path"
+deploy-federated: "false"
 mon-cfg: 
   grafana-cloud-url: "url-for-your-prom-remote-write-endpoint"
   cluster-name: "forgejo"