From 600dd1c30a58083eb13ba7c6a5107c1666d09759 Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 6 Mar 2024 15:35:11 +0100 Subject: [PATCH 01/18] add hugo to image & generate netrc file --- infrastructure/build/image/resources/entrypoint.sh | 2 ++ infrastructure/build/image/resources/functions.sh | 5 +++++ infrastructure/build/image/resources/install.sh | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/infrastructure/build/image/resources/entrypoint.sh b/infrastructure/build/image/resources/entrypoint.sh index 05c3610..485a546 100755 --- a/infrastructure/build/image/resources/entrypoint.sh +++ b/infrastructure/build/image/resources/entrypoint.sh @@ -20,6 +20,8 @@ if [[ $currentHash == $newHash ]] echo "Nothing to do" else echo $currentHash > $HASHFILEDIR/$hashfilename + echo "Generate .netrc file" + generate-netrc-file echo "Downloading website data" get-website-data $filename unzip-website-data $filename diff --git a/infrastructure/build/image/resources/functions.sh b/infrastructure/build/image/resources/functions.sh index d105f6a..af2e7f5 100644 --- a/infrastructure/build/image/resources/functions.sh +++ b/infrastructure/build/image/resources/functions.sh @@ -1,5 +1,10 @@ #!/bin/bash +# TODO provide GITHOST +function generate-netrc-file() { + echo "machine $GITHOST password $AUTHTOKEN" > ~/.netrc +} + function get-website-data() { curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$1 $GITREPOURL } diff --git a/infrastructure/build/image/resources/install.sh b/infrastructure/build/image/resources/install.sh index 79a1c59..7616996 100755 --- a/infrastructure/build/image/resources/install.sh +++ b/infrastructure/build/image/resources/install.sh @@ -6,7 +6,7 @@ function main() { { upgradeSystem - apt-get install -qqy unzip rsync jq imagemagick curl + apt-get install -qqy unzip rsync jq imagemagick curl hugo install -d /etc/lein/ install -m 0700 /tmp/entrypoint.sh / From 248c33613aa36efe365e65ecd11b0f1bbb037d34 Mon Sep 17 00:00:00 2001 From: Clemens Date: Wed, 6 Mar 2024 15:35:53 +0100 Subject: [PATCH 02/18] started splitting build-secret and build-configmap --- src/main/cljc/dda/c4k_website/website.cljc | 29 +++++++++++++------ .../resources/website/build-configmap.yaml | 10 +++++++ src/main/resources/website/build-secret.yaml | 2 -- .../cljc/dda/c4k_website/website_test.cljc | 4 +-- 4 files changed, 31 insertions(+), 14 deletions(-) create mode 100644 src/main/resources/website/build-configmap.yaml diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index 4f30c5a..fcef41f 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -94,7 +94,6 @@ #(str "rewrite ^" (first %1) "\\$ " (second %1) " permanent;") redirects)))) - (defn-spec generate-nginx-configmap map? [config websiteconfig?] (let [{:keys [fqdns unique-name]} config @@ -114,20 +113,20 @@ (generate-redirects config 2))))))) -(defn-spec generate-build-secret pred/map-or-seq? - [config websiteconfig? - auth websiteauth?] + +; TODO generate git path without username +; TODO add test +(defn-spec generate-build-configmap pred/map-or-seq? + [config websiteconfig?] (let [{:keys [unique-name forgejo-host forgejo-repo branchname]} config - {:keys [authtoken - username]} auth - name (replace-dots-by-minus unique-name)] + name (replace-dots-by-minus unique-name) + username "TODO"] (-> - (yaml/load-as-edn "website/build-secret.yaml") + (yaml/load-as-edn "website/build-configmap.yaml") (replace-all-matching-prefixes "NAME" name) - (cm/replace-all-matching-values-by-new-value "TOKEN" (b64/encode authtoken)) (cm/replace-all-matching-values-by-new-value "REPOURL" (b64/encode (generate-gitrepourl forgejo-host @@ -140,6 +139,18 @@ forgejo-repo username)))))) +(defn-spec generate-build-secret pred/map-or-seq? + [config websiteconfig? + auth websiteauth?] + (let [{:keys [unique-name]} config + {:keys [authtoken + username]} auth + name (replace-dots-by-minus unique-name)] + (-> + (yaml/load-as-edn "website/build-secret.yaml") + (replace-all-matching-prefixes "NAME" name) + (cm/replace-all-matching-values-by-new-value "TOKEN" (b64/encode authtoken))))) + (defn-spec generate-content-pvc map? [config websiteconfig?] diff --git a/src/main/resources/website/build-configmap.yaml b/src/main/resources/website/build-configmap.yaml new file mode 100644 index 0000000..fce312c --- /dev/null +++ b/src/main/resources/website/build-configmap.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: build-configmap + namespace: NAME + labels: + app.kubernetes.part-of: NAME-website +data: + GITREPOURL: REPOURL + GITCOMMITURL: COMMITURL \ No newline at end of file diff --git a/src/main/resources/website/build-secret.yaml b/src/main/resources/website/build-secret.yaml index 4151d5c..de659ef 100644 --- a/src/main/resources/website/build-secret.yaml +++ b/src/main/resources/website/build-secret.yaml @@ -7,5 +7,3 @@ metadata: app.kubernetes.part-of: NAME-website data: AUTHTOKEN: TOKEN - GITREPOURL: REPOURL - GITCOMMITURL: COMMITURL diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index c337ef1..db1b860 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -217,9 +217,7 @@ :namespace "test-io", :labels {:app.kubernetes.part-of "test-io-website"}}, :data - {:AUTHTOKEN "YWJlZGpnYmFzZG9kag==", - :GITREPOURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vYXJjaGl2ZS9tYWluLnppcA==", - :GITCOMMITURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vZ2l0L2NvbW1pdHMvSEVBRA=="}} + {:AUTHTOKEN "YWJlZGpnYmFzZG9kag=="}} (cut/generate-build-secret {:fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-repo "repo", :issuer "staging", From e923c9d5c36955ff5e011a8a05868203a270eb14 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 09:16:50 +0100 Subject: [PATCH 03/18] moved repo-user from auth to config --- src/main/cljc/dda/c4k_website/core.cljc | 10 +++++--- src/main/cljc/dda/c4k_website/website.cljc | 24 +++++++++---------- src/main/cljs/dda/c4k_website/browser.cljs | 12 +++++----- src/test/cljc/dda/c4k_website/core_test.cljc | 24 ++++++++++++------- .../cljc/dda/c4k_website/website_test.cljc | 16 +++++++++++-- .../resources/website-test/valid-auth.yaml | 2 -- .../resources/website-test/valid-config.yaml | 2 ++ 7 files changed, 55 insertions(+), 35 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index 8c3a4a1..28f1cd0 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -19,17 +19,20 @@ (s/def ::authtoken ::web/authtoken) (s/def ::fqdns ::web/fqdns) (s/def ::forgejo-host ::web/forgejo-host) +(s/def ::repo-user ::web/repo-user) (s/def ::forgejo-repo ::web/forgejo-repo) (s/def ::branchname ::web/branchname) -(s/def ::username ::web/username) (s/def ::build-cpu-request ::web/build-cpu-request) (s/def ::build-memory-request ::web/build-memory-request) (s/def ::build-cpu-limit ::web/build-cpu-limit) (s/def ::build-memory-limit ::web/build-memory-limit) +(s/def ::redirect ::web/redirect) +(s/def ::redirects ::web/redirects) (def websiteconfig? (s/keys :req-un [::unique-name ::fqdns ::forgejo-host + ::repo-user ::forgejo-repo ::branchname] :opt-un [::issuer @@ -37,8 +40,9 @@ ::build-cpu-request ::build-cpu-limit ::build-memory-request - ::build-memory-limit])) -(def websiteauth? (s/keys :req-un [::unique-name ::username ::authtoken])) + ::build-memory-limit + ::redirects])) +(def websiteauth? (s/keys :req-un [::unique-name ::authtoken])) (s/def ::websiteconfigs (s/coll-of websiteconfig?)) (s/def ::websiteauths (s/coll-of websiteauth?)) diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index fcef41f..dc9a3d7 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -22,7 +22,7 @@ (s/def ::forgejo-host pred/fqdn-string?) (s/def ::forgejo-repo string?) (s/def ::branchname string?) -(s/def ::username string?) +(s/def ::repo-user string?) (s/def ::build-cpu-request string?) (s/def ::build-memory-request string?) (s/def ::build-cpu-limit string?) @@ -30,10 +30,11 @@ (s/def ::redirect (s/tuple string? string?)) (s/def ::redirects (s/coll-of ::redirect)) - +; TODO: is this websiteconfig? & websiteauth? used or the ones in core? (def websiteconfig? (s/keys :req-un [::unique-name ::fqdns ::forgejo-host + ::repo-user ::forgejo-repo ::branchname ::issuer @@ -44,7 +45,7 @@ ::build-memory-limit ::redirects])) -(def websiteauth? (s/keys :req-un [::unique-name ::username ::authtoken])) +(def websiteauth? (s/keys :req-un [::unique-name ::authtoken])) (s/def ::websiteconfigs (s/coll-of websiteconfig?)) @@ -112,18 +113,15 @@ #"REDIRECTS" (generate-redirects config 2))))))) - - -; TODO generate git path without username ; TODO add test (defn-spec generate-build-configmap pred/map-or-seq? [config websiteconfig?] (let [{:keys [unique-name forgejo-host + repo-user forgejo-repo branchname]} config - name (replace-dots-by-minus unique-name) - username "TODO"] + name (replace-dots-by-minus unique-name)] (-> (yaml/load-as-edn "website/build-configmap.yaml") (replace-all-matching-prefixes "NAME" name) @@ -131,20 +129,20 @@ (generate-gitrepourl forgejo-host forgejo-repo - username + repo-user branchname))) (cm/replace-all-matching-values-by-new-value "COMMITURL" (b64/encode (generate-gitcommiturl forgejo-host forgejo-repo - username)))))) - + repo-user)))))) +; TODO: remove config (defn-spec generate-build-secret pred/map-or-seq? [config websiteconfig? auth websiteauth?] (let [{:keys [unique-name]} config - {:keys [authtoken - username]} auth + {:keys [unique-name + authtoken]} auth name (replace-dots-by-minus unique-name)] (-> (yaml/load-as-edn "website/build-secret.yaml") diff --git a/src/main/cljs/dda/c4k_website/browser.cljs b/src/main/cljs/dda/c4k_website/browser.cljs index 04a8944..584ea14 100644 --- a/src/main/cljs/dda/c4k_website/browser.cljs +++ b/src/main/cljs/dda/c4k_website/browser.cljs @@ -28,14 +28,16 @@ "{ :websiteconfigs [{:unique-name \"test.io\", :fqdns [\"test.de\" \"www.test.de\"], - :gitea-host \"githost.de\", - :gitea-repo \"repo\", + :forgejo-host \"githost.de\", + :repo-user \"someuser\", + :forgejo-repo \"repo\", :branchname \"main\", :sha256sum-output \"123456789ab123cd345de script-file-name.sh\"} {:unique-name \"example.io \", :fqdns [\"example.org\" \"www.example.org\"], - :gitea-host \"githost.org\", - :gitea-repo \"repo\", + :forgejo-host \"githost.org\", + :repo-user \"someuser\", + :forgejo-repo \"repo\", :branchname \"main\", :build-cpu-request \"1500m\", :build-cpu-limit \"3000m\", @@ -51,10 +53,8 @@ :grafana-cloud-password \"your-cloud-password\"} :websiteauths [{:unique-name \"test.io\", - :username \"someuser\", :authtoken \"abedjgbasdodj\"} {:unique-name \"example.io\", - :username \"someuser\", :authtoken \"abedjgbasdodj\"}]}" "7")) [(br/generate-br)] diff --git a/src/test/cljc/dda/c4k_website/core_test.cljc b/src/test/cljc/dda/c4k_website/core_test.cljc index f369ec6..db5e3b5 100644 --- a/src/test/cljc/dda/c4k_website/core_test.cljc +++ b/src/test/cljc/dda/c4k_website/core_test.cljc @@ -30,11 +30,13 @@ [{:unique-name "example.io" :fqdns ["example.org", "www.example.com"] :forgejo-host "finegitehost.net" + :repo-user "someuser" :forgejo-repo "repo" :branchname "main"} {:unique-name "test.io" :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] :forgejo-host "gitlab.de" + :repo-user "someuser" :forgejo-repo "repo" :branchname "main"}]}) @@ -43,42 +45,40 @@ [{:unique-name "test.io" :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] :forgejo-host "gitlab.de" + :repo-user "someuser" :forgejo-repo "repo" :branchname "main"} {:unique-name "example.io" :fqdns ["example.org", "www.example.com"] :forgejo-host "finegitehost.net" + :repo-user "someuser" :forgejo-repo "repo" :branchname "main"}]}) (def auth1 {:websiteauths [{:unique-name "example.io" - :username "someuser" :authtoken "abedjgbasdodj"} {:unique-name "test.io" - :username "someuser" :authtoken "abedjgbasdodj"}]}) (def auth2 {:websiteauths [{:unique-name "test.io" - :username "someuser" :authtoken "abedjgbasdodj"} {:unique-name "example.io" - :username "someuser" :authtoken "abedjgbasdodj"}]}) (def flattened-and-reduced-config {:unique-name "example.io", :fqdns ["example.org" "www.example.com"], :forgejo-host "finegitehost.net", + :repo-user "someuser", :forgejo-repo "repo", :branchname "main"}) (def flattened-and-reduced-auth {:unique-name "example.io", - :username "someuser", :authtoken "abedjgbasdodj"}) (deftest sorts-config @@ -87,11 +87,13 @@ [{:unique-name "example.io", :fqdns ["example.org" "www.example.com"], :forgejo-host "finegitehost.net", + :repo-user "someuser", :forgejo-repo "repo", :branchname "main"}, {:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-host "gitlab.de", + :repo-user "someuser", :forgejo-repo "repo", :branchname "main", :sha256sum-output "123456789ab123cd345de script-file-name.sh"}], @@ -102,12 +104,14 @@ [{:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-host "gitlab.de", + :repo-user "someuser", :forgejo-repo "repo", :branchname "main", :sha256sum-output "123456789ab123cd345de script-file-name.sh"} {:unique-name "example.io", :fqdns ["example.org" "www.example.com"], :forgejo-host "finegitehost.net", + :repo-user "someuser", :forgejo-repo "repo", :branchname "main"}], :mon-cfg {:grafana-cloud-url "url-for-your-prom-remote-write-endpoint", :cluster-name "jitsi", :cluster-stage "test"}})))) @@ -158,13 +162,14 @@ :path "/", :backend {:service {:name "test-io", :port {:number 80}}}}]}}] (get-in - (cut/generate-ingress {:forgejo-host "gitlab.de", + (cut/generate-ingress {:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], + :forgejo-host "gitlab.de", + :repo-user "someuser", :forgejo-repo "repo", :sha256sum-output "123456789ab123cd345de script-file-name.sh", :issuer "staging", :branchname "main", - :unique-name "test.io" :build-cpu-request "500m" :build-cpu-limit "1700m" :build-memory-request "256Mi" @@ -174,13 +179,14 @@ [2 :spec :rules]))) (is (= "test-io" (get-in - (cut/generate-ingress {:forgejo-host "gitlab.de", + (cut/generate-ingress {:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], + :forgejo-host "gitlab.de", + :repo-user "someuser", :forgejo-repo "repo", :sha256sum-output "123456789ab123cd345de script-file-name.sh", :issuer "staging", :branchname "main", - :unique-name "test.io" :build-cpu-request "500m" :build-cpu-limit "1700m" :build-memory-request "256Mi" diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index db1b860..42a6371 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -31,6 +31,7 @@ :redirects [["/products.html", "/offer.html"] ["/one-more", "/redirect"]] :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]} @@ -45,6 +46,7 @@ :unique-name "test.io", :redirects [] :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]} @@ -54,6 +56,7 @@ (deftest should-generate-resource-requests (is (= {:requests {:cpu "1500m", :memory "512Mi"}, :limits {:cpu "3000m", :memory "1024Mi"}} (-> (cut/generate-nginx-deployment {:forgejo-host "gitlab.de", + :repo-user "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-repo "repo", :issuer "staging", @@ -68,6 +71,7 @@ :spec :template :spec :initContainers first :resources))) (is (= "test-io" (-> (cut/generate-nginx-deployment {:forgejo-host "gitlab.de", + :repo-user "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-repo "repo", :issuer "staging", @@ -92,6 +96,7 @@ :redirects [["/products.html", "/offer.html"] ["/one-more", "/redirect"]] :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))) @@ -107,6 +112,7 @@ :redirects [["/products.html", "/offer.html"] ["/one-more", "/redirect"]] :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))) @@ -121,6 +127,7 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))) @@ -139,6 +146,7 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}) @@ -162,6 +170,7 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))) @@ -204,6 +213,7 @@ :build-memory-limit "512Mi" :volume-size "3" :forgejo-host "gitlab.de", + :repo-user "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-repo "repo", :branchname "main", @@ -225,14 +235,14 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitlab.de" + :repo-user "someuser", :build-cpu-request "500m" :build-cpu-limit "1700m" :build-memory-request "256Mi" :build-memory-limit "512Mi" :volume-size "3"} {:unique-name "test.io", - :authtoken "abedjgbasdodj", - :username "someuser"})))) + :authtoken "abedjgbasdodj"})))) (deftest should-generate-content-pvc (is (= {:apiVersion "v1", @@ -254,6 +264,7 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))) @@ -278,6 +289,7 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", + :repo-user "someuser", :forgejo-repo "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))) \ No newline at end of file diff --git a/src/test/resources/website-test/valid-auth.yaml b/src/test/resources/website-test/valid-auth.yaml index ddb2e8b..ec50104 100644 --- a/src/test/resources/website-test/valid-auth.yaml +++ b/src/test/resources/website-test/valid-auth.yaml @@ -1,9 +1,7 @@ websiteauths: - unique-name: "test.io" - username: "someuser" authtoken: "abedjgbasdodj" - unique-name: "example.io" - username: "someuser" authtoken: "abedjgbasdodj" mon-auth: grafana-cloud-user: "user" diff --git a/src/test/resources/website-test/valid-config.yaml b/src/test/resources/website-test/valid-config.yaml index a39014c..6d0028a 100644 --- a/src/test/resources/website-test/valid-config.yaml +++ b/src/test/resources/website-test/valid-config.yaml @@ -3,6 +3,7 @@ websiteconfigs: - unique-name: "test.io" fqdns: ["test.de", "test.org", "www.test.de", "www.test.org"] forgejo-host: "codeberg.org" + repo-user: "someuser" forgejo-repo: "repo" branchname: "main" redirects: @@ -11,6 +12,7 @@ websiteconfigs: - unique-name: "example.io" fqdns: ["example.org", "www.example.com"] forgejo-host: "fineForgejoHost.net" + repo-user: "someotheruser" forgejo-repo: "repo" branchname: "main" mon-cfg: From e02f4949fc796132f2440600b6d65107d8e97c19 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 11:05:49 +0100 Subject: [PATCH 04/18] small adjustments --- src/main/cljc/dda/c4k_website/core.cljc | 19 +++++++++++-------- src/main/cljc/dda/c4k_website/website.cljc | 13 ++----------- src/main/cljs/dda/c4k_website/browser.cljs | 6 +++--- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index 28f1cd0..64e4f2d 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -16,6 +16,7 @@ (s/def ::unique-name ::web/unique-name) (s/def ::issuer ::web/issuer) (s/def ::volume-size ::web/volume-size) + (s/def ::authtoken ::web/authtoken) (s/def ::fqdns ::web/fqdns) (s/def ::forgejo-host ::web/forgejo-host) @@ -26,7 +27,6 @@ (s/def ::build-memory-request ::web/build-memory-request) (s/def ::build-cpu-limit ::web/build-cpu-limit) (s/def ::build-memory-limit ::web/build-memory-limit) -(s/def ::redirect ::web/redirect) (s/def ::redirects ::web/redirects) (def websiteconfig? (s/keys :req-un [::unique-name @@ -42,7 +42,9 @@ ::build-memory-request ::build-memory-limit ::redirects])) -(def websiteauth? (s/keys :req-un [::unique-name ::authtoken])) +(def websiteauth? web/websiteauth?) +(def websiteauths? (s/keys :req-un [::websiteauths])) + (s/def ::websiteconfigs (s/coll-of websiteconfig?)) (s/def ::websiteauths (s/coll-of websiteauth?)) @@ -76,6 +78,7 @@ (-> unsorted-auth (assoc-in [:websiteauths] sorted-auth)))) +; TODO: Replace this with a function that merges defaults into website config (defn-spec flatten-and-reduce-config map? [config config?] (let @@ -103,22 +106,22 @@ (defn-spec generate seq? [config config? auth auth?] - (loop [config (sort-config config) + (loop [sorted-config (sort-config config) sorted-auth (sort-auth auth) result []] - (if (and (empty? (config :websiteconfigs)) (empty? (sorted-auth :websiteauths))) + (if (and (empty? (sorted-config :websiteconfigs)) (empty? (sorted-auth :websiteauths))) result (recur (-> - config - (assoc-in [:websiteconfigs] (rest (config :websiteconfigs)))) + sorted-config + (assoc-in [:websiteconfigs] (rest (sorted-config :websiteconfigs)))) (-> - auth + sorted-auth (assoc-in [:websiteauths] (rest (sorted-auth :websiteauths)))) (let [final-config (merge website-config-defaults - (flatten-and-reduce-config config)) + (flatten-and-reduce-config sorted-config)) name (web/replace-dots-by-minus (:unique-name final-config))] (cm/concat-vec result diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index dc9a3d7..33746b7 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -30,31 +30,22 @@ (s/def ::redirect (s/tuple string? string?)) (s/def ::redirects (s/coll-of ::redirect)) -; TODO: is this websiteconfig? & websiteauth? used or the ones in core? (def websiteconfig? (s/keys :req-un [::unique-name ::fqdns ::forgejo-host ::repo-user ::forgejo-repo ::branchname - ::issuer - ::volume-size ::build-cpu-request ::build-cpu-limit ::build-memory-request ::build-memory-limit + ::issuer + ::volume-size ::redirects])) (def websiteauth? (s/keys :req-un [::unique-name ::authtoken])) -(s/def ::websiteconfigs (s/coll-of websiteconfig?)) - -(s/def ::websiteauths (s/coll-of websiteauth?)) - -(def websiteconfigs? (s/keys :req-un [::websiteconfigs])) - -(def auth? (s/keys :req-un [::websiteauths])) - (defn-spec replace-dots-by-minus string? [fqdn pred/fqdn-string?] (str/replace fqdn #"\." "-")) diff --git a/src/main/cljs/dda/c4k_website/browser.cljs b/src/main/cljs/dda/c4k_website/browser.cljs index 584ea14..8ddddc0 100644 --- a/src/main/cljs/dda/c4k_website/browser.cljs +++ b/src/main/cljs/dda/c4k_website/browser.cljs @@ -84,12 +84,12 @@ :grafana-cloud-url mon-cloud-url}})))) (defn validate-all! [] - (br/validate! "websiteconfigs" website/websiteconfigs? :deserializer edn/read-string) - (br/validate! "issuer" ::website/issuer :optional true) + (br/validate! "websiteconfigs" core/websiteconfigs? :deserializer edn/read-string) + (br/validate! "issuer" ::core/issuer :optional true) (br/validate! "mon-cluster-name" ::mon/cluster-name :optional true) (br/validate! "mon-cluster-stage" ::mon/cluster-stage :optional true) (br/validate! "mon-cloud-url" ::mon/grafana-cloud-url :optional true) - (br/validate! "auth" website/auth? :deserializer edn/read-string) + (br/validate! "auth" core/websiteauths? :deserializer edn/read-string) (br/set-form-validated!)) (defn add-validate-listener [name] From 9e75c22337bf07f4d254d8c6a5b5db07d4e9c957 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 11:26:49 +0100 Subject: [PATCH 05/18] added build-configmap generation --- src/main/cljc/dda/c4k_website/core.cljc | 24 +++++++++---------- src/main/cljc/dda/c4k_website/website.cljc | 10 ++++---- src/test/cljc/dda/c4k_website/core_test.cljc | 2 +- .../cljc/dda/c4k_website/website_test.cljc | 15 +----------- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index 64e4f2d..af5383e 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -118,23 +118,23 @@ (-> sorted-auth (assoc-in [:websiteauths] (rest (sorted-auth :websiteauths)))) - (let [final-config + (let [curr-flat-websiteconfig (merge website-config-defaults (flatten-and-reduce-config sorted-config)) - name (web/replace-dots-by-minus (:unique-name final-config))] + name (web/replace-dots-by-minus (:unique-name curr-flat-websiteconfig))] (cm/concat-vec result - (ns/generate (merge {:namespace name} final-config)) - [(web/generate-nginx-deployment final-config) - (web/generate-nginx-configmap final-config) - (web/generate-nginx-service final-config) - (web/generate-content-pvc final-config) - (web/generate-hash-state-pvc final-config) - (web/generate-build-cron final-config) - (web/generate-build-secret final-config - (flatten-and-reduce-auth auth))] - (generate-ingress final-config))))))) + (ns/generate (merge {:namespace name} curr-flat-websiteconfig)) + [(web/generate-nginx-deployment curr-flat-websiteconfig) + (web/generate-nginx-configmap curr-flat-websiteconfig) + (web/generate-nginx-service curr-flat-websiteconfig) + (web/generate-content-pvc curr-flat-websiteconfig) + (web/generate-hash-state-pvc curr-flat-websiteconfig) + (web/generate-build-cron curr-flat-websiteconfig) + (web/generate-build-configmap curr-flat-websiteconfig) + (web/generate-build-secret (flatten-and-reduce-auth auth))] + (generate-ingress curr-flat-websiteconfig))))))) (defn-spec k8s-objects cp/map-or-seq? [config config? diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index 33746b7..9dd6142 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -104,7 +104,7 @@ #"REDIRECTS" (generate-redirects config 2))))))) -; TODO add test +; TODO add test & add to build-cron env (defn-spec generate-build-configmap pred/map-or-seq? [config websiteconfig?] (let [{:keys [unique-name @@ -127,12 +127,10 @@ forgejo-host forgejo-repo repo-user)))))) -; TODO: remove config + (defn-spec generate-build-secret pred/map-or-seq? - [config websiteconfig? - auth websiteauth?] - (let [{:keys [unique-name]} config - {:keys [unique-name + [auth websiteauth?] + (let [{:keys [unique-name authtoken]} auth name (replace-dots-by-minus unique-name)] (-> diff --git a/src/test/cljc/dda/c4k_website/core_test.cljc b/src/test/cljc/dda/c4k_website/core_test.cljc index db5e3b5..e028819 100644 --- a/src/test/cljc/dda/c4k_website/core_test.cljc +++ b/src/test/cljc/dda/c4k_website/core_test.cljc @@ -131,7 +131,7 @@ (cut/flatten-and-reduce-auth (cut/sort-auth auth2))))) (deftest test-generate - (is (= 22 + (is (= 24 (count (cut/generate (yaml/load-as-edn "website-test/valid-config.yaml") (yaml/load-as-edn "website-test/valid-auth.yaml")))))) diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index 42a6371..a94dffd 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -228,20 +228,7 @@ :labels {:app.kubernetes.part-of "test-io-website"}}, :data {:AUTHTOKEN "YWJlZGpnYmFzZG9kag=="}} - (cut/generate-build-secret {:fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], - :forgejo-repo "repo", - :issuer "staging", - :branchname "main", - :unique-name "test.io", - :redirects [], - :forgejo-host "gitlab.de" - :repo-user "someuser", - :build-cpu-request "500m" - :build-cpu-limit "1700m" - :build-memory-request "256Mi" - :build-memory-limit "512Mi" - :volume-size "3"} - {:unique-name "test.io", + (cut/generate-build-secret {:unique-name "test.io", :authtoken "abedjgbasdodj"})))) (deftest should-generate-content-pvc From f1e41b3333330e69a0609d138ca6af56598fea48 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 11:44:59 +0100 Subject: [PATCH 06/18] added test for generate-build-configmap --- src/main/cljc/dda/c4k_website/website.cljc | 26 +++++++--------- .../cljc/dda/c4k_website/website_test.cljc | 31 +++++++++++++++++++ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index 9dd6142..8065ca6 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -53,17 +53,17 @@ ; https://your.gitea.host/api/v1/repos///archive/.zip (defn-spec generate-gitrepourl string? [host pred/fqdn-string? + owner string? repo string? - user string? branch string?] - (str "https://" host "/api/v1/repos/" user "/" repo "/archive/" branch ".zip")) + (str "https://" host "/api/v1/repos/" owner "/" repo "/archive/" branch ".zip")) ; https://your.gitea.host/api/v1/repos///git/commits/HEAD (defn-spec generate-gitcommiturl string? [host pred/fqdn-string? - repo string? - user string?] - (str "https://" host "/api/v1/repos/" user "/" repo "/git/" "commits/" "HEAD")) + owner string? + repo string?] + (str "https://" host "/api/v1/repos/" owner "/" repo "/git/" "commits/" "HEAD")) (defn-spec replace-all-matching-prefixes map? @@ -104,7 +104,7 @@ #"REDIRECTS" (generate-redirects config 2))))))) -; TODO add test & add to build-cron env +; TODO add to build-cron env (defn-spec generate-build-configmap pred/map-or-seq? [config websiteconfig?] (let [{:keys [unique-name @@ -116,17 +116,15 @@ (-> (yaml/load-as-edn "website/build-configmap.yaml") (replace-all-matching-prefixes "NAME" name) - (cm/replace-all-matching-values-by-new-value "REPOURL" (b64/encode - (generate-gitrepourl + (cm/replace-all-matching-values-by-new-value "REPOURL" (generate-gitrepourl forgejo-host - forgejo-repo repo-user - branchname))) - (cm/replace-all-matching-values-by-new-value "COMMITURL" (b64/encode - (generate-gitcommiturl + forgejo-repo + branchname)) + (cm/replace-all-matching-values-by-new-value "COMMITURL" (generate-gitcommiturl forgejo-host - forgejo-repo - repo-user)))))) + repo-user + forgejo-repo))))) (defn-spec generate-build-secret pred/map-or-seq? [auth websiteauth?] diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index a94dffd..d6de269 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -19,6 +19,14 @@ (st/instrument `cut/generate-nginx-service) +(deftest should-generate-gitrepourl + (is (= "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" + (cut/generate-gitrepourl "mygit.de" "someuser" "repo" "main")))) + +(deftest should-generate-gitcommiturl + (is (= "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD" + (cut/generate-gitcommiturl "mygit.de" "someuser" "repo")))) + (deftest should-generate-redirects (is (= "rewrite ^/products.html\\$ /offer.html permanent;\n rewrite ^/one-more\\$ /redirect permanent;" (cut/generate-redirects {:issuer "staging" @@ -220,6 +228,29 @@ :unique-name "test.io", :redirects []})))) +(deftest should-generate-build-configmap + (is (= {:apiVersion "v1", + :kind "ConfigMap", + :metadata {:name "build-configmap", + :namespace "test-io", + :labels {:app.kubernetes.part-of "test-io-website"}}, + :data + {:GITREPOURL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" + :GITCOMMITURL "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD"}} + (cut/generate-build-configmap {:issuer "staging" + :build-cpu-request "500m" + :build-cpu-limit "1700m" + :build-memory-request "256Mi" + :build-memory-limit "512Mi" + :volume-size "3" + :forgejo-host "mygit.de", + :repo-user "someuser", + :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], + :forgejo-repo "repo", + :branchname "main", + :unique-name "test.io", + :redirects []})))) + (deftest should-generate-build-secret (is (= {:apiVersion "v1", :kind "Secret", From 548dd140919b7538cec9f228cd67f417295935da Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 12:01:19 +0100 Subject: [PATCH 07/18] Added build-configmap as env to build-cron --- src/main/cljc/dda/c4k_website/website.cljc | 1 - src/main/resources/website/build-cron.yaml | 2 ++ src/test/cljc/dda/c4k_website/website_test.cljc | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index 8065ca6..d1dc3ac 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -104,7 +104,6 @@ #"REDIRECTS" (generate-redirects config 2))))))) -; TODO add to build-cron env (defn-spec generate-build-configmap pred/map-or-seq? [config websiteconfig?] (let [{:keys [unique-name diff --git a/src/main/resources/website/build-cron.yaml b/src/main/resources/website/build-cron.yaml index afb5622..e8534a4 100644 --- a/src/main/resources/website/build-cron.yaml +++ b/src/main/resources/website/build-cron.yaml @@ -31,6 +31,8 @@ spec: memory: BUILD_MEMORY_LIMIT command: ["/entrypoint.sh"] envFrom: + - configMapRef: + name: build-configmap - secretRef: name: build-secret volumeMounts: diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index d6de269..21895fc 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -208,7 +208,8 @@ :imagePullPolicy "IfNotPresent", :resources {:requests {:cpu "500m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}}, :command ["/entrypoint.sh"], - :envFrom [{:secretRef {:name "build-secret"}}], + :envFrom [{:configMapRef {:name "build-configmap"}} + {:secretRef {:name "build-secret"}}], :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"} {:name "hash-state-volume", :mountPath "/var/hashfile.d"}]}], :volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "content-volume"}} From ebca7a3454e88cb9ae62d0e9547b5e40ca9b4056 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 12:12:37 +0100 Subject: [PATCH 08/18] added git host as env var --- infrastructure/build/image/resources/functions.sh | 3 +-- src/main/cljc/dda/c4k_website/core.cljc | 1 - src/main/cljc/dda/c4k_website/website.cljc | 1 + src/main/resources/website/build-configmap.yaml | 1 + src/test/cljc/dda/c4k_website/website_test.cljc | 3 ++- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/infrastructure/build/image/resources/functions.sh b/infrastructure/build/image/resources/functions.sh index af2e7f5..8333acb 100644 --- a/infrastructure/build/image/resources/functions.sh +++ b/infrastructure/build/image/resources/functions.sh @@ -1,8 +1,7 @@ #!/bin/bash -# TODO provide GITHOST function generate-netrc-file() { - echo "machine $GITHOST password $AUTHTOKEN" > ~/.netrc + echo "machine $GIT_HOST password $AUTHTOKEN" > ~/.netrc } function get-website-data() { diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index af5383e..83c63b4 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -78,7 +78,6 @@ (-> unsorted-auth (assoc-in [:websiteauths] sorted-auth)))) -; TODO: Replace this with a function that merges defaults into website config (defn-spec flatten-and-reduce-config map? [config config?] (let diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index d1dc3ac..4c61147 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -115,6 +115,7 @@ (-> (yaml/load-as-edn "website/build-configmap.yaml") (replace-all-matching-prefixes "NAME" name) + (cm/replace-all-matching-values-by-new-value "GITHOST" forgejo-host) (cm/replace-all-matching-values-by-new-value "REPOURL" (generate-gitrepourl forgejo-host repo-user diff --git a/src/main/resources/website/build-configmap.yaml b/src/main/resources/website/build-configmap.yaml index fce312c..96dbb86 100644 --- a/src/main/resources/website/build-configmap.yaml +++ b/src/main/resources/website/build-configmap.yaml @@ -6,5 +6,6 @@ metadata: labels: app.kubernetes.part-of: NAME-website data: + GIT_HOST: GITHOST GITREPOURL: REPOURL GITCOMMITURL: COMMITURL \ No newline at end of file diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index 21895fc..2ed5fb3 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -236,7 +236,8 @@ :namespace "test-io", :labels {:app.kubernetes.part-of "test-io-website"}}, :data - {:GITREPOURL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" + {:GIT_HOST "mygit.de" + :GITREPOURL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" :GITCOMMITURL "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD"}} (cut/generate-build-configmap {:issuer "staging" :build-cpu-request "500m" From 164f66d4b795cfa9da2b77dfa6dc217e70e984a4 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 12:19:28 +0100 Subject: [PATCH 09/18] renamed env variables according to convention --- infrastructure/build/image/Dockerfile | 8 ++++---- infrastructure/build/image/resources/entrypoint.sh | 10 +++++----- infrastructure/build/image/resources/functions.sh | 14 +++++++------- src/main/resources/website/build-configmap.yaml | 4 ++-- src/main/resources/website/build-secret.yaml | 2 +- src/test/cljc/dda/c4k_website/website_test.cljc | 6 +++--- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/infrastructure/build/image/Dockerfile b/infrastructure/build/image/Dockerfile index 4d3289b..84e606e 100644 --- a/infrastructure/build/image/Dockerfile +++ b/infrastructure/build/image/Dockerfile @@ -3,9 +3,9 @@ FROM clojure:lein # Prepare Entrypoint Script ADD resources /tmp -ENV BUILDDIR="/etc/website" -ENV SOURCEDIR="/etc/websitesource" -ENV WEBSITEROOT="/var/www/html/website/" -ENV HASHFILEDIR="/var/hashfile.d" +ENV BUILD_DIR="/etc/website" +ENV SOURCE_DIR="/etc/websitesource" +ENV WEBSITE_ROOT="/var/www/html/website/" +ENV HASHFILE_DIR="/var/hashfile.d" RUN /tmp/install.sh diff --git a/infrastructure/build/image/resources/entrypoint.sh b/infrastructure/build/image/resources/entrypoint.sh index 485a546..e50684b 100755 --- a/infrastructure/build/image/resources/entrypoint.sh +++ b/infrastructure/build/image/resources/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash -mkdir $BUILDDIR -mkdir $SOURCEDIR +mkdir $BUILD_DIR +mkdir $SOURCE_DIR set -euo pipefail @@ -11,15 +11,15 @@ filename="website.zip" hashfilename="hashfile" echo "Check for new content" -touch $HASHFILEDIR/$hashfilename -currentHash=$( cat $HASHFILEDIR/$hashfilename ) +touch $HASHFILE_DIR/$hashfilename +currentHash=$( cat $HASHFILE_DIR/$hashfilename ) newHash=$( get-hash-data ) if [[ $currentHash == $newHash ]] then echo "Nothing to do" else - echo $currentHash > $HASHFILEDIR/$hashfilename + echo $currentHash > $HASHFILE_DIR/$hashfilename echo "Generate .netrc file" generate-netrc-file echo "Downloading website data" diff --git a/infrastructure/build/image/resources/functions.sh b/infrastructure/build/image/resources/functions.sh index 8333acb..60f192d 100644 --- a/infrastructure/build/image/resources/functions.sh +++ b/infrastructure/build/image/resources/functions.sh @@ -1,29 +1,29 @@ #!/bin/bash function generate-netrc-file() { - echo "machine $GIT_HOST password $AUTHTOKEN" > ~/.netrc + echo "machine $GIT_HOST password $AUTH_TOKEN" > ~/.netrc } function get-website-data() { - curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$1 $GITREPOURL + curl -H "Authorization: token $AUTH_TOKEN" -o $SOURCE_DIR/$1 $GIT_REPO_URL } function get-hash-data() { - curl -s -H "Authorization: token $AUTHTOKEN" $GITCOMMITURL | jq '.sha' + curl -s -H "Authorization: token $AUTH_TOKEN" $GIT_COMMIT_URL | jq '.sha' } function write-hash-data() { - echo $1 > $HASHFILEDIR/$2 + echo $1 > $HASHFILE_DIR/$2 } function unzip-website-data() { - unzip $SOURCEDIR/$1 -d $BUILDDIR + unzip $SOURCE_DIR/$1 -d $BUILD_DIR } function build-website() { - (cd $BUILDDIR; dir=$(ls); cd $dir; ./generate.sh;) + (cd $BUILD_DIR; dir=$(ls); cd $dir; ./generate.sh;) } function move-website-files-to-target() { - (cd $BUILDDIR; dir=$(ls); cd $dir; rsync -ru --exclude-from "/etc/exclude.pattern" --delete target/html/* $WEBSITEROOT;) + (cd $BUILD_DIR; dir=$(ls); cd $dir; rsync -ru --exclude-from "/etc/exclude.pattern" --delete target/html/* $WEBSITE_ROOT;) } diff --git a/src/main/resources/website/build-configmap.yaml b/src/main/resources/website/build-configmap.yaml index 96dbb86..f3a812a 100644 --- a/src/main/resources/website/build-configmap.yaml +++ b/src/main/resources/website/build-configmap.yaml @@ -7,5 +7,5 @@ metadata: app.kubernetes.part-of: NAME-website data: GIT_HOST: GITHOST - GITREPOURL: REPOURL - GITCOMMITURL: COMMITURL \ No newline at end of file + GIT_REPO_URL: REPOURL + GIT_COMMIT_URL: COMMITURL \ No newline at end of file diff --git a/src/main/resources/website/build-secret.yaml b/src/main/resources/website/build-secret.yaml index de659ef..71bec46 100644 --- a/src/main/resources/website/build-secret.yaml +++ b/src/main/resources/website/build-secret.yaml @@ -6,4 +6,4 @@ metadata: labels: app.kubernetes.part-of: NAME-website data: - AUTHTOKEN: TOKEN + AUTH_TOKEN: TOKEN diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index 2ed5fb3..9d7aec5 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -237,8 +237,8 @@ :labels {:app.kubernetes.part-of "test-io-website"}}, :data {:GIT_HOST "mygit.de" - :GITREPOURL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" - :GITCOMMITURL "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD"}} + :GIT_REPO_URL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" + :GIT_COMMIT_URL "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD"}} (cut/generate-build-configmap {:issuer "staging" :build-cpu-request "500m" :build-cpu-limit "1700m" @@ -260,7 +260,7 @@ :namespace "test-io", :labels {:app.kubernetes.part-of "test-io-website"}}, :data - {:AUTHTOKEN "YWJlZGpnYmFzZG9kag=="}} + {:AUTH_TOKEN "YWJlZGpnYmFzZG9kag=="}} (cut/generate-build-secret {:unique-name "test.io", :authtoken "abedjgbasdodj"})))) From 7e146a34631b03e6200924fb1953a255a52d23ba Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 12:23:13 +0100 Subject: [PATCH 10/18] renamed repo-user and forgejo-repo --- doc/architecture.md | 4 +- src/main/cljc/dda/c4k_website/core.cljc | 8 +-- src/main/cljc/dda/c4k_website/website.cljc | 20 +++---- src/main/cljs/dda/c4k_website/browser.cljs | 8 +-- src/test/cljc/dda/c4k_website/core_test.cljc | 44 ++++++++-------- .../cljc/dda/c4k_website/website_test.cljc | 52 +++++++++---------- .../resources/website-test/valid-config.yaml | 8 +-- 7 files changed, 72 insertions(+), 72 deletions(-) diff --git a/doc/architecture.md b/doc/architecture.md index ca28e46..62d36b9 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -28,12 +28,12 @@ websiteconfigs: - unique-name: "test.io" fqdns: ["test.de", "test.org", "www.test.de", "www.test.org"] forgejo-host: "codeberg.org" - forgejo-repo: "repo" + repo-name: "repo" branchname: "main" - unique-name: "example.io" fqdns: ["example.org", "www.example.com"] forgejo-host: "fineForgejoHost.net" - forgejo-repo: "repo" + repo-name: "repo" branchname: "main" mon-cfg: grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index 83c63b4..8172e15 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -20,8 +20,8 @@ (s/def ::authtoken ::web/authtoken) (s/def ::fqdns ::web/fqdns) (s/def ::forgejo-host ::web/forgejo-host) -(s/def ::repo-user ::web/repo-user) -(s/def ::forgejo-repo ::web/forgejo-repo) +(s/def ::repo-owner ::web/repo-owner) +(s/def ::repo-name ::web/repo-name) (s/def ::branchname ::web/branchname) (s/def ::build-cpu-request ::web/build-cpu-request) (s/def ::build-memory-request ::web/build-memory-request) @@ -32,8 +32,8 @@ (def websiteconfig? (s/keys :req-un [::unique-name ::fqdns ::forgejo-host - ::repo-user - ::forgejo-repo + ::repo-owner + ::repo-name ::branchname] :opt-un [::issuer ::volume-size diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index 4c61147..c1fbf12 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -20,9 +20,9 @@ (s/def ::authtoken pred/bash-env-string?) (s/def ::fqdns (s/coll-of pred/fqdn-string?)) (s/def ::forgejo-host pred/fqdn-string?) -(s/def ::forgejo-repo string?) +(s/def ::repo-name string?) (s/def ::branchname string?) -(s/def ::repo-user string?) +(s/def ::repo-owner string?) (s/def ::build-cpu-request string?) (s/def ::build-memory-request string?) (s/def ::build-cpu-limit string?) @@ -33,8 +33,8 @@ (def websiteconfig? (s/keys :req-un [::unique-name ::fqdns ::forgejo-host - ::repo-user - ::forgejo-repo + ::repo-owner + ::repo-name ::branchname ::build-cpu-request ::build-cpu-limit @@ -108,8 +108,8 @@ [config websiteconfig?] (let [{:keys [unique-name forgejo-host - repo-user - forgejo-repo + repo-owner + repo-name branchname]} config name (replace-dots-by-minus unique-name)] (-> @@ -118,13 +118,13 @@ (cm/replace-all-matching-values-by-new-value "GITHOST" forgejo-host) (cm/replace-all-matching-values-by-new-value "REPOURL" (generate-gitrepourl forgejo-host - repo-user - forgejo-repo + repo-owner + repo-name branchname)) (cm/replace-all-matching-values-by-new-value "COMMITURL" (generate-gitcommiturl forgejo-host - repo-user - forgejo-repo))))) + repo-owner + repo-name))))) (defn-spec generate-build-secret pred/map-or-seq? [auth websiteauth?] diff --git a/src/main/cljs/dda/c4k_website/browser.cljs b/src/main/cljs/dda/c4k_website/browser.cljs index 8ddddc0..cae51b0 100644 --- a/src/main/cljs/dda/c4k_website/browser.cljs +++ b/src/main/cljs/dda/c4k_website/browser.cljs @@ -29,15 +29,15 @@ [{:unique-name \"test.io\", :fqdns [\"test.de\" \"www.test.de\"], :forgejo-host \"githost.de\", - :repo-user \"someuser\", - :forgejo-repo \"repo\", + :repo-owner \"someuser\", + :repo-name \"repo\", :branchname \"main\", :sha256sum-output \"123456789ab123cd345de script-file-name.sh\"} {:unique-name \"example.io \", :fqdns [\"example.org\" \"www.example.org\"], :forgejo-host \"githost.org\", - :repo-user \"someuser\", - :forgejo-repo \"repo\", + :repo-owner \"someuser\", + :repo-name \"repo\", :branchname \"main\", :build-cpu-request \"1500m\", :build-cpu-limit \"3000m\", diff --git a/src/test/cljc/dda/c4k_website/core_test.cljc b/src/test/cljc/dda/c4k_website/core_test.cljc index e028819..4f35f3c 100644 --- a/src/test/cljc/dda/c4k_website/core_test.cljc +++ b/src/test/cljc/dda/c4k_website/core_test.cljc @@ -30,14 +30,14 @@ [{:unique-name "example.io" :fqdns ["example.org", "www.example.com"] :forgejo-host "finegitehost.net" - :repo-user "someuser" - :forgejo-repo "repo" + :repo-owner "someuser" + :repo-name "repo" :branchname "main"} {:unique-name "test.io" :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] :forgejo-host "gitlab.de" - :repo-user "someuser" - :forgejo-repo "repo" + :repo-owner "someuser" + :repo-name "repo" :branchname "main"}]}) (def websites2 @@ -45,14 +45,14 @@ [{:unique-name "test.io" :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"] :forgejo-host "gitlab.de" - :repo-user "someuser" - :forgejo-repo "repo" + :repo-owner "someuser" + :repo-name "repo" :branchname "main"} {:unique-name "example.io" :fqdns ["example.org", "www.example.com"] :forgejo-host "finegitehost.net" - :repo-user "someuser" - :forgejo-repo "repo" + :repo-owner "someuser" + :repo-name "repo" :branchname "main"}]}) (def auth1 @@ -73,8 +73,8 @@ {:unique-name "example.io", :fqdns ["example.org" "www.example.com"], :forgejo-host "finegitehost.net", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :branchname "main"}) (def flattened-and-reduced-auth @@ -87,14 +87,14 @@ [{:unique-name "example.io", :fqdns ["example.org" "www.example.com"], :forgejo-host "finegitehost.net", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :branchname "main"}, {:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-host "gitlab.de", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :branchname "main", :sha256sum-output "123456789ab123cd345de script-file-name.sh"}], :mon-cfg {:grafana-cloud-url "url-for-your-prom-remote-write-endpoint", :cluster-name "jitsi", :cluster-stage "test"}} @@ -104,15 +104,15 @@ [{:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-host "gitlab.de", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :branchname "main", :sha256sum-output "123456789ab123cd345de script-file-name.sh"} {:unique-name "example.io", :fqdns ["example.org" "www.example.com"], :forgejo-host "finegitehost.net", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :branchname "main"}], :mon-cfg {:grafana-cloud-url "url-for-your-prom-remote-write-endpoint", :cluster-name "jitsi", :cluster-stage "test"}})))) @@ -165,8 +165,8 @@ (cut/generate-ingress {:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-host "gitlab.de", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :sha256sum-output "123456789ab123cd345de script-file-name.sh", :issuer "staging", :branchname "main", @@ -182,8 +182,8 @@ (cut/generate-ingress {:unique-name "test.io", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], :forgejo-host "gitlab.de", - :repo-user "someuser", - :forgejo-repo "repo", + :repo-owner "someuser", + :repo-name "repo", :sha256sum-output "123456789ab123cd345de script-file-name.sh", :issuer "staging", :branchname "main", diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index 9d7aec5..47a93d9 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -39,8 +39,8 @@ :redirects [["/products.html", "/offer.html"] ["/one-more", "/redirect"]] :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]} 2))) @@ -54,8 +54,8 @@ :unique-name "test.io", :redirects [] :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]} 0)))) @@ -64,9 +64,9 @@ (deftest should-generate-resource-requests (is (= {:requests {:cpu "1500m", :memory "512Mi"}, :limits {:cpu "3000m", :memory "1024Mi"}} (-> (cut/generate-nginx-deployment {:forgejo-host "gitlab.de", - :repo-user "someuser", + :repo-owner "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], - :forgejo-repo "repo", + :repo-name "repo", :issuer "staging", :branchname "main", :unique-name "test.io", @@ -79,9 +79,9 @@ :spec :template :spec :initContainers first :resources))) (is (= "test-io" (-> (cut/generate-nginx-deployment {:forgejo-host "gitlab.de", - :repo-user "someuser", + :repo-owner "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], - :forgejo-repo "repo", + :repo-name "repo", :issuer "staging", :branchname "main", :unique-name "test.io", @@ -104,8 +104,8 @@ :redirects [["/products.html", "/offer.html"] ["/one-more", "/redirect"]] :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))) " /offer.html permanent;\n")) @@ -120,8 +120,8 @@ :redirects [["/products.html", "/offer.html"] ["/one-more", "/redirect"]] :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))) " /redirect permanent;\n")) @@ -135,8 +135,8 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))) "server_name test.de www.test.de test-it.de www.test-it.de;")) @@ -154,8 +154,8 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}) :data)))) @@ -178,8 +178,8 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))) @@ -222,9 +222,9 @@ :build-memory-limit "512Mi" :volume-size "3" :forgejo-host "gitlab.de", - :repo-user "someuser", + :repo-owner "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], - :forgejo-repo "repo", + :repo-name "repo", :branchname "main", :unique-name "test.io", :redirects []})))) @@ -246,9 +246,9 @@ :build-memory-limit "512Mi" :volume-size "3" :forgejo-host "mygit.de", - :repo-user "someuser", + :repo-owner "someuser", :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], - :forgejo-repo "repo", + :repo-name "repo", :branchname "main", :unique-name "test.io", :redirects []})))) @@ -284,8 +284,8 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))) @@ -309,7 +309,7 @@ :unique-name "test.io", :redirects [], :forgejo-host "gitea.evilorg", - :repo-user "someuser", - :forgejo-repo "none", + :repo-owner "someuser", + :repo-name "none", :branchname "mablain", :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))) \ No newline at end of file diff --git a/src/test/resources/website-test/valid-config.yaml b/src/test/resources/website-test/valid-config.yaml index 6d0028a..141944b 100644 --- a/src/test/resources/website-test/valid-config.yaml +++ b/src/test/resources/website-test/valid-config.yaml @@ -3,8 +3,8 @@ websiteconfigs: - unique-name: "test.io" fqdns: ["test.de", "test.org", "www.test.de", "www.test.org"] forgejo-host: "codeberg.org" - repo-user: "someuser" - forgejo-repo: "repo" + repo-owner: "someuser" + repo-name: "repo" branchname: "main" redirects: - ["/products.html", "/offer.html"] @@ -12,8 +12,8 @@ websiteconfigs: - unique-name: "example.io" fqdns: ["example.org", "www.example.com"] forgejo-host: "fineForgejoHost.net" - repo-user: "someotheruser" - forgejo-repo: "repo" + repo-owner: "someotheruser" + repo-name: "repo" branchname: "main" mon-cfg: grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" From d1764c64ed361e215e4529030b7d5824c332f069 Mon Sep 17 00:00:00 2001 From: Clemens Date: Thu, 7 Mar 2024 13:29:35 +0100 Subject: [PATCH 11/18] added lein inst --- project.clj | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/project.clj b/project.clj index 9f6cfdd..41813ba 100644 --- a/project.clj +++ b/project.clj @@ -33,4 +33,16 @@ ["vcs" "commit"] ["vcs" "tag" "v" "--no-sign"] ["change" "version" "leiningen.release/bump-version"]] + :aliases {"native" ["shell" + "native-image" + "--report-unsupported-elements-at-runtime" + "--initialize-at-build-time" + "-jar" "target/uberjar/c4k-website-standalone.jar" + "-H:ResourceConfigurationFiles=graalvm-resource-config.json" + "-H:Log=registerResource" + "-H:Name=target/graalvm/${:name}"] + "inst" ["shell" + "sh" + "-c" + "lein uberjar && sudo install -m=755 target/uberjar/c4k-website-standalone.jar /usr/local/bin/c4k-website-standalone.jar"]} ) From 26a687c4883acf8e170eb064813d16e2fac31e93 Mon Sep 17 00:00:00 2001 From: Clemens Date: Mon, 11 Mar 2024 12:02:47 +0100 Subject: [PATCH 12/18] fix generation of all build-secrets --- src/main/cljc/dda/c4k_website/core.cljc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index 8172e15..5871074 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -132,7 +132,7 @@ (web/generate-hash-state-pvc curr-flat-websiteconfig) (web/generate-build-cron curr-flat-websiteconfig) (web/generate-build-configmap curr-flat-websiteconfig) - (web/generate-build-secret (flatten-and-reduce-auth auth))] + (web/generate-build-secret (flatten-and-reduce-auth sorted-auth))] (generate-ingress curr-flat-websiteconfig))))))) (defn-spec k8s-objects cp/map-or-seq? From 9e8af08ec25c2cca2492c44d1cb8ef499fe6b0f4 Mon Sep 17 00:00:00 2001 From: Clemens Date: Mon, 11 Mar 2024 12:30:30 +0100 Subject: [PATCH 13/18] added configMapRef for nginx initContainer --- src/main/resources/website/nginx-deployment.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/website/nginx-deployment.yaml b/src/main/resources/website/nginx-deployment.yaml index 8ccbb55..a9f8f8f 100644 --- a/src/main/resources/website/nginx-deployment.yaml +++ b/src/main/resources/website/nginx-deployment.yaml @@ -46,6 +46,8 @@ spec: memory: BUILD_MEMORY_LIMIT command: ["/entrypoint.sh"] envFrom: + - configMapRef: + name: build-configmap - secretRef: name: build-secret volumeMounts: From 889b95f572f689da96a8e59a8c405e789e705f7d Mon Sep 17 00:00:00 2001 From: Clemens Date: Mon, 11 Mar 2024 14:41:15 +0100 Subject: [PATCH 14/18] revert env var renaming --- infrastructure/build/image/Dockerfile | 6 +++--- infrastructure/build/image/resources/entrypoint.sh | 10 +++++----- infrastructure/build/image/resources/functions.sh | 14 +++++++------- src/main/resources/website/build-configmap.yaml | 6 +++--- src/main/resources/website/build-secret.yaml | 2 +- src/test/cljc/dda/c4k_website/website_test.cljc | 8 ++++---- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/infrastructure/build/image/Dockerfile b/infrastructure/build/image/Dockerfile index 84e606e..8647345 100644 --- a/infrastructure/build/image/Dockerfile +++ b/infrastructure/build/image/Dockerfile @@ -3,9 +3,9 @@ FROM clojure:lein # Prepare Entrypoint Script ADD resources /tmp -ENV BUILD_DIR="/etc/website" -ENV SOURCE_DIR="/etc/websitesource" +ENV BUILDDIR="/etc/website" +ENV SOURCEDIR="/etc/websitesource" ENV WEBSITE_ROOT="/var/www/html/website/" -ENV HASHFILE_DIR="/var/hashfile.d" +ENV HASHFILEDIR="/var/hashfile.d" RUN /tmp/install.sh diff --git a/infrastructure/build/image/resources/entrypoint.sh b/infrastructure/build/image/resources/entrypoint.sh index e50684b..485a546 100755 --- a/infrastructure/build/image/resources/entrypoint.sh +++ b/infrastructure/build/image/resources/entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/bash -mkdir $BUILD_DIR -mkdir $SOURCE_DIR +mkdir $BUILDDIR +mkdir $SOURCEDIR set -euo pipefail @@ -11,15 +11,15 @@ filename="website.zip" hashfilename="hashfile" echo "Check for new content" -touch $HASHFILE_DIR/$hashfilename -currentHash=$( cat $HASHFILE_DIR/$hashfilename ) +touch $HASHFILEDIR/$hashfilename +currentHash=$( cat $HASHFILEDIR/$hashfilename ) newHash=$( get-hash-data ) if [[ $currentHash == $newHash ]] then echo "Nothing to do" else - echo $currentHash > $HASHFILE_DIR/$hashfilename + echo $currentHash > $HASHFILEDIR/$hashfilename echo "Generate .netrc file" generate-netrc-file echo "Downloading website data" diff --git a/infrastructure/build/image/resources/functions.sh b/infrastructure/build/image/resources/functions.sh index 60f192d..10a43ad 100644 --- a/infrastructure/build/image/resources/functions.sh +++ b/infrastructure/build/image/resources/functions.sh @@ -1,29 +1,29 @@ #!/bin/bash function generate-netrc-file() { - echo "machine $GIT_HOST password $AUTH_TOKEN" > ~/.netrc + echo "machine $GITHOST password $AUTHTOKEN" > ~/.netrc } function get-website-data() { - curl -H "Authorization: token $AUTH_TOKEN" -o $SOURCE_DIR/$1 $GIT_REPO_URL + curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$1 $GITREPOURL } function get-hash-data() { - curl -s -H "Authorization: token $AUTH_TOKEN" $GIT_COMMIT_URL | jq '.sha' + curl -s -H "Authorization: token $AUTHTOKEN" $GITCOMMITURL | jq '.sha' } function write-hash-data() { - echo $1 > $HASHFILE_DIR/$2 + echo $1 > $HASHFILEDIR/$2 } function unzip-website-data() { - unzip $SOURCE_DIR/$1 -d $BUILD_DIR + unzip $SOURCEDIR/$1 -d $BUILDDIR } function build-website() { - (cd $BUILD_DIR; dir=$(ls); cd $dir; ./generate.sh;) + (cd $BUILDDIR; dir=$(ls); cd $dir; ./generate.sh;) } function move-website-files-to-target() { - (cd $BUILD_DIR; dir=$(ls); cd $dir; rsync -ru --exclude-from "/etc/exclude.pattern" --delete target/html/* $WEBSITE_ROOT;) + (cd $BUILDDIR; dir=$(ls); cd $dir; rsync -ru --exclude-from "/etc/exclude.pattern" --delete target/html/* $WEBSITE_ROOT;) } diff --git a/src/main/resources/website/build-configmap.yaml b/src/main/resources/website/build-configmap.yaml index f3a812a..6c877cc 100644 --- a/src/main/resources/website/build-configmap.yaml +++ b/src/main/resources/website/build-configmap.yaml @@ -6,6 +6,6 @@ metadata: labels: app.kubernetes.part-of: NAME-website data: - GIT_HOST: GITHOST - GIT_REPO_URL: REPOURL - GIT_COMMIT_URL: COMMITURL \ No newline at end of file + GITHOST: GITHOST + GITREPOURL: REPOURL + GITCOMMITURL: COMMITURL \ No newline at end of file diff --git a/src/main/resources/website/build-secret.yaml b/src/main/resources/website/build-secret.yaml index 71bec46..de659ef 100644 --- a/src/main/resources/website/build-secret.yaml +++ b/src/main/resources/website/build-secret.yaml @@ -6,4 +6,4 @@ metadata: labels: app.kubernetes.part-of: NAME-website data: - AUTH_TOKEN: TOKEN + AUTHTOKEN: TOKEN diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index 47a93d9..b052adf 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -236,9 +236,9 @@ :namespace "test-io", :labels {:app.kubernetes.part-of "test-io-website"}}, :data - {:GIT_HOST "mygit.de" - :GIT_REPO_URL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" - :GIT_COMMIT_URL "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD"}} + {:GITHOST "mygit.de" + :GITREPOURL "https://mygit.de/api/v1/repos/someuser/repo/archive/main.zip" + :GITCOMMITURL "https://mygit.de/api/v1/repos/someuser/repo/git/commits/HEAD"}} (cut/generate-build-configmap {:issuer "staging" :build-cpu-request "500m" :build-cpu-limit "1700m" @@ -260,7 +260,7 @@ :namespace "test-io", :labels {:app.kubernetes.part-of "test-io-website"}}, :data - {:AUTH_TOKEN "YWJlZGpnYmFzZG9kag=="}} + {:AUTHTOKEN "YWJlZGpnYmFzZG9kag=="}} (cut/generate-build-secret {:unique-name "test.io", :authtoken "abedjgbasdodj"})))) From 56a08653e9e89f4a682f920d193b85df0fa6dc72 Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 15 Mar 2024 10:03:36 +0100 Subject: [PATCH 15/18] rename website root envvar --- infrastructure/build/image/Dockerfile | 2 +- infrastructure/build/image/resources/functions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/build/image/Dockerfile b/infrastructure/build/image/Dockerfile index 8647345..4d3289b 100644 --- a/infrastructure/build/image/Dockerfile +++ b/infrastructure/build/image/Dockerfile @@ -5,7 +5,7 @@ ADD resources /tmp ENV BUILDDIR="/etc/website" ENV SOURCEDIR="/etc/websitesource" -ENV WEBSITE_ROOT="/var/www/html/website/" +ENV WEBSITEROOT="/var/www/html/website/" ENV HASHFILEDIR="/var/hashfile.d" RUN /tmp/install.sh diff --git a/infrastructure/build/image/resources/functions.sh b/infrastructure/build/image/resources/functions.sh index 10a43ad..896e2d0 100644 --- a/infrastructure/build/image/resources/functions.sh +++ b/infrastructure/build/image/resources/functions.sh @@ -25,5 +25,5 @@ function build-website() { } function move-website-files-to-target() { - (cd $BUILDDIR; dir=$(ls); cd $dir; rsync -ru --exclude-from "/etc/exclude.pattern" --delete target/html/* $WEBSITE_ROOT;) + (cd $BUILDDIR; dir=$(ls); cd $dir; rsync -ru --exclude-from "/etc/exclude.pattern" --delete target/html/* $WEBSITEROOT;) } From 3944c28562555c3e0571091ae60d1276125d91a5 Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 15 Mar 2024 11:01:35 +0100 Subject: [PATCH 16/18] adjust calling of generate.sh --- infrastructure/build/image/resources/functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/build/image/resources/functions.sh b/infrastructure/build/image/resources/functions.sh index 896e2d0..88c5ec4 100644 --- a/infrastructure/build/image/resources/functions.sh +++ b/infrastructure/build/image/resources/functions.sh @@ -21,7 +21,7 @@ function unzip-website-data() { } function build-website() { - (cd $BUILDDIR; dir=$(ls); cd $dir; ./generate.sh;) + (cd $BUILDDIR; dir=$(ls); cd $dir; bash generate.sh;) } function move-website-files-to-target() { From e1d43cddc5e057b89425915da2355c05bd8166f4 Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 15 Mar 2024 11:52:10 +0100 Subject: [PATCH 17/18] added rate limits --- src/main/cljc/dda/c4k_website/core.cljc | 16 +++++++++++++--- .../resources/website-test/valid-config.yaml | 1 + 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index 5871074..d3f3ed5 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -16,6 +16,8 @@ (s/def ::unique-name ::web/unique-name) (s/def ::issuer ::web/issuer) (s/def ::volume-size ::web/volume-size) +(s/def ::average-rate ::ing/average-rate) +(s/def ::burst-rate ::ing/burst-rate) (s/def ::authtoken ::web/authtoken) (s/def ::fqdns ::web/fqdns) @@ -51,7 +53,9 @@ (def config? (s/keys :req-un [::websiteconfigs] :opt-un [::issuer ::volume-size - ::mon-cfg])) + ::mon-cfg + ::average-rate + ::burst-rate])) (def auth? (s/keys :req-un [::websiteauths] :opt-un [::mon-auth])) @@ -64,7 +68,9 @@ :build-memory-request "256Mi" :build-memory-limit "512Mi" :volume-size "3" - :redirects []}) + :redirects [] + :average-rate 50 + :burst-rate 30}) (defn-spec sort-config map? [unsorted-config config?] @@ -86,7 +92,11 @@ (when (contains? config :issuer) {:issuer (config :issuer)}) (when (contains? config :volume-size) - {:volume-size (config :volume-size)})))) + {:volume-size (config :volume-size)}) + (when (contains? config :average-rate) + {:average-rate (config :average-rate)}) + (when (contains? config :burst-rate) + {:burst-rate (config :burst-rate)})))) (defn-spec flatten-and-reduce-auth map? [auth auth?] diff --git a/src/test/resources/website-test/valid-config.yaml b/src/test/resources/website-test/valid-config.yaml index 141944b..5bea303 100644 --- a/src/test/resources/website-test/valid-config.yaml +++ b/src/test/resources/website-test/valid-config.yaml @@ -19,3 +19,4 @@ mon-cfg: grafana-cloud-url: "url-for-your-prom-remote-write-endpoint" cluster-name: "website" cluster-stage: "test" +average-rate: 50 From a1803e79a62598253739894b8867b117ac880eaa Mon Sep 17 00:00:00 2001 From: Clemens Date: Fri, 15 Mar 2024 11:54:51 +0100 Subject: [PATCH 18/18] adjust defaults --- src/main/cljc/dda/c4k_website/core.cljc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index d3f3ed5..f9122a1 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -69,8 +69,8 @@ :build-memory-limit "512Mi" :volume-size "3" :redirects [] - :average-rate 50 - :burst-rate 30}) + :average-rate 20 + :burst-rate 40}) (defn-spec sort-config map? [unsorted-config config?]