diff --git a/infrastructure/c4k-website-build/image/Dockerfile b/infrastructure/c4k-website-build/image/Dockerfile index 270be95..4d3289b 100644 --- a/infrastructure/c4k-website-build/image/Dockerfile +++ b/infrastructure/c4k-website-build/image/Dockerfile @@ -3,8 +3,9 @@ FROM clojure:lein # Prepare Entrypoint Script ADD resources /tmp -ENV SOURCEDIR="/etc/websitesource" ENV BUILDDIR="/etc/website" +ENV SOURCEDIR="/etc/websitesource" ENV WEBSITEROOT="/var/www/html/website/" +ENV HASHFILEDIR="/var/hashfile.d" RUN /tmp/install.sh diff --git a/infrastructure/c4k-website-build/image/resources/entrypoint.sh b/infrastructure/c4k-website-build/image/resources/entrypoint.sh index ddb1a37..f18d5bd 100755 --- a/infrastructure/c4k-website-build/image/resources/entrypoint.sh +++ b/infrastructure/c4k-website-build/image/resources/entrypoint.sh @@ -3,18 +3,34 @@ mkdir $BUILDDIR mkdir $SOURCEDIR -set -o nounset -set -o xtrace -set -o errexit -set -eo pipefail +set -euo pipefail source /usr/local/bin/functions.sh -echo "Downloading website" -get-and-unzip-website-data -echo "Executing Custom Scripts, if applicable" -execute-scripts-when-existing -echo "Building website" -build-and-extract-website -echo "Moving files" -move-website-files-to-target +filename="website.zip" +hashfilename="hashfile" + +echo "Check for new content" +touch $HASHFILEDIR/$hashfilename +currentHash=$( cat $HASHFILEDIR/$hashfilename ) +newHash=$( get-hash-data ) + +if [[ $currentHash == $newHash ]] + then + echo "Nothing to do" + else + echo $currentHash > $HASHFILEDIR/$hashfilename + echo "Downloading website data" + get-website-data $filename + unzip-website-data $filename + echo "Executing Custom Scripts, if applicable" + execute-scripts-when-existing + echo "Building website" + build-website + echo "Moving files" + move-website-files-to-target +fi + + + + diff --git a/infrastructure/c4k-website-build/image/resources/functions.sh b/infrastructure/c4k-website-build/image/resources/functions.sh index 323831a..7ab68db 100644 --- a/infrastructure/c4k-website-build/image/resources/functions.sh +++ b/infrastructure/c4k-website-build/image/resources/functions.sh @@ -1,12 +1,22 @@ #!/bin/bash -function get-and-unzip-website-data() { - filename="website.zip" - curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$filename $GITREPOURL - unzip $SOURCEDIR/$filename -d $BUILDDIR +function get-website-data() { + curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$1 $GITREPOURL } -function execute-scripts-when-existing { +function get-hash-data() { + curl -s -H "Authorization: token $AUTHTOKEN" $GITCOMMITURL | jq '.sha' +} + +function write-hash-data() { + echo $1 > $HASHFILEDIR/$2 +} + +function unzip-website-data() { + unzip $SOURCEDIR/$1 -d $BUILDDIR +} + +function execute-scripts-when-existing() { websitedir=$(ls $BUILDDIR) if [[ -f $BUILDDIR/$websitedir/$SCRIPTFILE ]] then @@ -26,7 +36,7 @@ function execute-scripts-when-existing { fi } -function build-and-extract-website() { +function build-website() { (cd $BUILDDIR; dir=$(ls); cd $dir; lein run;) } diff --git a/infrastructure/c4k-website-build/image/resources/install.sh b/infrastructure/c4k-website-build/image/resources/install.sh index 9137061..cd78ccb 100755 --- a/infrastructure/c4k-website-build/image/resources/install.sh +++ b/infrastructure/c4k-website-build/image/resources/install.sh @@ -2,7 +2,7 @@ apt update > /dev/null; -apt install -y unzip rsync +apt install -y unzip rsync jq imagemagick mkdir /etc/lein/ diff --git a/infrastructure/c4k-website-build/test/Dockerfile b/infrastructure/c4k-website-build/test/Dockerfile index f2e19b6..41aadf5 100644 --- a/infrastructure/c4k-website-build/test/Dockerfile +++ b/infrastructure/c4k-website-build/test/Dockerfile @@ -1,4 +1,4 @@ -FROM c4k-jira-backup +FROM c4k-website-build RUN apt update RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless diff --git a/infrastructure/c4k-website-build/test/serverspec.edn b/infrastructure/c4k-website-build/test/serverspec.edn index 000622f..87038f9 100644 --- a/infrastructure/c4k-website-build/test/serverspec.edn +++ b/infrastructure/c4k-website-build/test/serverspec.edn @@ -1,5 +1,4 @@ -{:file [{:path "/usr/local/bin/install.sh" :mod "700"} +{:file [{:path "/entrypoint.sh" :mod "700"} {:path "/usr/local/bin/functions.sh" :mod "700"} - {:path "/usr/local/bin/exclude.pattern" :mod "700"} - {:path "/usr/local/bin/project.clj" :mod "700"} - {:path "/entrypoint.sh" :mod "700"}]} + {:path "/etc/exclude.pattern" :mod "700"} + {:path "/etc/lein/project.clj" :mod "700"}]} diff --git a/src/main/cljc/dda/c4k_website/core.cljc b/src/main/cljc/dda/c4k_website/core.cljc index d4fc44b..6a4d92f 100644 --- a/src/main/cljc/dda/c4k_website/core.cljc +++ b/src/main/cljc/dda/c4k_website/core.cljc @@ -45,6 +45,7 @@ (website/generate-nginx-configmap (flatten-and-reduce-config config)) (website/generate-nginx-service (flatten-and-reduce-config config)) (website/generate-website-content-volume (flatten-and-reduce-config config)) + (website/generate-hashfile-volume (flatten-and-reduce-config config)) (website/generate-website-ingress (flatten-and-reduce-config config)) (website/generate-website-certificate (flatten-and-reduce-config config)) (website/generate-website-build-cron (flatten-and-reduce-config config)) diff --git a/src/main/cljc/dda/c4k_website/website.cljc b/src/main/cljc/dda/c4k_website/website.cljc index c82cad2..f9177f0 100644 --- a/src/main/cljc/dda/c4k_website/website.cljc +++ b/src/main/cljc/dda/c4k_website/website.cljc @@ -98,7 +98,14 @@ branch string?] (str "https://" host "/api/v1/repos/" user "/" repo "/archive/" branch ".zip")) -(defn-spec replace-all-matching-subvalues-in-string-start pred/map-or-seq? +; https://your.gitea.host/api/v1/repos///git/commits/HEAD +(defn-spec generate-gitcommiturl string? + [host pred/fqdn-string? + repo string? + user string?] + (str "https://" host "/api/v1/repos/" user "/" repo "/git/" "commits/" "HEAD")) + +(defn-spec replace-all-matching-substrings-beginning-with pred/map-or-seq? [col pred/map-or-seq? value-to-partly-match string? value-to-inplace string?] @@ -114,7 +121,7 @@ (-> (yaml/load-as-edn resource-file) (assoc-in [:metadata :labels :app.kubernetes.part-of] (generate-app-name unique-name)) - (replace-all-matching-subvalues-in-string-start "NAME" (replace-dots-by-minus unique-name))))) + (replace-all-matching-substrings-beginning-with "NAME" (replace-dots-by-minus unique-name))))) (defn-spec replace-build-data pred/map-or-seq? [resource-file string? @@ -139,6 +146,7 @@ "website/website-build-cron.yaml" (rc/inline "website/website-build-cron.yaml") "website/website-build-secret.yaml" (rc/inline "website/website-build-secret.yaml") "website/website-content-volume.yaml" (rc/inline "website/website-content-volume.yaml") + "website/hashfile-volume.yaml" (rc/inline "website/hashfile-volume.yaml") (throw (js/Error. "Undefined Resource!"))))) (defn-spec generate-website-ingress pred/map-or-seq? @@ -185,6 +193,10 @@ (replace-common-data "website/website-content-volume.yaml" config) (cm/replace-all-matching-values-by-new-value "WEBSITESTORAGESIZE" (str volume-size "Gi"))))) +(defn-spec generate-hashfile-volume pred/map-or-seq? + [config flattened-and-reduced-config?] + (replace-common-data "website/hashfile-volume.yaml" config)) + (defn-spec generate-website-build-cron pred/map-or-seq? [config flattened-and-reduced-config?] (replace-build-data "website/website-build-cron.yaml" config)) @@ -199,9 +211,15 @@ (-> (replace-common-data "website/website-build-secret.yaml" auth) (cm/replace-all-matching-values-by-new-value "TOKEN" (b64/encode authtoken)) - (cm/replace-all-matching-values-by-new-value "URL" (b64/encode - (generate-gitrepourl + (cm/replace-all-matching-values-by-new-value "REPOURL" (b64/encode + (generate-gitrepourl gitea-host gitea-repo username - branchname)))))) \ No newline at end of file + branchname))) + (cm/replace-all-matching-values-by-new-value "COMMITURL" (b64/encode + (generate-gitcommiturl + gitea-host + gitea-repo + username)))))) + diff --git a/src/main/resources/website/hashfile-volume.yaml b/src/main/resources/website/hashfile-volume.yaml new file mode 100644 index 0000000..99f35db --- /dev/null +++ b/src/main/resources/website/hashfile-volume.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: NAME-hashfile-volume + namespace: default + labels: + app: NAME-nginx + app.kubernetes.part-of: NAME-website +spec: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 16Mi + \ No newline at end of file diff --git a/src/main/resources/website/nginx-deployment.yaml b/src/main/resources/website/nginx-deployment.yaml index 234bc7e..9133cd2 100644 --- a/src/main/resources/website/nginx-deployment.yaml +++ b/src/main/resources/website/nginx-deployment.yaml @@ -52,6 +52,8 @@ spec: volumeMounts: - name: content-volume mountPath: /var/www/html/website + - name: hashfile-volume + mountPath: /var/hashfile.d volumes: - name: nginx-config-volume configMap: @@ -68,4 +70,7 @@ spec: - name: content-volume persistentVolumeClaim: claimName: NAME-content-volume + - name: hashfile-volume + persistentVolumeClaim: + claimName: NAME-hashfile-volume \ No newline at end of file diff --git a/src/main/resources/website/website-build-cron.yaml b/src/main/resources/website/website-build-cron.yaml index 371444c..1c79d09 100644 --- a/src/main/resources/website/website-build-cron.yaml +++ b/src/main/resources/website/website-build-cron.yaml @@ -35,9 +35,14 @@ spec: volumeMounts: - name: content-volume mountPath: /var/www/html/website + - name: hashfile-volume + mountPath: /var/hashfile.d volumes: - name: content-volume persistentVolumeClaim: claimName: NAME-content-volume + - name: hashfile-volume + persistentVolumeClaim: + claimName: NAME-hashfile-volume restartPolicy: OnFailure \ No newline at end of file diff --git a/src/main/resources/website/website-build-secret.yaml b/src/main/resources/website/website-build-secret.yaml index 405eaf8..f173d46 100644 --- a/src/main/resources/website/website-build-secret.yaml +++ b/src/main/resources/website/website-build-secret.yaml @@ -6,6 +6,5 @@ metadata: app.kubernetes.part-of: NAME-website data: AUTHTOKEN: TOKEN - GITREPOURL: URL - - \ No newline at end of file + GITREPOURL: REPOURL + GITCOMMITURL: COMMITURL diff --git a/src/test/cljc/dda/c4k_website/website_test.cljc b/src/test/cljc/dda/c4k_website/website_test.cljc index bf92958..51facb0 100644 --- a/src/test/cljc/dda/c4k_website/website_test.cljc +++ b/src/test/cljc/dda/c4k_website/website_test.cljc @@ -12,6 +12,7 @@ (st/instrument `cut/generate-nginx-deployment) (st/instrument `cut/generate-nginx-service) (st/instrument `cut/generate-website-content-volume) +(st/instrument `cut/generate-hashfile-volume) (st/instrument `cut/generate-website-ingress) (st/instrument `cut/generate-website-certificate) (st/instrument `cut/generate-website-build-cron) @@ -101,11 +102,12 @@ [{:image "domaindrivenarchitecture/c4k-website-build", :name "test-io-init-build-container", :imagePullPolicy "IfNotPresent", - :resources {:requests {:cpu "1000m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}}, + :resources {:requests {:cpu "500m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}}, :command ["/entrypoint.sh"], :envFrom [{:secretRef {:name "test-io-secret"}}], :env [{:name "SHA256SUM", :value "123456789ab123cd345de"} {:name "SCRIPTFILE", :value "script-file-name.sh"}], - :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}]}], + :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"} + {:name "hashfile-volume", :mountPath "/var/hashfile.d"}]}], :volumes [{:name "nginx-config-volume", :configMap @@ -115,7 +117,8 @@ {:key "website.conf", :path "conf.d/website.conf"} {:key "mime.types", :path "mime.types"}]}} {:name "log", :emptyDir {}} - {:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}]}}}} + {:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}} + {:name "hashfile-volume", :persistentVolumeClaim {:claimName "test-io-hashfile-volume"}}]}}}} (cut/generate-nginx-deployment {:authtoken "abedjgbasdodj", :gitea-host "gitlab.de", :username "someuser", @@ -196,8 +199,10 @@ :command ["/entrypoint.sh"], :envFrom [{:secretRef {:name "test-io-secret"}}], :env [{:name "SHA256SUM", :value "123456789ab123cd345de"} {:name "SCRIPTFILE", :value "script-file-name.sh"}], - :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}]}], - :volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}], + :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"} + {:name "hashfile-volume", :mountPath "/var/hashfile.d"}]}], + :volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}} + {:name "hashfile-volume", :persistentVolumeClaim {:claimName "test-io-hashfile-volume"}}], :restartPolicy "OnFailure"}}}}}} (cut/generate-website-build-cron {:authtoken "abedjgbasdodj", :gitea-host "gitlab.de", @@ -209,30 +214,25 @@ :branchname "main", :unique-name "test.io"})))) + + (deftest should-generate-website-build-secret - (is (= {:name-c1 "test-io-secret", - :name-c2 "test-org-secret", - :AUTHTOKEN-c1 (b64/encode "token1"), - :AUTHTOKEN-c2 (b64/encode "token2"), - :GITREPOURL-c1 (b64/encode "https://gitlab.org/api/v1/repos/dumpty/websitebau/archive/testname.zip"), - :GITREPOURL-c2 (b64/encode "https://github.com/api/v1/repos/humpty/websitedachs/archive/testname.zip"), - :app.kubernetes.part-of-c1 "test-io-website", - :app.kubernetes.part-of-c2 "test-org-website"} - (th/map-diff (cut/generate-website-build-secret {:unique-name "test.io", - :authtoken "token1", - :gitea-host "gitlab.org", - :gitea-repo "websitebau", - :username "dumpty", - :branchname "testname", - :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]} - ) - (cut/generate-website-build-secret {:unique-name "test.org", - :authtoken "token2", - :gitea-host "github.com", - :gitea-repo "websitedachs", - :username "humpty", - :branchname "testname", - :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}))))) + (is (= {:apiVersion "v1", + :kind "Secret", + :metadata {:name "test-io-secret", :labels {:app.kubernetes.part-of "test-io-website"}}, + :data + {:AUTHTOKEN "YWJlZGpnYmFzZG9kag==", + :GITREPOURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vYXJjaGl2ZS9tYWluLnppcA==", + :GITCOMMITURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vZ2l0L2NvbW1pdHMvSEVBRA=="}} + (cut/generate-website-build-secret {:authtoken "abedjgbasdodj", + :gitea-host "gitlab.de", + :username "someuser", + :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"], + :gitea-repo "repo", + :sha256sum-output "123456789ab123cd345de script-file-name.sh", + :issuer "staging", + :branchname "main", + :unique-name "test.io"})))) (deftest should-generate-website-content-volume (is (= {:name-c1 "test-io-content-volume", @@ -255,3 +255,19 @@ :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"] :username "someuser" :authtoken "abedjgbasdodj"}))))) + +(deftest should-generate-hashfile-volume + (is (= {:apiVersion "v1", + :kind "PersistentVolumeClaim", + :metadata + {:name "test-io-hashfile-volume", + :namespace "default", + :labels {:app "test-io-nginx", :app.kubernetes.part-of "test-io-website"}}, + :spec {:storageClassName "local-path", :accessModes ["ReadWriteOnce"], :resources {:requests {:storage "16Mi"}}}} + (cut/generate-hashfile-volume {:unique-name "test.io", + :gitea-host "gitea.evilorg", + :gitea-repo "none", + :branchname "mablain", + :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"] + :username "someuser" + :authtoken "abedjgbasdodj"}))))