Merge branch 'hash-zip-before-unpack' into 'main'

Check and compare commit hashes before downloading files

See merge request domaindrivenarchitecture/c4k-website!4

infrastructure/c4k-website-build/image/Dockerfile

@@ -3,8 +3,9 @@ FROM clojure:lein
 # Prepare Entrypoint Script
 ADD resources /tmp
 
-ENV SOURCEDIR="/etc/websitesource"
 ENV BUILDDIR="/etc/website"
+ENV SOURCEDIR="/etc/websitesource"
 ENV WEBSITEROOT="/var/www/html/website/"
+ENV HASHFILEDIR="/var/hashfile.d"
 
 RUN /tmp/install.sh

infrastructure/c4k-website-build/image/resources/entrypoint.sh

@@ -3,18 +3,34 @@
 mkdir $BUILDDIR
 mkdir $SOURCEDIR
 
-set -o nounset
+set -euo pipefail
-set -o xtrace
-set -o errexit
-set -eo pipefail
 
 source /usr/local/bin/functions.sh
 
-echo "Downloading website"
+filename="website.zip"
-get-and-unzip-website-data
+hashfilename="hashfile"
-echo "Executing Custom Scripts, if applicable"
+
-execute-scripts-when-existing
+echo "Check for new content"
-echo "Building website"
+touch $HASHFILEDIR/$hashfilename
-build-and-extract-website
+currentHash=$( cat $HASHFILEDIR/$hashfilename )
-echo "Moving files"
+newHash=$( get-hash-data )
-move-website-files-to-target    
+
+if [[ $currentHash == $newHash ]]
+    then
+        echo "Nothing to do"
+    else
+        echo $currentHash > $HASHFILEDIR/$hashfilename
+        echo "Downloading website data"
+        get-website-data $filename
+        unzip-website-data $filename
+        echo "Executing Custom Scripts, if applicable"
+        execute-scripts-when-existing
+        echo "Building website"
+        build-website
+        echo "Moving files"
+        move-website-files-to-target
+fi
+
+
+
+

infrastructure/c4k-website-build/image/resources/functions.sh

@@ -1,12 +1,22 @@
 #!/bin/bash
 
-function get-and-unzip-website-data() {
+function get-website-data() {    
-    filename="website.zip"
+    curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$1 $GITREPOURL
-    curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$filename $GITREPOURL
-    unzip $SOURCEDIR/$filename -d $BUILDDIR
 }
 
-function execute-scripts-when-existing {
+function get-hash-data() {
+    curl -s -H "Authorization: token $AUTHTOKEN" $GITCOMMITURL | jq '.sha'
+}
+
+function write-hash-data() {
+    echo $1 > $HASHFILEDIR/$2
+}
+
+function unzip-website-data() {
+    unzip $SOURCEDIR/$1 -d $BUILDDIR
+}
+
+function execute-scripts-when-existing() {
     websitedir=$(ls $BUILDDIR)
     if [[ -f $BUILDDIR/$websitedir/$SCRIPTFILE ]]
         then 
@@ -26,7 +36,7 @@ function execute-scripts-when-existing {
     fi
 }
 
-function build-and-extract-website() {
+function build-website() {
     (cd $BUILDDIR; dir=$(ls); cd $dir; lein run;)
 }
 

infrastructure/c4k-website-build/image/resources/install.sh

@@ -2,7 +2,7 @@
 
 apt update > /dev/null;
 
-apt install -y unzip rsync
+apt install -y unzip rsync jq imagemagick
 
 mkdir /etc/lein/
 

infrastructure/c4k-website-build/test/Dockerfile

@@ -1,4 +1,4 @@
-FROM c4k-jira-backup
+FROM c4k-website-build
 
 RUN apt update
 RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless

infrastructure/c4k-website-build/test/serverspec.edn

@@ -1,5 +1,4 @@
-{:file [{:path "/usr/local/bin/install.sh" :mod "700"}
+{:file [{:path "/entrypoint.sh" :mod "700"}
         {:path "/usr/local/bin/functions.sh" :mod "700"}
-        {:path "/usr/local/bin/exclude.pattern" :mod "700"}
+        {:path "/etc/exclude.pattern" :mod "700"}
-        {:path "/usr/local/bin/project.clj" :mod "700"}
+        {:path "/etc/lein/project.clj" :mod "700"}]}
-        {:path "/entrypoint.sh" :mod "700"}]}

src/main/cljc/dda/c4k_website/core.cljc

@@ -45,6 +45,7 @@
                    (website/generate-nginx-configmap (flatten-and-reduce-config config))
                    (website/generate-nginx-service (flatten-and-reduce-config config))
                    (website/generate-website-content-volume (flatten-and-reduce-config config))
+                   (website/generate-hashfile-volume (flatten-and-reduce-config config))
                    (website/generate-website-ingress (flatten-and-reduce-config config))
                    (website/generate-website-certificate (flatten-and-reduce-config config))
                    (website/generate-website-build-cron (flatten-and-reduce-config config))                   

src/main/cljc/dda/c4k_website/website.cljc

@@ -98,7 +98,14 @@
    branch string?]
   (str "https://" host "/api/v1/repos/" user "/" repo "/archive/" branch ".zip"))
 
-(defn-spec replace-all-matching-subvalues-in-string-start pred/map-or-seq?
+; https://your.gitea.host/api/v1/repos/<owner>/<repo>/git/commits/HEAD
+(defn-spec generate-gitcommiturl string?
+  [host pred/fqdn-string?
+   repo string?
+   user string?]
+  (str "https://" host "/api/v1/repos/" user "/" repo "/git/" "commits/" "HEAD"))
+
+(defn-spec replace-all-matching-substrings-beginning-with pred/map-or-seq?
   [col pred/map-or-seq?
    value-to-partly-match string?
    value-to-inplace string?]
@@ -114,7 +121,7 @@
     (->
      (yaml/load-as-edn resource-file)
      (assoc-in [:metadata :labels :app.kubernetes.part-of] (generate-app-name unique-name))
-     (replace-all-matching-subvalues-in-string-start "NAME" (replace-dots-by-minus unique-name)))))
+     (replace-all-matching-substrings-beginning-with "NAME" (replace-dots-by-minus unique-name)))))
 
 (defn-spec replace-build-data pred/map-or-seq?
   [resource-file string?
@@ -139,6 +146,7 @@
        "website/website-build-cron.yaml" (rc/inline "website/website-build-cron.yaml")
        "website/website-build-secret.yaml" (rc/inline "website/website-build-secret.yaml")
        "website/website-content-volume.yaml" (rc/inline "website/website-content-volume.yaml")
+       "website/hashfile-volume.yaml" (rc/inline "website/hashfile-volume.yaml")
        (throw (js/Error. "Undefined Resource!")))))
 
 (defn-spec generate-website-ingress pred/map-or-seq?
@@ -185,6 +193,10 @@
      (replace-common-data "website/website-content-volume.yaml" config)     
      (cm/replace-all-matching-values-by-new-value "WEBSITESTORAGESIZE" (str volume-size "Gi")))))
 
+(defn-spec generate-hashfile-volume pred/map-or-seq?
+  [config flattened-and-reduced-config?]
+  (replace-common-data "website/hashfile-volume.yaml" config))
+
 (defn-spec generate-website-build-cron pred/map-or-seq?
   [config flattened-and-reduced-config?]
   (replace-build-data "website/website-build-cron.yaml" config))
@@ -199,9 +211,15 @@
     (->
      (replace-common-data "website/website-build-secret.yaml" auth)     
      (cm/replace-all-matching-values-by-new-value "TOKEN" (b64/encode authtoken))
-     (cm/replace-all-matching-values-by-new-value "URL" (b64/encode
+     (cm/replace-all-matching-values-by-new-value "REPOURL" (b64/encode
-                                                         (generate-gitrepourl
+                                                         (generate-gitrepourl                                                          
                                                           gitea-host
                                                           gitea-repo
                                                           username
-                                                          branchname))))))
+                                                          branchname)))
+     (cm/replace-all-matching-values-by-new-value "COMMITURL" (b64/encode
+                                                         (generate-gitcommiturl
+                                                          gitea-host
+                                                          gitea-repo
+                                                          username))))))
+

src/main/resources/website/hashfile-volume.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: NAME-hashfile-volume
+  namespace: default
+  labels:
+    app: NAME-nginx    
+    app.kubernetes.part-of: NAME-website
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 16Mi
+      

src/main/resources/website/nginx-deployment.yaml

@@ -52,6 +52,8 @@ spec:
         volumeMounts:
         - name: content-volume
           mountPath: /var/www/html/website
+        - name: hashfile-volume
+          mountPath: /var/hashfile.d
       volumes:
         - name: nginx-config-volume
           configMap:
@@ -68,4 +70,7 @@ spec:
         - name: content-volume
           persistentVolumeClaim:
             claimName: NAME-content-volume
+        - name: hashfile-volume
+          persistentVolumeClaim:
+            claimName: NAME-hashfile-volume
                 

src/main/resources/website/website-build-cron.yaml

@@ -35,9 +35,14 @@ spec:
             volumeMounts:
             - name: content-volume
               mountPath: /var/www/html/website
+            - name: hashfile-volume
+              mountPath: /var/hashfile.d
           volumes:
           - name: content-volume
             persistentVolumeClaim:
               claimName: NAME-content-volume
+          - name: hashfile-volume
+            persistentVolumeClaim:
+              claimName: NAME-hashfile-volume
           restartPolicy: OnFailure
           

src/main/resources/website/website-build-secret.yaml

@@ -6,6 +6,5 @@ metadata:
     app.kubernetes.part-of: NAME-website
 data:
     AUTHTOKEN: TOKEN
-    GITREPOURL: URL
+    GITREPOURL: REPOURL
-    
+    GITCOMMITURL: COMMITURL
-    

src/test/cljc/dda/c4k_website/website_test.cljc

@@ -12,6 +12,7 @@
 (st/instrument `cut/generate-nginx-deployment)
 (st/instrument `cut/generate-nginx-service)
 (st/instrument `cut/generate-website-content-volume)
+(st/instrument `cut/generate-hashfile-volume)
 (st/instrument `cut/generate-website-ingress)
 (st/instrument `cut/generate-website-certificate)
 (st/instrument `cut/generate-website-build-cron)
@@ -101,11 +102,12 @@
              [{:image "domaindrivenarchitecture/c4k-website-build",
                :name "test-io-init-build-container",
                :imagePullPolicy "IfNotPresent",
-               :resources {:requests {:cpu "1000m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}},
+               :resources {:requests {:cpu "500m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}},
                :command ["/entrypoint.sh"],
                :envFrom [{:secretRef {:name "test-io-secret"}}],
                :env [{:name "SHA256SUM", :value "123456789ab123cd345de"} {:name "SCRIPTFILE", :value "script-file-name.sh"}],
-               :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}]}],
+               :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}
+                              {:name "hashfile-volume", :mountPath "/var/hashfile.d"}]}],
              :volumes
              [{:name "nginx-config-volume",
                :configMap
@@ -115,7 +117,8 @@
                  {:key "website.conf", :path "conf.d/website.conf"}
                  {:key "mime.types", :path "mime.types"}]}}
               {:name "log", :emptyDir {}}
-              {:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}]}}}}
+              {:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}
+              {:name "hashfile-volume", :persistentVolumeClaim {:claimName "test-io-hashfile-volume"}}]}}}}
          (cut/generate-nginx-deployment {:authtoken "abedjgbasdodj",
                                          :gitea-host "gitlab.de",
                                          :username "someuser",
@@ -196,8 +199,10 @@
                  :command ["/entrypoint.sh"],
                  :envFrom [{:secretRef {:name "test-io-secret"}}],
                  :env [{:name "SHA256SUM", :value "123456789ab123cd345de"} {:name "SCRIPTFILE", :value "script-file-name.sh"}],
-                 :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}]}],
+                 :volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}
-               :volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}],
+                                {:name "hashfile-volume", :mountPath "/var/hashfile.d"}]}],
+               :volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}
+                         {:name "hashfile-volume", :persistentVolumeClaim {:claimName "test-io-hashfile-volume"}}],
                :restartPolicy "OnFailure"}}}}}}
          (cut/generate-website-build-cron {:authtoken "abedjgbasdodj",
                                            :gitea-host "gitlab.de",
@@ -209,30 +214,25 @@
                                            :branchname "main",
                                            :unique-name "test.io"}))))
 
+
+
 (deftest should-generate-website-build-secret
-  (is (= {:name-c1 "test-io-secret",
+  (is (= {:apiVersion "v1",
-          :name-c2 "test-org-secret",
+          :kind "Secret",
-          :AUTHTOKEN-c1 (b64/encode "token1"),
+          :metadata {:name "test-io-secret", :labels {:app.kubernetes.part-of "test-io-website"}},
-          :AUTHTOKEN-c2 (b64/encode "token2"),
+          :data
-          :GITREPOURL-c1 (b64/encode "https://gitlab.org/api/v1/repos/dumpty/websitebau/archive/testname.zip"),
+          {:AUTHTOKEN "YWJlZGpnYmFzZG9kag==",
-          :GITREPOURL-c2 (b64/encode "https://github.com/api/v1/repos/humpty/websitedachs/archive/testname.zip"),
+           :GITREPOURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vYXJjaGl2ZS9tYWluLnppcA==",
-          :app.kubernetes.part-of-c1 "test-io-website", 
+           :GITCOMMITURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vZ2l0L2NvbW1pdHMvSEVBRA=="}}
-          :app.kubernetes.part-of-c2 "test-org-website"}
+         (cut/generate-website-build-secret {:authtoken "abedjgbasdodj",
-         (th/map-diff (cut/generate-website-build-secret {:unique-name "test.io",
+                                             :gitea-host "gitlab.de",
-                                                          :authtoken "token1",
+                                             :username "someuser",
-                                                          :gitea-host "gitlab.org",
+                                             :fqdns ["test.de" "test.org" "www.test.de" "www.test.org"],
-                                                          :gitea-repo "websitebau",
+                                             :gitea-repo "repo",
-                                                          :username "dumpty",
+                                             :sha256sum-output "123456789ab123cd345de script-file-name.sh",
-                                                          :branchname "testname",
+                                             :issuer "staging",
-                                                          :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}
+                                             :branchname "main",
-                                                         )
+                                             :unique-name "test.io"}))))
-                      (cut/generate-website-build-secret {:unique-name "test.org",
-                                                          :authtoken "token2",
-                                                          :gitea-host "github.com",
-                                                          :gitea-repo "websitedachs",
-                                                          :username "humpty",
-                                                          :branchname "testname",
-                                                          :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))))
 
 (deftest should-generate-website-content-volume
   (is (= {:name-c1 "test-io-content-volume",
@@ -255,3 +255,19 @@
                                                             :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]
                                                             :username "someuser"
                                                             :authtoken "abedjgbasdodj"})))))
+
+(deftest should-generate-hashfile-volume
+  (is (= {:apiVersion "v1",
+          :kind "PersistentVolumeClaim",
+          :metadata
+          {:name "test-io-hashfile-volume",
+           :namespace "default",
+           :labels {:app "test-io-nginx", :app.kubernetes.part-of "test-io-website"}},
+          :spec {:storageClassName "local-path", :accessModes ["ReadWriteOnce"], :resources {:requests {:storage "16Mi"}}}}
+         (cut/generate-hashfile-volume {:unique-name "test.io",
+                                        :gitea-host "gitea.evilorg",
+                                        :gitea-repo "none",
+                                        :branchname "mablain",
+                                        :fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]
+                                        :username "someuser"
+                                        :authtoken "abedjgbasdodj"}))))