meissa/c4k-website
6
5
Fork 0
Code Issues Pull requests Projects Releases 8 Packages Wiki Activity

Merge branch 'hash-zip-before-unpack' into 'main'

Browse source 
Check and compare commit hashes before downloading files

See merge request domaindrivenarchitecture/c4k-website!4
This commit is contained in:
Pat Dyn 2022-12-09 11:24:22 +00:00
commit 63f4b05e02
13 changed files with 147 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
infrastructure/c4k-website-build/image/Dockerfile
View file

@ -3,8 +3,9 @@ FROM clojure:lein
# Prepare Entrypoint Script
ADD resources /tmp
ENV SOURCEDIR="/etc/websitesource"
ENV BUILDDIR="/etc/website"
ENV SOURCEDIR="/etc/websitesource"
ENV WEBSITEROOT="/var/www/html/website/"
ENV HASHFILEDIR="/var/hashfile.d"
RUN /tmp/install.sh

40
infrastructure/c4k-website-build/image/resources/entrypoint.sh
View file

@ -3,18 +3,34 @@
mkdir $BUILDDIR
mkdir $SOURCEDIR
set -o nounset
set -o xtrace
set -o errexit
set -eo pipefail
set -euo pipefail
source /usr/local/bin/functions.sh
echo "Downloading website"
get-and-unzip-website-data
echo "Executing Custom Scripts, if applicable"
execute-scripts-when-existing
echo "Building website"
build-and-extract-website
echo "Moving files"
move-website-files-to-target
filename="website.zip"
hashfilename="hashfile"
echo "Check for new content"
touch $HASHFILEDIR/$hashfilename
currentHash=$( cat $HASHFILEDIR/$hashfilename )
newHash=$( get-hash-data )
if [[ $currentHash == $newHash ]]
then
echo "Nothing to do"
else
echo $currentHash > $HASHFILEDIR/$hashfilename
echo "Downloading website data"
get-website-data $filename
unzip-website-data $filename
echo "Executing Custom Scripts, if applicable"
execute-scripts-when-existing
echo "Building website"
build-website
echo "Moving files"
move-website-files-to-target
fi

22
infrastructure/c4k-website-build/image/resources/functions.sh
View file

@ -1,12 +1,22 @@
#!/bin/bash
function get-and-unzip-website-data() {
filename="website.zip"
curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$filename $GITREPOURL
unzip $SOURCEDIR/$filename -d $BUILDDIR
function get-website-data() {
curl -H "Authorization: token $AUTHTOKEN" -o $SOURCEDIR/$1 $GITREPOURL
}
function execute-scripts-when-existing {
function get-hash-data() {
curl -s -H "Authorization: token $AUTHTOKEN" $GITCOMMITURL | jq '.sha'
}
function write-hash-data() {
echo $1 > $HASHFILEDIR/$2
}
function unzip-website-data() {
unzip $SOURCEDIR/$1 -d $BUILDDIR
}
function execute-scripts-when-existing() {
websitedir=$(ls $BUILDDIR)
if [[ -f $BUILDDIR/$websitedir/$SCRIPTFILE ]]
then
@ -26,7 +36,7 @@ function execute-scripts-when-existing {
fi
}
function build-and-extract-website() {
function build-website() {
(cd $BUILDDIR; dir=$(ls); cd $dir; lein run;)
}

2
infrastructure/c4k-website-build/image/resources/install.sh
View file

@ -2,7 +2,7 @@
apt update > /dev/null;
apt install -y unzip rsync
apt install -y unzip rsync jq imagemagick
mkdir /etc/lein/

2
infrastructure/c4k-website-build/test/Dockerfile
View file

@ -1,4 +1,4 @@
FROM c4k-jira-backup
FROM c4k-website-build
RUN apt update
RUN apt -yqq --no-install-recommends --yes install curl default-jre-headless

7
infrastructure/c4k-website-build/test/serverspec.edn
View file

@ -1,5 +1,4 @@
{:file [{:path "/usr/local/bin/install.sh" :mod "700"}
{:file [{:path "/entrypoint.sh" :mod "700"}
{:path "/usr/local/bin/functions.sh" :mod "700"}
{:path "/usr/local/bin/exclude.pattern" :mod "700"}
{:path "/usr/local/bin/project.clj" :mod "700"}
{:path "/entrypoint.sh" :mod "700"}]}
{:path "/etc/exclude.pattern" :mod "700"}
{:path "/etc/lein/project.clj" :mod "700"}]}

1
src/main/cljc/dda/c4k_website/core.cljc
View file

@ -45,6 +45,7 @@
(website/generate-nginx-configmap (flatten-and-reduce-config config))
(website/generate-nginx-service (flatten-and-reduce-config config))
(website/generate-website-content-volume (flatten-and-reduce-config config))
(website/generate-hashfile-volume (flatten-and-reduce-config config))
(website/generate-website-ingress (flatten-and-reduce-config config))
(website/generate-website-certificate (flatten-and-reduce-config config))
(website/generate-website-build-cron (flatten-and-reduce-config config))

28
src/main/cljc/dda/c4k_website/website.cljc
View file

@ -98,7 +98,14 @@
branch string?]
(str "https://" host "/api/v1/repos/" user "/" repo "/archive/" branch ".zip"))
(defn-spec replace-all-matching-subvalues-in-string-start pred/map-or-seq?
; https://your.gitea.host/api/v1/repos/<owner>/<repo>/git/commits/HEAD
(defn-spec generate-gitcommiturl string?
[host pred/fqdn-string?
repo string?
user string?]
(str "https://" host "/api/v1/repos/" user "/" repo "/git/" "commits/" "HEAD"))
(defn-spec replace-all-matching-substrings-beginning-with pred/map-or-seq?
[col pred/map-or-seq?
value-to-partly-match string?
value-to-inplace string?]
@ -114,7 +121,7 @@
(->
(yaml/load-as-edn resource-file)
(assoc-in [:metadata :labels :app.kubernetes.part-of] (generate-app-name unique-name))
(replace-all-matching-subvalues-in-string-start "NAME" (replace-dots-by-minus unique-name)))))
(replace-all-matching-substrings-beginning-with "NAME" (replace-dots-by-minus unique-name)))))
(defn-spec replace-build-data pred/map-or-seq?
[resource-file string?
@ -139,6 +146,7 @@
"website/website-build-cron.yaml" (rc/inline "website/website-build-cron.yaml")
"website/website-build-secret.yaml" (rc/inline "website/website-build-secret.yaml")
"website/website-content-volume.yaml" (rc/inline "website/website-content-volume.yaml")
"website/hashfile-volume.yaml" (rc/inline "website/hashfile-volume.yaml")
(throw (js/Error. "Undefined Resource!")))))
(defn-spec generate-website-ingress pred/map-or-seq?
@ -185,6 +193,10 @@
(replace-common-data "website/website-content-volume.yaml" config)
(cm/replace-all-matching-values-by-new-value "WEBSITESTORAGESIZE" (str volume-size "Gi")))))
(defn-spec generate-hashfile-volume pred/map-or-seq?
[config flattened-and-reduced-config?]
(replace-common-data "website/hashfile-volume.yaml" config))
(defn-spec generate-website-build-cron pred/map-or-seq?
[config flattened-and-reduced-config?]
(replace-build-data "website/website-build-cron.yaml" config))
@ -199,9 +211,15 @@
(->
(replace-common-data "website/website-build-secret.yaml" auth)
(cm/replace-all-matching-values-by-new-value "TOKEN" (b64/encode authtoken))
(cm/replace-all-matching-values-by-new-value "URL" (b64/encode
(generate-gitrepourl
(cm/replace-all-matching-values-by-new-value "REPOURL" (b64/encode
(generate-gitrepourl
gitea-host
gitea-repo
username
branchname))))))
branchname)))
(cm/replace-all-matching-values-by-new-value "COMMITURL" (b64/encode
(generate-gitcommiturl
gitea-host
gitea-repo
username))))))

16
src/main/resources/website/hashfile-volume.yaml Normal file
View file

@ -0,0 +1,16 @@
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: NAME-hashfile-volume
namespace: default
labels:
app: NAME-nginx
app.kubernetes.part-of: NAME-website
spec:
storageClassName: local-path
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 16Mi

5
src/main/resources/website/nginx-deployment.yaml
View file

@ -52,6 +52,8 @@ spec:
volumeMounts:
- name: content-volume
mountPath: /var/www/html/website
- name: hashfile-volume
mountPath: /var/hashfile.d
volumes:
- name: nginx-config-volume
configMap:
@ -68,4 +70,7 @@ spec:
- name: content-volume
persistentVolumeClaim:
claimName: NAME-content-volume
- name: hashfile-volume
persistentVolumeClaim:
claimName: NAME-hashfile-volume

5
src/main/resources/website/website-build-cron.yaml
View file

@ -35,9 +35,14 @@ spec:
volumeMounts:
- name: content-volume
mountPath: /var/www/html/website
- name: hashfile-volume
mountPath: /var/hashfile.d
volumes:
- name: content-volume
persistentVolumeClaim:
claimName: NAME-content-volume
- name: hashfile-volume
persistentVolumeClaim:
claimName: NAME-hashfile-volume
restartPolicy: OnFailure

5
src/main/resources/website/website-build-secret.yaml
View file

@ -6,6 +6,5 @@ metadata:
app.kubernetes.part-of: NAME-website
data:
AUTHTOKEN: TOKEN
GITREPOURL: URL
GITREPOURL: REPOURL
GITCOMMITURL: COMMITURL

72
src/test/cljc/dda/c4k_website/website_test.cljc
View file

@ -12,6 +12,7 @@
(st/instrument `cut/generate-nginx-deployment)
(st/instrument `cut/generate-nginx-service)
(st/instrument `cut/generate-website-content-volume)
(st/instrument `cut/generate-hashfile-volume)
(st/instrument `cut/generate-website-ingress)
(st/instrument `cut/generate-website-certificate)
(st/instrument `cut/generate-website-build-cron)
@ -101,11 +102,12 @@
[{:image "domaindrivenarchitecture/c4k-website-build",
:name "test-io-init-build-container",
:imagePullPolicy "IfNotPresent",
:resources {:requests {:cpu "1000m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}},
:resources {:requests {:cpu "500m", :memory "256Mi"}, :limits {:cpu "1700m", :memory "512Mi"}},
:command ["/entrypoint.sh"],
:envFrom [{:secretRef {:name "test-io-secret"}}],
:env [{:name "SHA256SUM", :value "123456789ab123cd345de"} {:name "SCRIPTFILE", :value "script-file-name.sh"}],
:volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}]}],
:volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}
{:name "hashfile-volume", :mountPath "/var/hashfile.d"}]}],
:volumes
[{:name "nginx-config-volume",
:configMap
@ -115,7 +117,8 @@
{:key "website.conf", :path "conf.d/website.conf"}
{:key "mime.types", :path "mime.types"}]}}
{:name "log", :emptyDir {}}
{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}]}}}}
{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}
{:name "hashfile-volume", :persistentVolumeClaim {:claimName "test-io-hashfile-volume"}}]}}}}
(cut/generate-nginx-deployment {:authtoken "abedjgbasdodj",
:gitea-host "gitlab.de",
:username "someuser",
@ -196,8 +199,10 @@
:command ["/entrypoint.sh"],
:envFrom [{:secretRef {:name "test-io-secret"}}],
:env [{:name "SHA256SUM", :value "123456789ab123cd345de"} {:name "SCRIPTFILE", :value "script-file-name.sh"}],
:volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}]}],
:volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}],
:volumeMounts [{:name "content-volume", :mountPath "/var/www/html/website"}
{:name "hashfile-volume", :mountPath "/var/hashfile.d"}]}],
:volumes [{:name "content-volume", :persistentVolumeClaim {:claimName "test-io-content-volume"}}
{:name "hashfile-volume", :persistentVolumeClaim {:claimName "test-io-hashfile-volume"}}],
:restartPolicy "OnFailure"}}}}}}
(cut/generate-website-build-cron {:authtoken "abedjgbasdodj",
:gitea-host "gitlab.de",
@ -209,30 +214,25 @@
:branchname "main",
:unique-name "test.io"}))))
(deftest should-generate-website-build-secret
(is (= {:name-c1 "test-io-secret",
:name-c2 "test-org-secret",
:AUTHTOKEN-c1 (b64/encode "token1"),
:AUTHTOKEN-c2 (b64/encode "token2"),
:GITREPOURL-c1 (b64/encode "https://gitlab.org/api/v1/repos/dumpty/websitebau/archive/testname.zip"),
:GITREPOURL-c2 (b64/encode "https://github.com/api/v1/repos/humpty/websitedachs/archive/testname.zip"),
:app.kubernetes.part-of-c1 "test-io-website",
:app.kubernetes.part-of-c2 "test-org-website"}
(th/map-diff (cut/generate-website-build-secret {:unique-name "test.io",
:authtoken "token1",
:gitea-host "gitlab.org",
:gitea-repo "websitebau",
:username "dumpty",
:branchname "testname",
:fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]}
)
(cut/generate-website-build-secret {:unique-name "test.org",
:authtoken "token2",
:gitea-host "github.com",
:gitea-repo "websitedachs",
:username "humpty",
:branchname "testname",
:fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]})))))
(is (= {:apiVersion "v1",
:kind "Secret",
:metadata {:name "test-io-secret", :labels {:app.kubernetes.part-of "test-io-website"}},
:data
{:AUTHTOKEN "YWJlZGpnYmFzZG9kag==",
:GITREPOURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vYXJjaGl2ZS9tYWluLnppcA==",
:GITCOMMITURL "aHR0cHM6Ly9naXRsYWIuZGUvYXBpL3YxL3JlcG9zL3NvbWV1c2VyL3JlcG8vZ2l0L2NvbW1pdHMvSEVBRA=="}}
(cut/generate-website-build-secret {:authtoken "abedjgbasdodj",
:gitea-host "gitlab.de",
:username "someuser",
:fqdns ["test.de" "test.org" "www.test.de" "www.test.org"],
:gitea-repo "repo",
:sha256sum-output "123456789ab123cd345de script-file-name.sh",
:issuer "staging",
:branchname "main",
:unique-name "test.io"}))))
(deftest should-generate-website-content-volume
(is (= {:name-c1 "test-io-content-volume",
@ -255,3 +255,19 @@
:fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]
:username "someuser"
:authtoken "abedjgbasdodj"})))))
(deftest should-generate-hashfile-volume
(is (= {:apiVersion "v1",
:kind "PersistentVolumeClaim",
:metadata
{:name "test-io-hashfile-volume",
:namespace "default",
:labels {:app "test-io-nginx", :app.kubernetes.part-of "test-io-website"}},
:spec {:storageClassName "local-path", :accessModes ["ReadWriteOnce"], :resources {:requests {:storage "16Mi"}}}}
(cut/generate-hashfile-volume {:unique-name "test.io",
:gitea-host "gitea.evilorg",
:gitea-repo "none",
:branchname "mablain",
:fqdns ["test.de" "www.test.de" "test-it.de" "www.test-it.de"]
:username "someuser"
:authtoken "abedjgbasdodj"}))))