From 02ce6336a2c8443357cd08b379f3c1738a863b9b Mon Sep 17 00:00:00 2001 From: ansgarz Date: Fri, 16 Feb 2024 10:33:43 +0100 Subject: [PATCH] update k3s-install.sh --- .../provs/server/infrastructure/K3s.kt | 1 + .../server/infrastructure/k3s/k3s-install.sh | 135 ++++++++++++++---- 2 files changed, 105 insertions(+), 31 deletions(-) diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt index 89f4bc4..61fd7bb 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/server/infrastructure/K3s.kt @@ -10,6 +10,7 @@ import java.io.File // ----------------------------------- versions -------------------------------- +// when updating it is recommended to update also file k3s-install.sh in this repo (under: src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/k3s-install.sh) const val K3S_VERSION = "v1.29.1+k3s2" // ----------------------------------- directories -------------------------------- diff --git a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/k3s-install.sh b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/k3s-install.sh index 64d224a..e1e5201 100644 --- a/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/k3s-install.sh +++ b/src/main/resources/org/domaindrivenarchitecture/provs/server/infrastructure/k3s/k3s-install.sh @@ -46,6 +46,10 @@ set -o noglob # Commit of k3s to download from temporary cloud storage. # * (for developer & QA use) # +# - INSTALL_K3S_PR +# PR build of k3s to download from Github Artifacts. +# * (for developer & QA use) +# # - INSTALL_K3S_BIN_DIR # Directory to install k3s binary, links, and uninstall script to, or use # /usr/local/bin as the default @@ -94,6 +98,7 @@ set -o noglob # Defaults to 'stable'. GITHUB_URL=https://github.com/k3s-io/k3s/releases +GITHUB_PR_URL="" STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com DOWNLOADER= @@ -277,11 +282,11 @@ can_skip_download_binary() { fi } -can_skip_download_selinux() { - if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then - return 1 - fi -} +can_skip_download_selinux() { + if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then + return 1 + fi +} # --- verify an executable k3s binary is installed --- verify_k3s_is_executable() { @@ -339,6 +344,7 @@ verify_downloader() { setup_tmp() { TMP_DIR=$(mktemp -d -t k3s-install.XXXXXXXXXX) TMP_HASH=${TMP_DIR}/k3s.hash + TMP_ZIP=${TMP_DIR}/k3s.zip TMP_BIN=${TMP_DIR}/k3s.bin cleanup() { code=$? @@ -352,7 +358,10 @@ setup_tmp() { # --- use desired k3s version if defined or find version from channel --- get_release_version() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then + if [ -n "${INSTALL_K3S_PR}" ]; then + VERSION_K3S="PR ${INSTALL_K3S_PR}" + get_pr_artifact_url + elif [ -n "${INSTALL_K3S_COMMIT}" ]; then VERSION_K3S="commit ${INSTALL_K3S_COMMIT}" elif [ -n "${INSTALL_K3S_VERSION}" ]; then VERSION_K3S=${INSTALL_K3S_VERSION} @@ -378,7 +387,7 @@ get_release_version() { get_k3s_selinux_version() { available_version="k3s-selinux-1.2-2.${rpm_target}.noarch.rpm" info "Finding available k3s-selinux versions" - + # run verify_downloader in case it binary installation was skipped verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files' @@ -416,7 +425,7 @@ get_k3s_selinux_version() { # --- download from github url --- download() { [ $# -eq 2 ] || fatal 'download needs exactly 2 arguments' - + set +e case $DOWNLOADER in curl) curl -o $1 -sfL $2 @@ -431,17 +440,25 @@ download() { # Abort if download command failed [ $? -eq 0 ] || fatal 'Download failed' + set -e } # --- download hash from github url --- download_hash() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then - HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum + if [ -n "${INSTALL_K3S_PR}" ]; then + info "Downloading hash ${GITHUB_PR_URL}" + curl -o ${TMP_ZIP} -H "Authorization: Bearer $GITHUB_TOKEN" -L ${GITHUB_PR_URL} + unzip -p ${TMP_ZIP} k3s.sha256sum > ${TMP_HASH} + sed -i 's/dist\/artifacts\/k3s/k3s/g' ${TMP_HASH} else - HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt + if [ -n "${INSTALL_K3S_COMMIT}" ]; then + HASH_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT}.sha256sum + else + HASH_URL=${GITHUB_URL}/download/${VERSION_K3S}/sha256sum-${ARCH}.txt + fi + info "Downloading hash ${HASH_URL}" + download ${TMP_HASH} ${HASH_URL} fi - info "Downloading hash ${HASH_URL}" - download ${TMP_HASH} ${HASH_URL} HASH_EXPECTED=$(grep " k3s${SUFFIX}$" ${TMP_HASH}) HASH_EXPECTED=${HASH_EXPECTED%%[[:blank:]]*} } @@ -458,9 +475,48 @@ installed_hash_matches() { return 1 } +# Use the GitHub API to identify the artifact associated with a given PR +get_pr_artifact_url() { + GITHUB_API_URL=https://api.github.com/repos/k3s-io/k3s + + # Check if jq is installed + if ! [ -x "$(command -v jq)" ]; then + echo "jq is required to use INSTALL_K3S_PR. Please install jq and try again" + exit 1 + fi + + if [ -z "${GITHUB_TOKEN}" ]; then + fatal "Installing PR builds requires GITHUB_TOKEN with k3s-io/k3s repo authorization" + fi + + # GET request to the GitHub API to retrieve the latest commit SHA from the pull request + COMMIT_ID=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/pulls/$INSTALL_K3S_PR" | jq -r '.head.sha') + + # GET request to the GitHub API to retrieve the Build workflow associated with the commit + wf_raw=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/commits/$COMMIT_ID/check-runs") + build_workflow=$(printf "%s" "$wf_raw" | jq -r '.check_runs[] | select(.name == "build / Build")') + + # Extract the Run ID from the build workflow and lookup artifacts associated with the run + RUN_ID=$(echo "$build_workflow" | jq -r ' .details_url' | awk -F'/' '{print $(NF-2)}') + + # Extract the artifat ID for the "k3s" artifact + artifacts=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "$GITHUB_API_URL/actions/runs/$RUN_ID/artifacts") + artifacts_url=$(echo "$artifacts" | jq -r '.artifacts[] | select(.name == "k3s") | .archive_download_url') + GITHUB_PR_URL=$artifacts_url +} + # --- download binary from github url --- download_binary() { - if [ -n "${INSTALL_K3S_COMMIT}" ]; then + if [ -n "${INSTALL_K3S_PR}" ]; then + # Since Binary and Hash are zipped together, check if TMP_ZIP already exists + if ! [ -f ${TMP_ZIP} ]; then + info "Downloading K3s artifact ${GITHUB_PR_URL}" + curl -o ${TMP_ZIP} -H "Authorization: Bearer $GITHUB_TOKEN" -L ${GITHUB_PR_URL} + fi + # extract k3s binary from zip + unzip -p ${TMP_ZIP} k3s > ${TMP_BIN} + return + elif [ -n "${INSTALL_K3S_COMMIT}" ]; then BIN_URL=${STORAGE_URL}/k3s${SUFFIX}-${INSTALL_K3S_COMMIT} else BIN_URL=${GITHUB_URL}/download/${VERSION_K3S}/k3s${SUFFIX} @@ -489,7 +545,7 @@ setup_binary() { # --- setup selinux policy --- setup_selinux() { - case ${INSTALL_K3S_CHANNEL} in + case ${INSTALL_K3S_CHANNEL} in *testing) rpm_channel=testing ;; @@ -511,7 +567,7 @@ setup_selinux() { rpm_target=sle rpm_site_infix=microos package_installer=zypper - if [ "${ID_LIKE:-}" = suse ] && [ "${VARIANT_ID:-}" = sle-micro ]; then + if [ "${ID_LIKE:-}" = suse ] && ( [ "${VARIANT_ID:-}" = sle-micro ] || [ "${ID:-}" = sle-micro ] ); then rpm_target=sle rpm_site_infix=slemicro package_installer=zypper @@ -549,11 +605,12 @@ setup_selinux() { if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then info "Skipping installation of SELinux RPM" - else - get_k3s_selinux_version - install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} + return fi + get_k3s_selinux_version + install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix} + policy_error=fatal if [ "$INSTALL_K3S_SELINUX_WARN" = true ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then policy_error=warn @@ -617,7 +674,7 @@ EOF if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then rpm_installer=dnf fi - if rpm -q --quiet k3s-selinux; then + if rpm -q --quiet k3s-selinux; then # remove k3s-selinux module before upgrade to allow container-selinux to upgrade safely if check_available_upgrades container-selinux ${3} && check_available_upgrades k3s-selinux ${3}; then MODULE_PRIORITY=$($SUDO semodule --list=full | grep k3s | cut -f1 -d" ") @@ -772,7 +829,7 @@ do_unmount_and_remove() { set +x while read -r _ path _; do case "$path" in $1*) echo "$path" ;; esac - done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount "$0" && rm -rf "$0"' + done < /proc/self/mounts | sort -r | xargs -r -t -n 1 sh -c 'umount -f "$0" && rm -rf "$0"' set -x } @@ -906,7 +963,7 @@ TasksMax=infinity TimeoutStartSec=0 Restart=always RestartSec=5s -ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service' +ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service 2>/dev/null' ExecStartPre=-/sbin/modprobe br_netfilter ExecStartPre=-/sbin/modprobe overlay ExecStart=${BIN_DIR}/k3s \\ @@ -963,11 +1020,16 @@ EOF # --- write systemd or openrc service file --- create_service_file() { - [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file + [ "${HAS_SYSTEMD}" = true ] && create_systemd_service_file && restore_systemd_service_file_context [ "${HAS_OPENRC}" = true ] && create_openrc_service_file return 0 } +restore_systemd_service_file_context() { + $SUDO restorecon -R -i ${FILE_K3S_SERVICE} 2>/dev/null || true + $SUDO restorecon -R -i ${FILE_K3S_ENV} 2>/dev/null || true +} + # --- get hashes of the current k3s bin and service files get_installed_hashes() { $SUDO sha256sum ${BIN_DIR}/k3s ${FILE_K3S_SERVICE} ${FILE_K3S_ENV} 2>&1 || true @@ -996,6 +1058,19 @@ openrc_start() { $SUDO ${FILE_K3S_SERVICE} restart } +has_working_xtables() { + if $SUDO sh -c "command -v \"$1-save\"" 1> /dev/null && $SUDO sh -c "command -v \"$1-restore\"" 1> /dev/null; then + if $SUDO $1-save 2>/dev/null | grep -q '^-A CNI-HOSTPORT-MASQ -j MASQUERADE$'; then + warn "Host $1-save/$1-restore tools are incompatible with existing rules" + else + return 0 + fi + else + info "Host $1-save/$1-restore tools not found" + fi + return 1 +} + # --- startup systemd or openrc service --- service_enable_and_start() { if [ -f "/proc/cgroups" ] && [ "$(grep memory /proc/cgroups | while read -r n n n enabled; do echo $enabled; done)" -eq 0 ]; @@ -1016,14 +1091,11 @@ service_enable_and_start() { return fi - if command -v iptables-save 1> /dev/null && command -v iptables-restore 1> /dev/null - then - $SUDO iptables-save | grep -v KUBE- | grep -iv flannel | $SUDO iptables-restore - fi - if command -v ip6tables-save 1> /dev/null && command -v ip6tables-restore 1> /dev/null - then - $SUDO ip6tables-save | grep -v KUBE- | grep -iv flannel | $SUDO ip6tables-restore - fi + for XTABLES in iptables ip6tables; do + if has_working_xtables ${XTABLES}; then + $SUDO ${XTABLES}-save 2>/dev/null | grep -v KUBE- | grep -iv flannel | $SUDO ${XTABLES}-restore + fi + done [ "${HAS_SYSTEMD}" = true ] && systemd_start [ "${HAS_OPENRC}" = true ] && openrc_start @@ -1047,3 +1119,4 @@ eval set -- $(escape "${INSTALL_K3S_EXEC}") $(quote "$@") create_service_file service_enable_and_start } +