set version k3s and kubectl to 1.27.0 and add test

This commit is contained in:
ansgarz 2023-08-21 22:45:52 +02:00
parent 11b13feb86
commit 3b18318921
7 changed files with 268 additions and 46 deletions

View file

@ -55,7 +55,7 @@ fun Prov.installKubectlAndTools(): ProvResult = task {
fun Prov.installKubectl(): ProvResult = task {
// see https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/
val kubectlVersion = "1.23.0"
val kubectlVersion = "1.27.4"
val tmpDir = "~/tmp"
// prerequisites -- see https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/

View file

@ -403,7 +403,7 @@ fun Prov.fileSize(filename: String, sudo: Boolean = false): Int? {
private fun ensureValidPosixFilePermission(posixFilePermission: String) {
if (!Regex("^[0-7]{3}$").matches(posixFilePermission)) throw IllegalArgumentException("Wrong file permission ($posixFilePermission), permission must consist of 3 digits as e.g. 664")
if (!Regex("^0?[0-7]{3}$").matches(posixFilePermission)) throw IllegalArgumentException("Wrong file permission ($posixFilePermission), permission must consist of 3 digits as e.g. 664")
}
/**

View file

@ -10,7 +10,7 @@ import java.io.File
// ----------------------------------- versions --------------------------------
const val K3S_VERSION = "v1.23.6+k3s1"
const val K3S_VERSION = "v1.27.4+k3s1"
// ----------------------------------- directories --------------------------------
const val k3sManualManifestsDir = "/etc/rancher/k3s/manifests/"
@ -31,7 +31,8 @@ private val k3sMiddleWareHttpsRedirect = File(k3sManualManifestsDir, "middleware
private val certManagerDeployment = File(k3sManualManifestsDir, "cert-manager.yaml")
private val certManagerIssuer = File(k3sManualManifestsDir, "le-issuer.yaml")
private val k3sEcho = File(k3sManualManifestsDir, "echo.yaml")
private val k3sEchoWithTls = File(k3sManualManifestsDir, "echo-tls.yaml")
private val k3sEchoNoTls = File(k3sManualManifestsDir, "echo-no-tls.yaml")
private val selfSignedCertificate = File(k3sManualManifestsDir, "selfsigned-certificate.yaml")
private val localPathProvisionerConfig = File(k3sManualManifestsDir, "local-path-provisioner-config.yaml")
@ -95,7 +96,7 @@ fun Prov.installK3s(k3sConfig: K3sConfig): ProvResult {
// metallb
applyK3sFileFromResource(File(k3sManualManifestsDir, "metallb-0.13.7-native-manifest.yaml"))
repeatTaskUntilSuccess(6, 10) {
repeatTaskUntilSuccess(10, 10) {
applyK3sFileFromResourceTemplate(
File(k3sManualManifestsDir, "metallb-config.yaml"),
k3sConfigMap,
@ -144,17 +145,20 @@ fun Prov.provisionK3sCertManager(certmanager: Certmanager) = task {
}
}
fun Prov.provisionK3sEcho(fqdn: String, endpoint: CertmanagerEndpoint? = null) = task {
val endpointName = endpoint?.name?.lowercase()
fun Prov.provisionK3sEcho(fqdn: String, endpoint: CertmanagerEndpoint? = null, withTls: Boolean = false) = task {
if (withTls) {
val endpointName = endpoint?.name?.lowercase()
val issuer = if (endpointName == null) {
createK3sFileFromResourceTemplate(selfSignedCertificate, mapOf("host" to fqdn))
"selfsigned-issuer"
val issuer = if (endpointName == null) {
createK3sFileFromResourceTemplate(selfSignedCertificate, mapOf("host" to fqdn))
"selfsigned-issuer"
} else {
endpointName
}
applyK3sFileFromResourceTemplate(k3sEchoWithTls, mapOf("fqdn" to fqdn, "issuer_name" to issuer))
} else {
endpointName
applyK3sFileFromResource(k3sEchoNoTls)
}
applyK3sFileFromResourceTemplate(k3sEcho, mapOf("fqdn" to fqdn, "issuer_name" to issuer))
}
fun Prov.provisionK3sApplication(applicationFile: ApplicationFile) = task {

View file

@ -0,0 +1,41 @@
kind: Ingress
apiVersion: networking.k8s.io/v1
metadata:
name: echo-ingress
annotations:
kubernetes.io/ingress.class: "traefik"
spec:
rules:
- http:
paths:
- pathType: Exact
path: /echo/ # traefik echo pod needs the trailing slash, otherwise it'll return bad request
backend:
service:
name: echo-service
port:
number: 80
---
kind: Pod
apiVersion: v1
metadata:
name: echo-app
labels:
app: echo
spec:
containers:
- name: echo-app
image: traefik/whoami
---
kind: Service
apiVersion: v1
metadata:
name: echo-service
spec:
selector:
app: echo
ports:
- port: 80 # Default port for image

View file

@ -10,8 +10,8 @@ spec:
- host: ${fqdn}
http:
paths:
- pathType: Prefix
path: /echo
- pathType: Exact
path: /echo/ # traefik echo pod needs the trailing slash, otherwise it'll return bad request
backend:
service:
name: echo-service

View file

@ -1,4 +1,6 @@
#!/bin/sh
# File taken from https://github.com/k3s-io/k3s/blob/master/install.sh
set -e
set -o noglob
@ -18,7 +20,7 @@ set -o noglob
# Environment variables which begin with K3S_ will be preserved for the
# systemd service to use. Setting K3S_URL without explicitly setting
# a systemd exec command will default the command to "agent", and we
# enforce that K3S_TOKEN or K3S_CLUSTER_SECRET is also set.
# enforce that K3S_TOKEN is also set.
#
# - INSTALL_K3S_SKIP_DOWNLOAD
# If set to true will not download k3s hash or binary.
@ -92,7 +94,7 @@ set -o noglob
# Defaults to 'stable'.
GITHUB_URL=https://github.com/k3s-io/k3s/releases
STORAGE_URL=https://storage.googleapis.com/k3s-ci-builds
STORAGE_URL=https://k3s-ci-builds.s3.amazonaws.com
DOWNLOADER=
# --- helper functions for logs ---
@ -170,8 +172,8 @@ setup_env() {
if [ -z "${K3S_URL}" ]; then
CMD_K3S=server
else
if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ] && [ -z "${K3S_CLUSTER_SECRET}" ]; then
fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN, K3S_TOKEN_FILE or K3S_CLUSTER_SECRET is not defined."
if [ -z "${K3S_TOKEN}" ] && [ -z "${K3S_TOKEN_FILE}" ]; then
fatal "Defaulted k3s exec command to 'agent' because K3S_URL is defined, but K3S_TOKEN or K3S_TOKEN_FILE is not defined."
fi
CMD_K3S=agent
fi
@ -217,11 +219,7 @@ setup_env() {
if [ -n "${INSTALL_K3S_TYPE}" ]; then
SYSTEMD_TYPE=${INSTALL_K3S_TYPE}
else
if [ "${CMD_K3S}" = server ]; then
SYSTEMD_TYPE=notify
else
SYSTEMD_TYPE=exec
fi
SYSTEMD_TYPE=notify
fi
# --- use binary install directory if defined or create default ---
@ -273,12 +271,18 @@ setup_env() {
}
# --- check if skip download environment variable set ---
can_skip_download() {
if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ]; then
can_skip_download_binary() {
if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != binary ]; then
return 1
fi
}
can_skip_download_selinux() {
if [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != true ] && [ "${INSTALL_K3S_SKIP_DOWNLOAD}" != selinux ]; then
return 1
fi
}
# --- verify an executable k3s binary is installed ---
verify_k3s_is_executable() {
if [ ! -x ${BIN_DIR}/k3s ]; then
@ -304,6 +308,10 @@ setup_verify_arch() {
ARCH=arm64
SUFFIX=-${ARCH}
;;
s390x)
ARCH=s390x
SUFFIX=-${ARCH}
;;
aarch64)
ARCH=arm64
SUFFIX=-${ARCH}
@ -366,6 +374,45 @@ get_release_version() {
info "Using ${VERSION_K3S} as release"
}
# --- get k3s-selinux version ---
get_k3s_selinux_version() {
available_version="k3s-selinux-1.2-2.${rpm_target}.noarch.rpm"
info "Finding available k3s-selinux versions"
# run verify_downloader in case it binary installation was skipped
verify_downloader curl || verify_downloader wget || fatal 'Can not find curl or wget for downloading files'
case $DOWNLOADER in
curl)
DOWNLOADER_OPTS="-s"
;;
wget)
DOWNLOADER_OPTS="-q -O -"
;;
*)
fatal "Incorrect downloader executable '$DOWNLOADER'"
;;
esac
for i in {1..3}; do
set +e
if [ "${rpm_channel}" = "testing" ]; then
version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm" | head -n 1)
else
version=$(timeout 5 ${DOWNLOADER} ${DOWNLOADER_OPTS} https://api.github.com/repos/k3s-io/k3s-selinux/releases/latest | grep browser_download_url | awk '{ print $2 }' | grep -oE "[^\/]+${rpm_target}\.noarch\.rpm")
fi
set -e
if [ "${version}" != "" ]; then
break
fi
sleep 1
done
if [ "${version}" == "" ]; then
warn "Failed to get available versions of k3s-selinux..defaulting to ${available_version}"
return
fi
available_version=${version}
}
# --- download from github url ---
download() {
[ $# -eq 2 ] || fatal 'download needs exactly 2 arguments'
@ -460,18 +507,35 @@ setup_selinux() {
fi
[ -r /etc/os-release ] && . /etc/os-release
if [ "${ID_LIKE%%[ ]*}" = "suse" ]; then
if [ `expr "${ID_LIKE}" : ".*suse.*"` != 0 ]; then
rpm_target=sle
rpm_site_infix=microos
package_installer=zypper
if [ "${ID_LIKE:-}" = suse ] && [ "${VARIANT_ID:-}" = sle-micro ]; then
rpm_target=sle
rpm_site_infix=slemicro
package_installer=zypper
fi
elif [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then
rpm_target=coreos
rpm_site_infix=coreos
package_installer=rpm-ostree
elif [ "${VERSION_ID%%.*}" = "7" ]; then
rpm_target=el7
rpm_site_infix=centos/7
package_installer=yum
else
elif [ "${VERSION_ID%%.*}" = "8" ] || [ "${VERSION_ID%%.*}" -gt "36" ]; then
rpm_target=el8
rpm_site_infix=centos/8
package_installer=yum
else
rpm_target=el9
rpm_site_infix=centos/9
package_installer=yum
fi
if [ "${package_installer}" = "rpm-ostree" ] && [ -x /bin/yum ]; then
package_installer=yum
fi
if [ "${package_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then
@ -480,12 +544,13 @@ setup_selinux() {
policy_hint="please install:
${package_installer} install -y container-selinux
${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/k3s-selinux-0.4-1.${rpm_target}.noarch.rpm
${package_installer} install -y https://${rpm_site}/k3s/${rpm_channel}/common/${rpm_site_infix}/noarch/${available_version}
"
if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download || [ ! -d /usr/share/selinux ]; then
if [ "$INSTALL_K3S_SKIP_SELINUX_RPM" = true ] || can_skip_download_selinux || [ ! -d /usr/share/selinux ]; then
info "Skipping installation of SELinux RPM"
elif [ "${ID_LIKE:-}" != coreos ] && [ "${VARIANT_ID:-}" != coreos ]; then
else
get_k3s_selinux_version
install_selinux_rpm ${rpm_site} ${rpm_channel} ${rpm_target} ${rpm_site_infix}
fi
@ -499,7 +564,7 @@ setup_selinux() {
$policy_error "Failed to apply container_runtime_exec_t to ${BIN_DIR}/k3s, ${policy_hint}"
fi
elif [ ! -f /usr/share/selinux/packages/k3s.pp ]; then
if [ -x /usr/sbin/transactional-update ]; then
if [ -x /usr/sbin/transactional-update ] || [ "${ID_LIKE:-}" = coreos ] || [ "${VARIANT_ID:-}" = coreos ]; then
warn "Please reboot your machine to activate the changes and avoid data loss."
else
$policy_error "Failed to find the k3s-selinux policy, ${policy_hint}"
@ -508,7 +573,7 @@ setup_selinux() {
}
install_selinux_rpm() {
if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then
if [ -r /etc/redhat-release ] || [ -r /etc/centos-release ] || [ -r /etc/oracle-release ] || [ -r /etc/fedora-release ] || [ "${ID_LIKE%%[ ]*}" = "suse" ]; then
repodir=/etc/yum.repos.d
if [ -d /etc/zypp/repos.d ]; then
repodir=/etc/zypp/repos.d
@ -533,9 +598,17 @@ EOF
sle)
rpm_installer="zypper --gpg-auto-import-keys"
if [ "${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then
transactional_update_run="transactional-update --no-selfupdate -d run"
rpm_installer="transactional-update --no-selfupdate -d run ${rpm_installer}"
: "${INSTALL_K3S_SKIP_START:=true}"
fi
# create the /var/lib/rpm-state in SLE systems to fix the prein selinux macro
${transactional_update_run} mkdir -p /var/lib/rpm-state
;;
coreos)
rpm_installer="rpm-ostree --idempotent"
# rpm_install_extra_args="--apply-live"
: "${INSTALL_K3S_SKIP_START:=true}"
;;
*)
rpm_installer="yum"
@ -543,6 +616,15 @@ EOF
esac
if [ "${rpm_installer}" = "yum" ] && [ -x /usr/bin/dnf ]; then
rpm_installer=dnf
fi
if rpm -q --quiet k3s-selinux; then
# remove k3s-selinux module before upgrade to allow container-selinux to upgrade safely
if check_available_upgrades container-selinux ${3} && check_available_upgrades k3s-selinux ${3}; then
MODULE_PRIORITY=$($SUDO semodule --list=full | grep k3s | cut -f1 -d" ")
if [ -n "${MODULE_PRIORITY}" ]; then
$SUDO semodule -X $MODULE_PRIORITY -r k3s || true
fi
fi
fi
# shellcheck disable=SC2086
$SUDO ${rpm_installer} install -y "k3s-selinux"
@ -550,9 +632,28 @@ EOF
return
}
check_available_upgrades() {
set +e
case ${2} in
sle)
available_upgrades=$($SUDO zypper -q -t -s 11 se -s -u --type package $1 | tail -n 1 | grep -v "No matching" | awk '{print $3}')
;;
coreos)
# currently rpm-ostree does not support search functionality https://github.com/coreos/rpm-ostree/issues/1877
;;
*)
available_upgrades=$($SUDO yum -q --refresh list $1 --upgrades | tail -n 1 | awk '{print $2}')
;;
esac
set -e
if [ -n "${available_upgrades}" ]; then
return 0
fi
return 1
}
# --- download and verify k3s ---
download_and_verify() {
if can_skip_download; then
if can_skip_download_binary; then
info 'Skipping k3s download and verify'
verify_k3s_is_executable
return
@ -640,6 +741,27 @@ killtree() {
) 2>/dev/null
}
remove_interfaces() {
# Delete network interface(s) that match 'master cni0'
ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
iface=${iface%%@*}
[ -z "$iface" ] || ip link delete $iface
done
# Delete cni related interfaces
ip link delete cni0
ip link delete flannel.1
ip link delete flannel-v6.1
ip link delete kube-ipvs0
ip link delete flannel-wg
ip link delete flannel-wg-v6
# Restart tailscale
if [ -n "$(command -v tailscale)" ]; then
tailscale set --advertise-routes=
fi
}
getshims() {
ps -e -o pid= -o args= | sed -e 's/^ *//; s/\s\s*/\t/;' | grep -w 'k3s/data/[^/]*/bin/containerd-shim' | cut -f1
}
@ -663,17 +785,11 @@ do_unmount_and_remove '/run/netns/cni-'
# Remove CNI namespaces
ip netns show 2>/dev/null | grep cni- | xargs -r -t -n 1 ip netns delete
# Delete network interface(s) that match 'master cni0'
ip link show 2>/dev/null | grep 'master cni0' | while read ignore iface ignore; do
iface=${iface%%@*}
[ -z "$iface" ] || ip link delete $iface
done
ip link delete cni0
ip link delete flannel.1
ip link delete flannel-v6.1
remove_interfaces
rm -rf /var/lib/cni/
iptables-save | grep -v KUBE- | grep -v CNI- | iptables-restore
ip6tables-save | grep -v KUBE- | grep -v CNI- | ip6tables-restore
iptables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | iptables-restore
ip6tables-save | grep -v KUBE- | grep -v CNI- | grep -iv flannel | ip6tables-restore
EOF
$SUDO chmod 755 ${KILLALL_K3S_SH}
$SUDO chown root:root ${KILLALL_K3S_SH}
@ -729,6 +845,9 @@ rm -f ${KILLALL_K3S_SH}
if type yum >/dev/null 2>&1; then
yum remove -y k3s-selinux
rm -f /etc/yum.repos.d/rancher-k3s-common*.repo
elif type rpm-ostree >/dev/null 2>&1; then
rpm-ostree uninstall k3s-selinux
rm -f /etc/yum.repos.d/rancher-k3s-common*.repo
elif type zypper >/dev/null 2>&1; then
uninstall_cmd="zypper remove -y k3s-selinux"
if [ "\${TRANSACTIONAL_UPDATE=false}" != "true" ] && [ -x /usr/sbin/transactional-update ]; then
@ -827,8 +946,8 @@ respawn_delay=5
respawn_max=0
set -o allexport
if [ -f /etc/environment ]; then source /etc/environment; fi
if [ -f ${FILE_K3S_ENV} ]; then source ${FILE_K3S_ENV}; fi
if [ -f /etc/environment ]; then . /etc/environment; fi
if [ -f ${FILE_K3S_ENV} ]; then . ${FILE_K3S_ENV}; fi
set +o allexport
EOF
$SUDO chmod 0755 ${FILE_K3S_SERVICE}
@ -897,6 +1016,15 @@ service_enable_and_start() {
return
fi
if command -v iptables-save 1> /dev/null && command -v iptables-restore 1> /dev/null
then
$SUDO iptables-save | grep -v KUBE- | grep -iv flannel | $SUDO iptables-restore
fi
if command -v ip6tables-save 1> /dev/null && command -v ip6tables-restore 1> /dev/null
then
$SUDO ip6tables-save | grep -v KUBE- | grep -iv flannel | $SUDO ip6tables-restore
fi
[ "${HAS_SYSTEMD}" = true ] && systemd_start
[ "${HAS_OPENRC}" = true ] && openrc_start
return 0

View file

@ -0,0 +1,49 @@
package org.domaindrivenarchitecture.provs.server.infrastructure
import org.domaindrivenarchitecture.provs.framework.core.Secret
import org.domaindrivenarchitecture.provs.framework.core.local
import org.domaindrivenarchitecture.provs.framework.core.remote
import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.createDir
import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.createSecretFile
import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.fileContent
import org.domaindrivenarchitecture.provs.framework.ubuntu.secret.secretSources.PromptSecretSource
import org.domaindrivenarchitecture.provs.server.domain.k3s.K3sConfig
import org.domaindrivenarchitecture.provs.server.domain.k3s.Node
import org.domaindrivenarchitecture.provs.server.domain.k3s.provisionK3s
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import java.lang.Thread.sleep
class K3sKtTest {
@Test // Extensive test, takes several minutes
@Disabled("update remoteIp and user and run manually")
fun installK3s() {
// given
val remoteHostIp = "192.168.56.146"
val user = "xxx"
// enable ssh connection either manually or by the commented-out code below to copy local authorized_keys to remote
// remote(remoteHostIp, user, PromptSecretSource("PW for $user on $remoteHostIp").secret()).task {
// val authorizedKeysFilename = ".ssh/authorized_keys"
// val publicSshKey = local().getSecret("cat $authorizedKeysFilename") ?: Secret("") // or set directly by: val publicSshKey = Secret("public ssh key")
// createDir(".ssh")
// createSecretFile(authorizedKeysFilename, publicSshKey, posixFilePermission = "0644")
// }
// when
val res = remote(remoteHostIp, user).task { // connect by ssh
provisionK3s(K3sConfig(remoteHostIp, Node(remoteHostIp), echo = true, reprovision = false))
}
// then
assertTrue(res.success)
// check response echo pod
sleep(10000) // if time too short, increase or check curl manually
val echoResponse = local().cmd("curl http://$remoteHostIp/echo/").out
assertTrue(echoResponse?.contains("Hostname: echo-app") ?: false)
assertTrue(echoResponse?.contains("Host: $remoteHostIp") ?: false)
}
}