add task for k3s running in docker containers

This commit is contained in:
az 2021-11-30 11:57:05 +01:00
parent b9245468ae
commit 8de2f25d00
9 changed files with 206 additions and 25 deletions

View file

@ -8,6 +8,21 @@ import org.domaindrivenarchitecture.provs.core.tags.Api
import java.io.File import java.io.File
import java.net.InetAddress import java.net.InetAddress
/**
* Repeats task until it returns success
*/
fun Prov.repeatTask(times: Int, sleepInSec: Int, func: Prov.() -> ProvResult) = requireLast {
require(times > 0)
var result = ProvResult(false, err = "Internal error") // Will only be returned if function is not executed at all, otherwise func's result is returned
for (i in 1..times) {
result = func()
if (result.success)
return@requireLast result
Thread.sleep(sleepInSec * 1000L)
}
return@requireLast result
}
/** /**
* Returns the name of the calling function but excluding some functions of the prov framework * Returns the name of the calling function but excluding some functions of the prov framework
* in order to return the "real" calling function. * in order to return the "real" calling function.
@ -63,10 +78,21 @@ fun String.escapeForShell(): String {
// see https://www.shellscript.sh/escape.html // see https://www.shellscript.sh/escape.html
return this.escapeBackslash().escapeBacktick().escapeDoubleQuote().escapeDollar() return this.escapeBackslash().escapeBacktick().escapeDoubleQuote().escapeDollar()
} }
/**
* Returns an echo command for the given String, which will be escaped for the bash
*/
internal fun echoCommandForText(text: String): String { internal fun echoCommandForText(text: String): String {
return "echo -n ${text.escapeAndEncloseByDoubleQuoteForShell()}" return "echo -n ${text.escapeAndEncloseByDoubleQuoteForShell()}"
} }
/**
* Returns an echo command for the given String, which will be escaped for the shell and ADDITIONALLY with newline, tabs, etc replaced by \n, \t, etc
*/
internal fun echoCommandForTextWithNewlinesReplaced(text: String): String {
return "echo -en ${text.escapeAndEncloseByDoubleQuoteForShell()}"
}
fun fileSeparator(): String = File.separator fun fileSeparator(): String = File.separator
fun fileSeparatorChar(): Char = File.separatorChar fun fileSeparatorChar(): Char = File.separatorChar

View file

@ -43,10 +43,11 @@ fun Prov.provideContainer(
imageName: String = "ubuntu", imageName: String = "ubuntu",
startMode: ContainerStartMode = ContainerStartMode.USE_RUNNING_ELSE_CREATE, startMode: ContainerStartMode = ContainerStartMode.USE_RUNNING_ELSE_CREATE,
sudo: Boolean = true, sudo: Boolean = true,
options: String = "" options: String = "",
command: String = ""
) : ProvResult { ) : ProvResult {
if (this is UbuntuProv) { if (this is UbuntuProv) {
return this.provideContainerPlatform(containerName, imageName, startMode, sudo, options) return this.provideContainerPlatform(containerName, imageName, startMode, sudo, options, command)
} else { } else {
throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass) throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass)
} }
@ -85,3 +86,14 @@ fun Prov.exitAndRmContainer(
throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass) throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass)
} }
} }
fun Prov.containerExec(containerName: String, cmd: String, sudo: Boolean = true): ProvResult {
if (this is UbuntuProv) {
return this.containerExecPlatform(containerName, cmd, sudo)
} else {
throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass)
}
}

View file

@ -3,8 +3,8 @@ package org.domaindrivenarchitecture.provs.core.docker.platforms
import org.domaindrivenarchitecture.provs.core.ProvResult import org.domaindrivenarchitecture.provs.core.ProvResult
import org.domaindrivenarchitecture.provs.core.docker.containerRuns import org.domaindrivenarchitecture.provs.core.docker.containerRuns
import org.domaindrivenarchitecture.provs.core.docker.dockerImageExists import org.domaindrivenarchitecture.provs.core.docker.dockerImageExists
import org.domaindrivenarchitecture.provs.core.docker.exitAndRmContainer
import org.domaindrivenarchitecture.provs.core.docker.dockerimages.DockerImage import org.domaindrivenarchitecture.provs.core.docker.dockerimages.DockerImage
import org.domaindrivenarchitecture.provs.core.docker.exitAndRmContainer
import org.domaindrivenarchitecture.provs.core.escapeSingleQuote import org.domaindrivenarchitecture.provs.core.escapeSingleQuote
import org.domaindrivenarchitecture.provs.core.fileSeparator import org.domaindrivenarchitecture.provs.core.fileSeparator
import org.domaindrivenarchitecture.provs.core.hostUserHome import org.domaindrivenarchitecture.provs.core.hostUserHome
@ -17,24 +17,28 @@ fun UbuntuProv.provideContainerPlatform(
imageName: String = "ubuntu", imageName: String = "ubuntu",
startMode: ContainerStartMode = ContainerStartMode.USE_RUNNING_ELSE_CREATE, startMode: ContainerStartMode = ContainerStartMode.USE_RUNNING_ELSE_CREATE,
sudo: Boolean = true, sudo: Boolean = true,
options: String = "" options: String = "",
command: String =""
): ProvResult = requireLast { ): ProvResult = requireLast {
val dockerCmd = dockerCommand(sudo) val dockerCmd = dockerCommand(sudo)
if (startMode == ContainerStartMode.CREATE_NEW_KILL_EXISTING) { if (startMode == ContainerStartMode.CREATE_NEW_KILL_EXISTING) {
exitAndRmContainer(containerName) exitAndRmContainer(containerName)
} }
val runCommand = dockerCmd + "run -dit $options --name=$containerName $imageName $command"
if ((startMode == ContainerStartMode.CREATE_NEW_KILL_EXISTING) || (startMode == ContainerStartMode.CREATE_NEW_FAIL_IF_EXISTING)) { if ((startMode == ContainerStartMode.CREATE_NEW_KILL_EXISTING) || (startMode == ContainerStartMode.CREATE_NEW_FAIL_IF_EXISTING)) {
if (!cmd(dockerCmd + "run -dit $options --name=$containerName $imageName").success) { if (!cmd(runCommand).success) {
throw RuntimeException("could not start docker") throw RuntimeException("could not start docker")
} }
} else if (startMode == ContainerStartMode.USE_RUNNING_ELSE_CREATE) { } else if (startMode == ContainerStartMode.USE_RUNNING_ELSE_CREATE) {
val runCheckResult = cmdNoEval(dockerCmd + "inspect -f '{{.State.Running}}' $containerName") val runCheckResult = cmdNoEval(dockerCmd + "inspect -f '{{.State.Running}}' $containerName")
// if either container not found or container found but not running // if either container not found or container found but not running => remove container and start again
if (!runCheckResult.success || "false\n" == runCheckResult.out) { if (!runCheckResult.success || "false\n" == runCheckResult.out) {
cmdNoEval(dockerCmd + "rm -f $containerName") cmdNoEval(dockerCmd + "rm -f $containerName")
cmd(dockerCmd + "run -dit $options --name=$containerName $imageName") cmd(runCommand)
} }
} }
ProvResult(containerRuns(containerName, sudo)) ProvResult(containerRuns(containerName, sudo))

View file

@ -38,17 +38,19 @@ open class ContainerUbuntuHostProcessor(
throw RuntimeException("Could not start docker image: " + r.toString(), r.exception) throw RuntimeException("Could not start docker image: " + r.toString(), r.exception)
} }
private val hostShell = "/bin/bash"
override fun x(vararg args: String): ProcessResult { override fun x(vararg args: String): ProcessResult {
return localExecution.x("sh", "-c", dockerCmd + "exec $containerName " + buildCommand(*args)) return localExecution.x(hostShell, "-c", dockerCmd + "exec $containerName " + buildCommand(*args))
} }
override fun xNoLog(vararg args: String): ProcessResult { override fun xNoLog(vararg args: String): ProcessResult {
return localExecution.xNoLog("sh", "-c", dockerCmd + "exec $containerName " + buildCommand(*args)) return localExecution.xNoLog(hostShell, "-c", dockerCmd + "exec $containerName " + buildCommand(*args))
} }
private fun exitAndRm() { private fun exitAndRm() {
localExecution.x(SHELL, "-c", dockerCmd + "stop $containerName") localExecution.x(hostShell, "-c", dockerCmd + "stop $containerName")
localExecution.x(SHELL, "-c", dockerCmd + "rm $containerName") localExecution.x(hostShell, "-c", dockerCmd + "rm $containerName")
} }
private fun quoteString(s: String): String { private fun quoteString(s: String): String {

View file

@ -46,8 +46,8 @@ class RemoteProcessor(ip: InetAddress, user: String, password: Secret? = null) :
try { try {
ssh.disconnect() ssh.disconnect()
} finally { } finally {
log.error("Got exception when initializing ssh: " + e.message) log.error("Got exception when initializing ssh (Username, password or ssh-key might be wrong): " + e.message)
throw RuntimeException("Error when initializing ssh", e) throw RuntimeException("Error when initializing ssh (Username, password or ssh-key might be wrong) ", e)
} }
} }
} }

View file

@ -0,0 +1,35 @@
package org.domaindrivenarchitecture.provs.extensions.server_software.k3s.domain
import org.domaindrivenarchitecture.provs.core.Prov
import org.domaindrivenarchitecture.provs.core.docker.provideContainer
import org.domaindrivenarchitecture.provs.core.echoCommandForTextWithNewlinesReplaced
import org.domaindrivenarchitecture.provs.core.repeatTask
/**
* Runs a k3s server and a k3s agent as containers.
*/
fun Prov.installK3sAsContainers(token: String = "12345678901234") = task {
cmd("docker volume create k3s-server")
provideContainer("k3s-server", "rancher/k3s", command = "server --cluster-init", options = "-d --privileged --tmpfs /run --tmpfs /var/run -e K3S_TOKEN=$token -e K3S_KUBECONFIG_OUTPUT=./kubeconfig.yaml -e K3S_KUBECONFIG_MODE=666 -v k3s-server:/var/lib/rancher/k3s:z -p 6443:6443 -p 80:80 -p 443:443")
provideContainer("k3s-agent", "rancher/k3s", options = "-d --privileged --tmpfs /run --tmpfs /var/run -e K3S_TOKEN=$token -e K3S_URL=https://server:6443")
// wait for config file
cmd("export timeout=60; while [ ! -f /var/lib/docker/volumes/k3s-server/_data/server/kubeconfig.yaml ]; do if [ \"${'$'}timeout\" == 0 ]; then echo \"ERROR: Timeout while waiting for file.\"; break; fi; sleep 1; ((timeout--)); done")
sh("""
mkdir -p ${'$'}HOME/.kube/
cp /var/lib/docker/volumes/k3s-server/_data/server/kubeconfig.yaml ${'$'}HOME/.kube/config
""".trimIndent())
}
/**
* Apply a config to kubernetes.
* Prerequisite: Kubectl has to be installed
*/
fun Prov.applyK8sConfig(configAsYaml: String, kubectlCommand: String = "kubectl") = task {
repeatTask(6, 10) {
cmd(echoCommandForTextWithNewlinesReplaced(configAsYaml) + " | $kubectlCommand apply -f -")
}
}

View file

@ -2,14 +2,22 @@ package org.domaindrivenarchitecture.provs.extensions.server_software.k3s.infras
import org.domaindrivenarchitecture.provs.core.Prov import org.domaindrivenarchitecture.provs.core.Prov
import org.domaindrivenarchitecture.provs.core.ProvResult import org.domaindrivenarchitecture.provs.core.ProvResult
import org.domaindrivenarchitecture.provs.core.repeatTask
fun Prov.checkAppleService(host: String = "127.0.0.1") = task { /**
val apple = cmd("curl -m 30 $host/apple").out * Checks if URL "$host/apple" is available and return text "apple"
if ("apple" == apple?.trim()) { */
addResultToEval(ProvResult(true)) fun Prov.checkAppleService(host: String = "127.0.0.1") = requireLast {
// repeat required as curl may return with "empty reply from server" or with "Recv failure: Connection reset by peer"
val res = repeatTask(12, 10) {
cmd("curl -m 30 $host/apple")
}.out?.trim()
return@requireLast if ("apple" == res) {
ProvResult(true, out = res)
} else { } else {
addResultToEval(ProvResult(false, err = "Apple service did not return \"apple\" but instead: " + apple)) ProvResult(false,err = "Url $host/apple did not return text \"apple\" but returned: $res")
} }
} }

View file

@ -0,0 +1,64 @@
package org.domaindrivenarchitecture.provs.extensions.server_software.k3s.domain
import org.domaindrivenarchitecture.provs.core.docker
import org.domaindrivenarchitecture.provs.core.docker.containerExec
import org.domaindrivenarchitecture.provs.core.docker.provideContainer
import org.domaindrivenarchitecture.provs.core.local
import org.domaindrivenarchitecture.provs.core.processors.ContainerStartMode
import org.domaindrivenarchitecture.provs.extensions.server_software.k3s.infrastructure.apple.appleConfig
import org.domaindrivenarchitecture.provs.extensions.server_software.k3s.infrastructure.apple.checkAppleService
import org.domaindrivenarchitecture.provs.test.tags.ContainerTest
import org.domaindrivenarchitecture.provs.test.tags.NonCi
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.Test
internal class K3dKtTest {
@Test
@ContainerTest
@NonCi
fun installK3sAsContainers() {
val containerName = "alpine-docker-dind"
local().task {
provideContainer(
containerName,
"yobasystems/alpine-docker:dind-amd64",
ContainerStartMode.CREATE_NEW_KILL_EXISTING, // for re-create a potentially existing container
sudo = false,
options = "--privileged"
)
// alpine does not have bash pre-installed - but bash is currently required for provs
containerExec(containerName, "sh -c \"apk add bash\"", sudo = false)
}
val result = docker(containerName, sudo = false).task {
// given
cmd("apk update")
cmd("apk add sudo curl")
task(
"Install kubectl"
) {
sh("""
curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
chmod +x ./kubectl
mv ./kubectl /usr/local/bin/kubectl
kubectl version --client
""".trimIndent())
}
// when
installK3sAsContainers()
applyK8sConfig(appleConfig())
cmd("kubectl wait --for=condition=ready --timeout=600s pod apple-app")
checkAppleService()
}
// then
assertTrue(result.success)
}
}

View file

@ -1,6 +1,7 @@
package org.domaindrivenarchitecture.provs.ubuntu.utils package org.domaindrivenarchitecture.provs.ubuntu.utils
import org.domaindrivenarchitecture.provs.core.Prov import org.domaindrivenarchitecture.provs.core.Prov
import org.domaindrivenarchitecture.provs.core.docker
import org.domaindrivenarchitecture.provs.core.echoCommandForText import org.domaindrivenarchitecture.provs.core.echoCommandForText
import org.domaindrivenarchitecture.provs.test.tags.ContainerTest import org.domaindrivenarchitecture.provs.test.tags.ContainerTest
import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertEquals
@ -8,12 +9,12 @@ import org.junit.jupiter.api.Test
internal class UtilsKtTest { internal class UtilsKtTest {
@ContainerTest
@Test
fun printToShell_escapes_String_successfully() {
// given // given
val a = Prov.defaultInstance() val a = Prov.defaultInstance()
@ContainerTest
@Test
fun printToShell_escapes_String_successfully() {
// when // when
val testString = "test if newline \n and apostrophe's ' \" and special chars $$%[]\\ äöüß \$variable and tabs \t are handled correctly" val testString = "test if newline \n and apostrophe's ' \" and special chars $$%[]\\ äöüß \$variable and tabs \t are handled correctly"
@ -23,12 +24,8 @@ internal class UtilsKtTest {
assertEquals(testString, res) assertEquals(testString, res)
} }
@ContainerTest
@Test @Test
fun printToShell_escapes_raw_String_successfully() { fun printToShell_escapes_raw_String_successfully() {
// given
val a = Prov.defaultInstance()
// when // when
val testMultiLineString = """ val testMultiLineString = """
test if newlines test if newlines
@ -41,4 +38,37 @@ internal class UtilsKtTest {
// then // then
assertEquals(testMultiLineString, resMl) assertEquals(testMultiLineString, resMl)
} }
@Test
fun printToShell_escapes_raw_String_from_function_successfully() {
// when
fun testMultiLineString() = """
test if newlines
\n
and apostrophe's ' " \" \' and special chars $ {} $\{something}!§$%[]\\ äöüß $\notakotlinvariable ${'$'}notakotlinvariable and tabs \t are handled correctly
"""
val resMl = a.cmd(echoCommandForText(testMultiLineString())).out
// then
assertEquals(testMultiLineString(), resMl)
}
@Test
fun echoCommandForText_in_ubuntu_container() {
// given
val prov = docker()
// when
val testMultiLineString = """
test if newlines
\n
and apostrophe's ' " \" \' and special chars $ {} $\{something}!§$%[]\\ äöüß $\notakotlinvariable ${'$'}notakotlinvariable and tabs \t are handled correctly
"""
val resMl = prov.cmd(echoCommandForText(testMultiLineString)).out
// then
assertEquals(testMultiLineString, resMl)
}
} }