add task for k3s running in docker containers

src/main/kotlin/org/domaindrivenarchitecture/provs/core/Utils.kt

@@ -8,6 +8,21 @@ import org.domaindrivenarchitecture.provs.core.tags.Api
 import java.io.File
 import java.net.InetAddress
 
+/**
+ * Repeats task until it returns success
+ */
+fun Prov.repeatTask(times: Int, sleepInSec: Int, func: Prov.() -> ProvResult) = requireLast {
+    require(times > 0)
+    var result = ProvResult(false, err = "Internal error")  // Will only be returned if function is not executed at all, otherwise func's result is returned
+    for (i in 1..times) {
+        result = func()
+        if (result.success)
+            return@requireLast result
+        Thread.sleep(sleepInSec * 1000L)
+    }
+    return@requireLast result
+}
+
 /**
  * Returns the name of the calling function but excluding some functions of the prov framework
  * in order to return the "real" calling function.
@@ -63,10 +78,21 @@ fun String.escapeForShell(): String {
     // see https://www.shellscript.sh/escape.html
     return this.escapeBackslash().escapeBacktick().escapeDoubleQuote().escapeDollar()
 }
+
+/**
+ * Returns an echo command for the given String, which will be escaped for the bash
+ */
 internal fun echoCommandForText(text: String): String {
     return "echo -n ${text.escapeAndEncloseByDoubleQuoteForShell()}"
 }
 
+/**
+ * Returns an echo command for the given String, which will be escaped for the shell and ADDITIONALLY with newline, tabs, etc replaced by \n, \t, etc
+ */
+internal fun echoCommandForTextWithNewlinesReplaced(text: String): String {
+    return "echo -en ${text.escapeAndEncloseByDoubleQuoteForShell()}"
+}
+
 
 fun fileSeparator(): String = File.separator
 fun fileSeparatorChar(): Char = File.separatorChar

src/main/kotlin/org/domaindrivenarchitecture/provs/core/docker/HostDocker.kt

@@ -43,10 +43,11 @@ fun Prov.provideContainer(
     imageName: String = "ubuntu",
     startMode: ContainerStartMode = ContainerStartMode.USE_RUNNING_ELSE_CREATE,
     sudo: Boolean = true,
-    options: String = ""
+    options: String = "",
+    command: String = ""
 ) : ProvResult {
     if (this is UbuntuProv) {
-        return this.provideContainerPlatform(containerName, imageName, startMode, sudo, options)
+        return this.provideContainerPlatform(containerName, imageName, startMode, sudo, options, command)
     } else {
         throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass)
     }
@@ -85,3 +86,14 @@ fun Prov.exitAndRmContainer(
         throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass)
     }
 }
+
+
+fun Prov.containerExec(containerName: String, cmd: String, sudo: Boolean = true): ProvResult {
+    if (this is UbuntuProv) {
+        return this.containerExecPlatform(containerName, cmd, sudo)
+    } else {
+        throw RuntimeException(DOCKER_NOT_SUPPORTED + (this as UbuntuProv).javaClass)
+    }
+}
+
+

src/main/kotlin/org/domaindrivenarchitecture/provs/core/docker/platforms/UbuntuHostDocker.kt

@@ -3,8 +3,8 @@ package org.domaindrivenarchitecture.provs.core.docker.platforms
 import org.domaindrivenarchitecture.provs.core.ProvResult
 import org.domaindrivenarchitecture.provs.core.docker.containerRuns
 import org.domaindrivenarchitecture.provs.core.docker.dockerImageExists
-import org.domaindrivenarchitecture.provs.core.docker.exitAndRmContainer
 import org.domaindrivenarchitecture.provs.core.docker.dockerimages.DockerImage
+import org.domaindrivenarchitecture.provs.core.docker.exitAndRmContainer
 import org.domaindrivenarchitecture.provs.core.escapeSingleQuote
 import org.domaindrivenarchitecture.provs.core.fileSeparator
 import org.domaindrivenarchitecture.provs.core.hostUserHome
@@ -17,24 +17,28 @@ fun UbuntuProv.provideContainerPlatform(
     imageName: String = "ubuntu",
     startMode: ContainerStartMode = ContainerStartMode.USE_RUNNING_ELSE_CREATE,
     sudo: Boolean = true,
-    options: String = ""
+    options: String = "",
+    command: String =""
 ): ProvResult = requireLast {
     val dockerCmd = dockerCommand(sudo)
 
     if (startMode == ContainerStartMode.CREATE_NEW_KILL_EXISTING) {
         exitAndRmContainer(containerName)
     }
+
+    val runCommand = dockerCmd + "run -dit $options --name=$containerName $imageName $command"
+
     if ((startMode == ContainerStartMode.CREATE_NEW_KILL_EXISTING) || (startMode == ContainerStartMode.CREATE_NEW_FAIL_IF_EXISTING)) {
-        if (!cmd(dockerCmd + "run -dit $options --name=$containerName $imageName").success) {
+        if (!cmd(runCommand).success) {
             throw RuntimeException("could not start docker")
         }
     } else if (startMode == ContainerStartMode.USE_RUNNING_ELSE_CREATE) {
         val runCheckResult = cmdNoEval(dockerCmd + "inspect -f '{{.State.Running}}' $containerName")
 
-        // if either container not found or container found but not running
+        // if either container not found or container found but not running => remove container and start again
         if (!runCheckResult.success || "false\n" == runCheckResult.out) {
             cmdNoEval(dockerCmd + "rm -f $containerName")
-            cmd(dockerCmd + "run -dit $options --name=$containerName $imageName")
+            cmd(runCommand)
         }
     }
     ProvResult(containerRuns(containerName, sudo))

src/main/kotlin/org/domaindrivenarchitecture/provs/core/processors/ContainerUbuntuHostProcessor.kt

@@ -38,17 +38,19 @@ open class ContainerUbuntuHostProcessor(
             throw RuntimeException("Could not start docker image: " + r.toString(), r.exception)
     }
 
+    private val hostShell = "/bin/bash"
+
     override fun x(vararg args: String): ProcessResult {
-        return localExecution.x("sh", "-c", dockerCmd + "exec $containerName " + buildCommand(*args))
+        return localExecution.x(hostShell, "-c", dockerCmd + "exec $containerName " + buildCommand(*args))
     }
 
     override fun xNoLog(vararg args: String): ProcessResult {
-        return localExecution.xNoLog("sh", "-c", dockerCmd + "exec $containerName " + buildCommand(*args))
+        return localExecution.xNoLog(hostShell, "-c", dockerCmd + "exec $containerName " + buildCommand(*args))
     }
 
     private fun exitAndRm() {
-        localExecution.x(SHELL, "-c", dockerCmd + "stop $containerName")
+        localExecution.x(hostShell, "-c", dockerCmd + "stop $containerName")
-        localExecution.x(SHELL, "-c", dockerCmd + "rm $containerName")
+        localExecution.x(hostShell, "-c", dockerCmd + "rm $containerName")
     }
 
     private fun quoteString(s: String): String {

src/main/kotlin/org/domaindrivenarchitecture/provs/core/processors/RemoteUbuntuProcessor.kt

@@ -46,8 +46,8 @@ class RemoteProcessor(ip: InetAddress, user: String, password: Secret? = null) :
             try {
                 ssh.disconnect()
             } finally {
-                log.error("Got exception when initializing ssh: " + e.message)
+                log.error("Got exception when initializing ssh (Username, password or ssh-key might be wrong): " + e.message)
-                throw RuntimeException("Error when initializing ssh", e)
+                throw RuntimeException("Error when initializing ssh (Username, password or ssh-key might be wrong) ", e)
             }
         }
     }

src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/server_software/k3s/domain/K3d.kt

@@ -0,0 +1,35 @@
+package org.domaindrivenarchitecture.provs.extensions.server_software.k3s.domain
+
+import org.domaindrivenarchitecture.provs.core.Prov
+import org.domaindrivenarchitecture.provs.core.docker.provideContainer
+import org.domaindrivenarchitecture.provs.core.echoCommandForTextWithNewlinesReplaced
+import org.domaindrivenarchitecture.provs.core.repeatTask
+
+
+/**
+ * Runs a k3s server and a k3s agent as containers.
+ */
+fun Prov.installK3sAsContainers(token: String = "12345678901234") = task {
+    cmd("docker volume create k3s-server")
+    provideContainer("k3s-server", "rancher/k3s", command = "server --cluster-init", options = "-d --privileged --tmpfs /run --tmpfs /var/run -e K3S_TOKEN=$token -e K3S_KUBECONFIG_OUTPUT=./kubeconfig.yaml -e K3S_KUBECONFIG_MODE=666 -v k3s-server:/var/lib/rancher/k3s:z -p 6443:6443 -p 80:80 -p 443:443")
+    provideContainer("k3s-agent", "rancher/k3s", options = "-d --privileged --tmpfs /run  --tmpfs /var/run -e K3S_TOKEN=$token -e K3S_URL=https://server:6443")
+
+    // wait for config file
+    cmd("export timeout=60; while [ ! -f /var/lib/docker/volumes/k3s-server/_data/server/kubeconfig.yaml ]; do if [ \"${'$'}timeout\" == 0 ]; then echo \"ERROR: Timeout while waiting for file.\"; break; fi; sleep 1; ((timeout--)); done")
+
+    sh("""
+        mkdir -p ${'$'}HOME/.kube/
+        cp /var/lib/docker/volumes/k3s-server/_data/server/kubeconfig.yaml ${'$'}HOME/.kube/config        
+    """.trimIndent())
+}
+
+
+/**
+ * Apply a config to kubernetes.
+ * Prerequisite: Kubectl has to be installed
+ */
+fun Prov.applyK8sConfig(configAsYaml: String, kubectlCommand: String = "kubectl") = task {
+    repeatTask(6, 10) {
+        cmd(echoCommandForTextWithNewlinesReplaced(configAsYaml) + " | $kubectlCommand apply -f -")
+    }
+}

src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/server_software/k3s/infrastructure/apple/Apple.kt

@@ -2,14 +2,22 @@ package org.domaindrivenarchitecture.provs.extensions.server_software.k3s.infras
 
 import org.domaindrivenarchitecture.provs.core.Prov
 import org.domaindrivenarchitecture.provs.core.ProvResult
+import org.domaindrivenarchitecture.provs.core.repeatTask
 
 
-fun Prov.checkAppleService(host: String = "127.0.0.1") = task {
+/**
-    val apple = cmd("curl -m 30 $host/apple").out
+ * Checks if URL "$host/apple" is available and return text "apple"
-    if ("apple" == apple?.trim()) {
+ */
-        addResultToEval(ProvResult(true))
+fun Prov.checkAppleService(host: String = "127.0.0.1") = requireLast {
+    // repeat required as curl may return with "empty reply from server" or with "Recv failure: Connection reset by peer"
+    val res = repeatTask(12, 10) {
+        cmd("curl -m 30 $host/apple")
+    }.out?.trim()
+
+    return@requireLast if ("apple" == res) {
+        ProvResult(true, out = res)
     } else {
-        addResultToEval(ProvResult(false, err = "Apple service did not return \"apple\" but instead: " + apple))
+        ProvResult(false,err = "Url $host/apple did not return text \"apple\" but returned: $res")
     }
 }
 

src/test/kotlin/org/domaindrivenarchitecture/provs/extensions/server_software/k3s/domain/K3dKtTest.kt

@@ -0,0 +1,64 @@
+package org.domaindrivenarchitecture.provs.extensions.server_software.k3s.domain
+
+import org.domaindrivenarchitecture.provs.core.docker
+import org.domaindrivenarchitecture.provs.core.docker.containerExec
+import org.domaindrivenarchitecture.provs.core.docker.provideContainer
+import org.domaindrivenarchitecture.provs.core.local
+import org.domaindrivenarchitecture.provs.core.processors.ContainerStartMode
+import org.domaindrivenarchitecture.provs.extensions.server_software.k3s.infrastructure.apple.appleConfig
+import org.domaindrivenarchitecture.provs.extensions.server_software.k3s.infrastructure.apple.checkAppleService
+import org.domaindrivenarchitecture.provs.test.tags.ContainerTest
+import org.domaindrivenarchitecture.provs.test.tags.NonCi
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.Test
+
+internal class K3dKtTest {
+
+    @Test
+    @ContainerTest
+    @NonCi
+    fun installK3sAsContainers() {
+
+        val containerName = "alpine-docker-dind"
+        local().task {
+            provideContainer(
+                containerName,
+                "yobasystems/alpine-docker:dind-amd64",
+                ContainerStartMode.CREATE_NEW_KILL_EXISTING,  // for re-create a potentially existing container
+                sudo = false,
+                options = "--privileged"
+            )
+
+            // alpine does not have bash pre-installed - but bash is currently required for provs
+            containerExec(containerName, "sh -c \"apk add bash\"", sudo = false)
+        }
+
+        val result = docker(containerName, sudo = false).task {
+
+            // given
+            cmd("apk update")
+            cmd("apk add sudo curl")
+            task(
+                "Install kubectl"
+            ) {
+                sh("""
+                    curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
+                    chmod +x ./kubectl
+                    mv ./kubectl /usr/local/bin/kubectl
+                    kubectl version --client
+                """.trimIndent())
+            }
+
+            // when
+            installK3sAsContainers()
+
+            applyK8sConfig(appleConfig())
+
+            cmd("kubectl wait --for=condition=ready --timeout=600s pod apple-app")
+            checkAppleService()
+        }
+
+        // then
+        assertTrue(result.success)
+    }
+}

src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/utils/UtilsKtTest.kt

@@ -1,6 +1,7 @@
 package org.domaindrivenarchitecture.provs.ubuntu.utils
 
 import org.domaindrivenarchitecture.provs.core.Prov
+import org.domaindrivenarchitecture.provs.core.docker
 import org.domaindrivenarchitecture.provs.core.echoCommandForText
 import org.domaindrivenarchitecture.provs.test.tags.ContainerTest
 import org.junit.jupiter.api.Assertions.assertEquals
@@ -8,12 +9,12 @@ import org.junit.jupiter.api.Test
 
 internal class UtilsKtTest {
 
+    // given
+    val a = Prov.defaultInstance()
+
     @ContainerTest
     @Test
     fun printToShell_escapes_String_successfully() {
-        // given
-        val a = Prov.defaultInstance()
-
         // when
         val testString = "test if newline \n and apostrophe's ' \" and special chars $ !§$%[]\\ äöüß \$variable and tabs     \t are handled correctly"
 
@@ -23,12 +24,8 @@ internal class UtilsKtTest {
         assertEquals(testString, res)
     }
 
-    @ContainerTest
     @Test
     fun printToShell_escapes_raw_String_successfully() {
-        // given
-        val a = Prov.defaultInstance()
-
         // when
         val testMultiLineString = """
             test if newlines
@@ -41,4 +38,37 @@ internal class UtilsKtTest {
         // then
         assertEquals(testMultiLineString, resMl)
     }
+
+    @Test
+    fun printToShell_escapes_raw_String_from_function_successfully() {
+        // when
+        fun testMultiLineString() = """
+            test if newlines
+            \n
+            and apostrophe's ' " \" \' and special chars $ {} $\{something}!§$%[]\\ äöüß $\notakotlinvariable ${'$'}notakotlinvariable and tabs     \t are handled correctly
+            """
+
+        val resMl = a.cmd(echoCommandForText(testMultiLineString())).out
+
+        // then
+        assertEquals(testMultiLineString(), resMl)
+    }
+
+    @Test
+    fun echoCommandForText_in_ubuntu_container() {
+        // given
+        val prov = docker()
+
+        // when
+        val testMultiLineString = """
+            test if newlines
+            \n
+            and apostrophe's ' " \" \' and special chars $ {} $\{something}!§$%[]\\ äöüß $\notakotlinvariable ${'$'}notakotlinvariable and tabs     \t are handled correctly
+            """
+
+        val resMl = prov.cmd(echoCommandForText(testMultiLineString)).out
+
+        // then
+        assertEquals(testMultiLineString, resMl)
+    }
 }