From a0fde38bd269acbd3ae486e386001106758bfe16 Mon Sep 17 00:00:00 2001 From: az Date: Wed, 25 Aug 2021 19:10:54 +0200 Subject: [PATCH] add sha256sum checksum check to downloadFormUrl --- .../provs/core/Prov.kt | 11 ++-- .../provs/core/Utils.kt | 3 +- .../core/docker/dockerimages/DockerImages.kt | 2 +- .../provs/extensions/workplace/base/Gopass.kt | 52 +++++++++++-------- .../extensions/workplace/base/GopassBridge.kt | 2 +- .../ubuntu/filesystem/base/Filesystem.kt | 4 +- .../provs/ubuntu/web/base/Web.kt | 40 ++++++++++---- .../filesystem/base/FilesystemKtTest.kt | 2 +- .../provs/ubuntu/web/base/WebKtTest.kt | 42 ++++++++++++++- 9 files changed, 111 insertions(+), 47 deletions(-) diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Prov.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Prov.kt index 97e0adc..db9458b 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Prov.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Prov.kt @@ -352,11 +352,10 @@ open class Prov protected constructor( "============================================== " ) for (result in internalResults) { - println( - result.toString().escapeNewline() - .replace("Success --", ANSI_BRIGHT_GREEN + "Success" + ANSI_RESET + " --") - .replace("FAILED --", ANSI_BRIGHT_RED + "FAILED" + ANSI_RESET + " --") - ) + val outputLine = result.toString().escapeControlChars() + .replaceFirst("Success --", ANSI_BRIGHT_GREEN + "Success" + ANSI_RESET + " --") + .replaceFirst("FAILED --", ANSI_BRIGHT_RED + "FAILED" + ANSI_RESET + " --") + println(outputLine) } if (internalResults.size > 1) { println("----------------------------------------------------------------------------------------------------- ") @@ -410,7 +409,7 @@ internal data class ResultLine(val level: Int, val method: String?, var provResu return if (provResult != null) { prefix(level) + (if (provResult.success) "Success -- " else "FAILED -- ") + method + " " + (provResult.cmd ?: "") + - (if (!provResult.success && provResult.err != null) " -- Error: " + provResult.err.escapeNewline() else "") + (if (!provResult.success && provResult.err != null) " -- Error: " + provResult.err.escapeControlChars() else "") } else prefix(level) + method + " " + "... in progress ... " diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Utils.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Utils.kt index 5375074..55336e7 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Utils.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/core/Utils.kt @@ -40,7 +40,8 @@ fun getCallingMethodName(): String? { } -fun String.escapeNewline(): String = this.replace("\r\n", "\\n").replace("\n", "\\n") +fun String.escapeNewline(): String = this.replace("\r", "\\r").replace("\n", "\\n") +fun String.escapeControlChars(): String = this.replace("\r", "\\r").replace("\n", "\\n").replace("\t", "\\t").replace("[\\p{Cntrl}]".toRegex(), "\\?") fun String.escapeBackslash(): String = this.replace("\\", "\\\\") fun String.escapeDoubleQuote(): String = this.replace("\"", "\\\"") fun String.escapeSingleQuote(): String = this.replace("'", "\'") diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/core/docker/dockerimages/DockerImages.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/core/docker/dockerimages/DockerImages.kt index 3278758..f2bdfbb 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/core/docker/dockerimages/DockerImages.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/core/docker/dockerimages/DockerImages.kt @@ -17,7 +17,7 @@ class UbuntuPlusUser(private val userName: String = "testuser") : DockerImage { override fun imageText(): String { return """ -FROM ubuntu:18.04 +FROM ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/Gopass.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/Gopass.kt index c5050fd..59238b3 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/Gopass.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/Gopass.kt @@ -2,29 +2,39 @@ package org.domaindrivenarchitecture.provs.extensions.workplace.base import org.domaindrivenarchitecture.provs.core.Prov import org.domaindrivenarchitecture.provs.core.ProvResult -import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.createDir -import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.createDirs -import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.createFile -import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.userHome +import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.* import org.domaindrivenarchitecture.provs.ubuntu.install.base.aptInstall import org.domaindrivenarchitecture.provs.ubuntu.install.base.isPackageInstalled +import org.domaindrivenarchitecture.provs.ubuntu.web.base.downloadFromURL fun Prov.installGopass(version: String = "1.12.7", enforceVersion: Boolean = false) = def { - if (isPackageInstalled("gopass") && !enforceVersion) { - ProvResult(true) - } else { - // install required dependencies - aptInstall("rng-tools gnupg2 git") - aptInstall("curl") - sh( - """ - curl -L https://github.com/gopasspw/gopass/releases/download/v${version}/gopass_${version}_linux_amd64.deb -o gopass_${version}_linux_amd64.deb - sudo dpkg -i gopass_${version}_linux_amd64.deb - """ - ) - gopassEnsureVersion(version) + val sha256sum = "0824d5110ff1e68bff1ba10c1be63acb67cb1ad8e3bccddd6b6fc989608beca8" // checksum for sha256sum version 8.30 (e.g. ubuntu 20.04) + + if (isPackageInstalled("gopass") && !enforceVersion) { + return@def ProvResult(true) + } + if (checkGopassVersion(version)) { + return@def ProvResult(true, out = "Version $version of gopass is already installed.") + } + + val path = "tmp" + // install required dependencies + aptInstall("rng-tools gnupg2 git") + val filename = "gopass_${version}_linux_amd64.deb" + val result = downloadFromURL( + "https://github.com/gopasspw/gopass/releases/download/v$version/$filename", + filename, + path, + sha256sum = sha256sum + ) + if (result.success) { + cmd("sudo dpkg -i $path/gopass_${version}_linux_amd64.deb") + // Cross-check if installation was successful + addResultToEval(ProvResult(checkGopassVersion(version))) + } else { + addResultToEval(ProvResult(false, err = "Gopass could not be installed. " + result.err)) } } @@ -76,13 +86,9 @@ mounts: {} * * @param version that is checked; specifies left part of text of installed version, e.g. both "1" and "1.12" will return true if installed version is "1.12.6+8d7a311b9273846bbb618e4bd9ddbae51b1db7b8" */ -internal fun Prov.gopassEnsureVersion(version: String) = def { +internal fun Prov.checkGopassVersion(version: String): Boolean { val installedGopassVersion = gopassVersion() - if (installedGopassVersion != null && installedGopassVersion.startsWith("gopass " + version)) { - ProvResult(true, out = "Required gopass version ($version) matches installed version ($installedGopassVersion)") - } else { - ProvResult(false, err = "Wrong gopass version. Expected $version but found $installedGopassVersion") - } + return installedGopassVersion != null && installedGopassVersion.startsWith("gopass " + version) } internal fun Prov.gopassVersion(): String? { diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/GopassBridge.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/GopassBridge.kt index 92f1531..ad63092 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/GopassBridge.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/extensions/workplace/base/GopassBridge.kt @@ -34,7 +34,7 @@ fun Prov.installGopassBridgeJsonApi() = def { if (installedJsonApiVersion == null) { if (chk("gopass ls")) { - if (gopassEnsureVersion(requiredGopassVersion).success) { + if (checkGopassVersion(requiredGopassVersion)) { aptInstall("git gnupg2") // required dependencies createDir(downloadDir) downloadFromURL(downloadUrl, filename, downloadDir) diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/Filesystem.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/Filesystem.kt index d97e4bd..92fb275 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/Filesystem.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/Filesystem.kt @@ -54,8 +54,8 @@ fun Prov.createSecretFile( } -fun Prov.deleteFile(file: String, sudo: Boolean = false): ProvResult = def { - cmd((if (sudo) "sudo " else "") + "rm $file") +fun Prov.deleteFile(file: String, path: String? = null, sudo: Boolean = false): ProvResult = def { + cmd((path?.let { "cd $path && " } ?: "") + (if (sudo) "sudo " else "") + "rm $file") } diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/Web.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/Web.kt index 5336d9d..e6503d8 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/Web.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/Web.kt @@ -2,25 +2,45 @@ package org.domaindrivenarchitecture.provs.ubuntu.web.base import org.domaindrivenarchitecture.provs.core.Prov import org.domaindrivenarchitecture.provs.core.ProvResult +import org.domaindrivenarchitecture.provs.core.tags.Api +import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.createDirs +import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.deleteFile import org.domaindrivenarchitecture.provs.ubuntu.install.base.aptInstall import org.domaindrivenarchitecture.provs.ubuntu.install.base.isPackageInstalled /** - * Downloads a file from the given URL using curl + * Downloads a file from the given URL using curl. * - * @param path where to download to - * @param url file to download - * @param filename filename after download + * ATTENTION: sha256sum uses the version available, which can differ in different versions of ubuntu; e.g. gopass download only works with sha256sum version 8.30 from ubuntu 20.04 ! */ -@Suppress("unused") // used externally -fun Prov.downloadFromURL(url: String, filename: String? = null, path: String? = null, sudo: Boolean = false) : ProvResult = def { +@Api +fun Prov.downloadFromURL( + url: String, + filename: String? = null, + path: String? = null, + sudo: Boolean = false, + followRedirect: Boolean = true, + sha256sum: String? = null +): ProvResult = def { - if (!isPackageInstalled("curl")) aptInstall("curl") + aptInstall("curl") - if (filename == null) { - cmd("curl $url", path, sudo) + val followRedirectOption = if (followRedirect) "-L" else "" + val filenameFromUrl = url.substringAfterLast("/") + + val finalFilename: String = filename ?: filenameFromUrl + + path?.let { createDirs(path) } + cmd("curl $followRedirectOption $url -o $finalFilename", path, sudo) + + if (sha256sum != null) { + if (!cmd("echo \"$sha256sum $finalFilename\" | sha256sum --check", path).success) { + deleteFile(finalFilename, path, sudo) + } else { + ProvResult(true, out = "Sha256sum is correct.") + } } else { - cmd("curl $url -o $filename", path, sudo) + ProvResult(true, out = "No sha256sum check requested.") } } \ No newline at end of file diff --git a/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/FilesystemKtTest.kt b/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/FilesystemKtTest.kt index df29d52..3c78530 100644 --- a/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/FilesystemKtTest.kt +++ b/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/filesystem/base/FilesystemKtTest.kt @@ -49,7 +49,7 @@ internal class FilesystemKtTest { val res4b = prov.fileContainsText(file, "some non-existing content") val res5 = prov.deleteFile(file) val res6 = prov.fileExists(file) - val res7 = prov.deleteFile(file, true) + val res7 = prov.deleteFile(file, sudo = true) val res8 = prov.fileExists(file) // then diff --git a/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/WebKtTest.kt b/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/WebKtTest.kt index 07f5879..1c26410 100644 --- a/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/WebKtTest.kt +++ b/src/test/kotlin/org/domaindrivenarchitecture/provs/ubuntu/web/base/WebKtTest.kt @@ -4,8 +4,8 @@ import org.domaindrivenarchitecture.provs.test.defaultTestContainer import org.domaindrivenarchitecture.provs.test.tags.ContainerTest import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.createFile import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.fileContent -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue +import org.domaindrivenarchitecture.provs.ubuntu.filesystem.base.fileExists +import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test internal class WebKtTest { @@ -27,4 +27,42 @@ internal class WebKtTest { assertTrue(res.success) assertEquals("hello", res2) } + + @ContainerTest + @Test + fun downloadFromURL_local_file_with_correct_checksum() { + // given + val a = defaultTestContainer() + val srcFile = "file3.txt" + val targetFile = "file3b.txt" + a.createFile("/tmp/" + srcFile, "hello") + + // when + val res = a.downloadFromURL("file:///tmp/" + srcFile, targetFile, "tmp", sha256sum ="2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824") // ubuntu 20.04 sha256sum version 8.30 + + // then + val res2 = a.fileContent("tmp/$targetFile") + + assertEquals("hello", res2) + assertTrue(res.success) + } + + @ContainerTest + @Test + fun downloadFromURL_local_file_with_incorrect_checksum() { + // given + val a = defaultTestContainer() + val srcFile = "file3.txt" + val targetFile = "file3b.txt" + a.createFile("/tmp/" + srcFile, "hello") + + // when + val res = a.downloadFromURL("file:///tmp/" + srcFile, targetFile, "tmp", sha256sum = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824WRONG") + + // then + val res2 = a.fileExists("tmp/$targetFile") + + assertFalse(res.success) + assertFalse(res2) + } } \ No newline at end of file