diff --git a/build.gradle b/build.gradle index f11973b..62e4b90 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ buildscript { - ext.kotlin_version = "1.6.10" + ext.kotlin_version = "1.7.0" ext.CI_PROJECT_ID = System.env.CI_PROJECT_ID repositories { mavenCentral() } @@ -80,6 +80,8 @@ dependencies { implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlin_version") implementation("com.hierynomus:sshj:0.32.0") + implementation("aws.sdk.kotlin:s3:0.17.1-beta") + testFixturesApi("org.junit.jupiter:junit-jupiter-api:5.8.2") testFixturesApi('io.mockk:mockk:1.12.3') diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/framework/core/Prov.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/framework/core/Prov.kt index 0a3e6e4..6420ccb 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/framework/core/Prov.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/framework/core/Prov.kt @@ -229,7 +229,7 @@ open class Prov protected constructor( for (cmd in linesNonEmpty) { if (success) { - success = success && cmd(cmd, dir, sudo).success + success = cmd(cmd, dir, sudo).success } } ProvResult(success) @@ -351,7 +351,7 @@ open class Prov protected constructor( private fun printResults() { println( "============================================== SUMMARY " + - (if (instanceName != null) "(" + instanceName + ") " else "") + + (if (instanceName != null) "($instanceName) " else "") + "=============================================" ) val successPerLevel = arrayListOf() diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/domain/SyspecConfig.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/domain/SyspecConfig.kt index 96356d9..ab554bb 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/domain/SyspecConfig.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/domain/SyspecConfig.kt @@ -12,6 +12,7 @@ data class SyspecConfig( val netcat: List? = null, val socket: List? = null, val certificate: List? = null, + val s3: List? = null, ) /** @@ -47,3 +48,11 @@ data class SocketSpec( @Serializable data class CertificateFileSpec(val name: String, val expirationDays: Long) + +/** + * ATTENTION: usage of this spec for non-public s3 buckets requires the correct setup of an aws credential file "~/.aws/credentials" + * For more information, see: https://docs.aws.amazon.com/sdk-for-kotlin/latest/developer-guide/setup.html + */ +@Serializable +data class S3ObjectSpec(val bucket: String, val prefix: String, val age: Long) + diff --git a/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/Verification.kt b/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/Verification.kt index 2195dbe..d2d8019 100644 --- a/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/Verification.kt +++ b/src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/Verification.kt @@ -1,5 +1,10 @@ package org.domaindrivenarchitecture.provs.syspec.infrastructure +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.ListObjectsRequest +import aws.sdk.kotlin.services.s3.model.ListObjectsResponse +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.runBlocking import org.domaindrivenarchitecture.provs.framework.core.Prov import org.domaindrivenarchitecture.provs.framework.core.ProvResult import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.checkDir @@ -8,9 +13,13 @@ import org.domaindrivenarchitecture.provs.framework.ubuntu.install.base.isPackag import org.domaindrivenarchitecture.provs.syspec.domain.* import java.text.ParseException import java.text.SimpleDateFormat +import java.time.Duration import java.util.* import java.util.concurrent.TimeUnit +/** + * Verifies all sub-specs of a SyspecConfig + */ fun Prov.verifySpecConfig(conf: SyspecConfig) = task { conf.command?.let { task("CommandSpecs") { for (spec in conf.command) verify(spec) } } conf.file?.let { task("FileSpecs") { for (spec in conf.file) verify(spec) } } @@ -20,8 +29,10 @@ fun Prov.verifySpecConfig(conf: SyspecConfig) = task { conf.netcat?.let { task("NetcatSpecs") { for (spec in conf.netcat) verify(spec) } } conf.socket?.let { task("SocketSpecs") { for (spec in conf.socket) verify(spec) } } conf.certificate?.let { task("CertificateFileSpecs") { for (spec in conf.certificate) verify(spec) } } + conf.s3?.let { task("CertificateFileSpecs") { for (spec in conf.s3) verify(spec) } } } +// ------------------------------- verification functions for individual specs -------------------------------- fun Prov.verify(cmd: CommandSpec) { val res = cmdNoEval(cmd.command) if (!res.success) { @@ -101,8 +112,29 @@ fun Prov.verify(cert: CertificateFileSpec) { } } +fun Prov.verify(s3ObjectSpec: S3ObjectSpec) { + val (bucket, prefix, maxAge) = s3ObjectSpec + val expectedAge = Duration.ofHours(s3ObjectSpec.age) -// -------------------------- helper --------------------------------- + val latestObject = getS3Objects(bucket, prefix).contents?.maxByOrNull { it.lastModified ?: Instant.fromEpochSeconds(0) } + + if (latestObject == null) { + verify(false, "Could not retrieve an s3 object with prefix $prefix") + } else { + // convert to java.time.Instant for easier comparison + val lastModified = java.time.Instant.ofEpochSecond(latestObject.lastModified?.epochSeconds ?: 0) + val actualAge = Duration.between(lastModified, java.time.Instant.now()) + + verify( + actualAge <= expectedAge, + "Age is $actualAge (expected: < $maxAge) for latest file with prefix \"$prefix\" " + + "--- modified date: $lastModified - size: ${(latestObject.size)} B - key: ${latestObject.key}" + ) + } +} + + +// -------------------------- helper functions --------------------------------- fun Prov.verifySocketSpec(socketConf: SocketSpec, outputLines: List): ProvResult { val headLine = outputLines[0] @@ -183,3 +215,14 @@ private fun Prov.verifyCertExpiration(enddate: String?, certName: String, expira ) } } + +private fun getS3Objects(bucketName: String, prefixIn: String): ListObjectsResponse { + + val request = ListObjectsRequest { bucket = bucketName; prefix = prefixIn } + + return runBlocking { + S3Client { region = "eu-central-1" }.use { s3 -> + s3.listObjects(request) + } + } +} diff --git a/src/test/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/SyspecConfigRepoKtTest.kt b/src/test/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/SyspecConfigRepoKtTest.kt index 0b38e52..cd7b491 100644 --- a/src/test/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/SyspecConfigRepoKtTest.kt +++ b/src/test/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/SyspecConfigRepoKtTest.kt @@ -8,14 +8,13 @@ import org.junit.jupiter.api.Test internal class SyspecConfigRepoKtTest { @Test - fun findSpecConfigFromFile_if_default_file_is_not_found_success() { + fun findSpecConfigFromFile_use_default_config_if_default_config_file_is_not_found() { // when - @Suppress("RECEIVER_NULLABILITY_MISMATCH_BASED_ON_JAVA_ANNOTATIONS") // null would reveal test error val res = findSpecConfigFromFile(ConfigFileName("syspec-config.yaml")) // then assertEquals( - "SyspecConfig(command=[CommandSpec(command=tfenv -h, out=null), CommandSpec(command=python3 --version, out=null), CommandSpec(command=pip3 --version, out=null), CommandSpec(command=terraform --version, out=1.0.8)], file=null, folder=null, host=null, package=[PackageSpec(name=firefox, installed=true), PackageSpec(name=thunderbird, installed=true), PackageSpec(name=ssh, installed=true), PackageSpec(name=git, installed=true), PackageSpec(name=leiningen, installed=true)], netcat=null, socket=null, certificate=null)", + "SyspecConfig(command=[CommandSpec(command=tfenv -h, out=null), CommandSpec(command=python3 --version, out=null), CommandSpec(command=pip3 --version, out=null), CommandSpec(command=terraform --version, out=1.0.8)], file=null, folder=null, host=null, package=[PackageSpec(name=firefox, installed=true), PackageSpec(name=thunderbird, installed=true), PackageSpec(name=ssh, installed=true), PackageSpec(name=git, installed=true), PackageSpec(name=leiningen, installed=true)], netcat=null, socket=null, certificate=null, s3=null)", res.getOrNull().toString()) }