add spec for s3 objects to syspec

build.gradle

@@ -1,5 +1,5 @@
 buildscript {
-    ext.kotlin_version = "1.6.10"
+    ext.kotlin_version = "1.7.0"
     ext.CI_PROJECT_ID = System.env.CI_PROJECT_ID
 
     repositories { mavenCentral() }
@@ -80,6 +80,8 @@ dependencies {
     implementation("org.jetbrains.kotlin:kotlin-reflect:$kotlin_version")
     implementation("com.hierynomus:sshj:0.32.0")
 
+    implementation("aws.sdk.kotlin:s3:0.17.1-beta")
+
     testFixturesApi("org.junit.jupiter:junit-jupiter-api:5.8.2")
     testFixturesApi('io.mockk:mockk:1.12.3')
 

src/main/kotlin/org/domaindrivenarchitecture/provs/framework/core/Prov.kt

@@ -229,7 +229,7 @@ open class Prov protected constructor(
 
         for (cmd in linesNonEmpty) {
             if (success) {
-                success = success && cmd(cmd, dir, sudo).success
+                success = cmd(cmd, dir, sudo).success
             }
         }
         ProvResult(success)
@@ -351,7 +351,7 @@ open class Prov protected constructor(
     private fun printResults() {
         println(
             "============================================== SUMMARY " +
-                    (if (instanceName != null) "(" + instanceName + ") " else "") +
+                    (if (instanceName != null) "($instanceName) " else "") +
                     "============================================="
         )
         val successPerLevel = arrayListOf<Boolean>()

src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/domain/SyspecConfig.kt

@@ -12,6 +12,7 @@ data class SyspecConfig(
     val netcat: List<NetcatSpec>? = null,
     val socket: List<SocketSpec>? = null,
     val certificate: List<CertificateFileSpec>? = null,
+    val s3: List<S3ObjectSpec>? = null,
 )
 
 /**
@@ -47,3 +48,11 @@ data class SocketSpec(
 
 @Serializable
 data class CertificateFileSpec(val name: String, val expirationDays: Long)
+
+/**
+ * ATTENTION: usage of this spec for non-public s3 buckets requires the correct setup of an aws credential file "~/.aws/credentials"
+ * For more information, see: https://docs.aws.amazon.com/sdk-for-kotlin/latest/developer-guide/setup.html
+ */
+@Serializable
+data class S3ObjectSpec(val bucket: String, val prefix: String, val age: Long)
+

src/main/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/Verification.kt

@@ -1,5 +1,10 @@
 package org.domaindrivenarchitecture.provs.syspec.infrastructure
 
+import aws.sdk.kotlin.services.s3.S3Client
+import aws.sdk.kotlin.services.s3.model.ListObjectsRequest
+import aws.sdk.kotlin.services.s3.model.ListObjectsResponse
+import aws.smithy.kotlin.runtime.time.Instant
+import kotlinx.coroutines.runBlocking
 import org.domaindrivenarchitecture.provs.framework.core.Prov
 import org.domaindrivenarchitecture.provs.framework.core.ProvResult
 import org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base.checkDir
@@ -8,9 +13,13 @@ import org.domaindrivenarchitecture.provs.framework.ubuntu.install.base.isPackag
 import org.domaindrivenarchitecture.provs.syspec.domain.*
 import java.text.ParseException
 import java.text.SimpleDateFormat
+import java.time.Duration
 import java.util.*
 import java.util.concurrent.TimeUnit
 
+/**
+ * Verifies all sub-specs of a SyspecConfig
+ */
 fun Prov.verifySpecConfig(conf: SyspecConfig) = task {
     conf.command?.let { task("CommandSpecs") { for (spec in conf.command) verify(spec) } }
     conf.file?.let { task("FileSpecs") { for (spec in conf.file) verify(spec) } }
@@ -20,8 +29,10 @@ fun Prov.verifySpecConfig(conf: SyspecConfig) = task {
     conf.netcat?.let { task("NetcatSpecs") { for (spec in conf.netcat) verify(spec) } }
     conf.socket?.let { task("SocketSpecs") { for (spec in conf.socket) verify(spec) } }
     conf.certificate?.let { task("CertificateFileSpecs") { for (spec in conf.certificate) verify(spec) } }
+    conf.s3?.let { task("CertificateFileSpecs") { for (spec in conf.s3) verify(spec) } }
 }
 
+// -------------------------------  verification functions for individual specs  --------------------------------
 fun Prov.verify(cmd: CommandSpec) {
     val res = cmdNoEval(cmd.command)
     if (!res.success) {
@@ -101,8 +112,29 @@ fun Prov.verify(cert: CertificateFileSpec) {
     }
 }
 
+fun Prov.verify(s3ObjectSpec: S3ObjectSpec) {
+    val (bucket, prefix, maxAge) = s3ObjectSpec
+    val expectedAge = Duration.ofHours(s3ObjectSpec.age)
 
-// --------------------------  helper  ---------------------------------
+    val latestObject = getS3Objects(bucket, prefix).contents?.maxByOrNull { it.lastModified ?: Instant.fromEpochSeconds(0) }
+
+    if (latestObject == null) {
+        verify(false, "Could not retrieve an s3 object with prefix $prefix")
+    } else {
+        // convert to java.time.Instant for easier comparison
+        val lastModified = java.time.Instant.ofEpochSecond(latestObject.lastModified?.epochSeconds ?: 0)
+        val actualAge = Duration.between(lastModified, java.time.Instant.now())
+
+        verify(
+            actualAge <= expectedAge,
+            "Age is $actualAge (expected: < $maxAge) for latest file with prefix \"$prefix\" " +
+                    "---  modified date: $lastModified - size: ${(latestObject.size)} B - key: ${latestObject.key}"
+        )
+    }
+}
+
+
+// --------------------------  helper functions  ---------------------------------
 
 fun Prov.verifySocketSpec(socketConf: SocketSpec, outputLines: List<String>): ProvResult {
     val headLine = outputLines[0]
@@ -183,3 +215,14 @@ private fun Prov.verifyCertExpiration(enddate: String?, certName: String, expira
         )
     }
 }
+
+private fun getS3Objects(bucketName: String, prefixIn: String): ListObjectsResponse {
+
+    val request = ListObjectsRequest { bucket = bucketName; prefix = prefixIn }
+
+    return runBlocking {
+        S3Client { region = "eu-central-1" }.use { s3 ->
+            s3.listObjects(request)
+        }
+    }
+}

src/test/kotlin/org/domaindrivenarchitecture/provs/syspec/infrastructure/SyspecConfigRepoKtTest.kt

@@ -8,14 +8,13 @@ import org.junit.jupiter.api.Test
 internal class SyspecConfigRepoKtTest {
 
     @Test
-    fun findSpecConfigFromFile_if_default_file_is_not_found_success() {
+    fun findSpecConfigFromFile_use_default_config_if_default_config_file_is_not_found() {
         // when
-        @Suppress("RECEIVER_NULLABILITY_MISMATCH_BASED_ON_JAVA_ANNOTATIONS")  // null would reveal test error
         val res = findSpecConfigFromFile(ConfigFileName("syspec-config.yaml"))
 
         // then
         assertEquals(
-            "SyspecConfig(command=[CommandSpec(command=tfenv -h, out=null), CommandSpec(command=python3 --version, out=null), CommandSpec(command=pip3 --version, out=null), CommandSpec(command=terraform --version, out=1.0.8)], file=null, folder=null, host=null, package=[PackageSpec(name=firefox, installed=true), PackageSpec(name=thunderbird, installed=true), PackageSpec(name=ssh, installed=true), PackageSpec(name=git, installed=true), PackageSpec(name=leiningen, installed=true)], netcat=null, socket=null, certificate=null)",
+            "SyspecConfig(command=[CommandSpec(command=tfenv -h, out=null), CommandSpec(command=python3 --version, out=null), CommandSpec(command=pip3 --version, out=null), CommandSpec(command=terraform --version, out=1.0.8)], file=null, folder=null, host=null, package=[PackageSpec(name=firefox, installed=true), PackageSpec(name=thunderbird, installed=true), PackageSpec(name=ssh, installed=true), PackageSpec(name=git, installed=true), PackageSpec(name=leiningen, installed=true)], netcat=null, socket=null, certificate=null, s3=null)",
             res.getOrNull().toString())
     }