add fileContent for large files

src/main/kotlin/org/domaindrivenarchitecture/provs/framework/ubuntu/filesystem/base/Filesystem.kt

@@ -4,6 +4,7 @@ import org.domaindrivenarchitecture.provs.framework.core.platforms.SHELL
 import org.domaindrivenarchitecture.provs.framework.core.*
 import org.domaindrivenarchitecture.provs.framework.core.getLocalFileContent
 import java.io.File
+import java.util.*
 
 
 /**
@@ -109,7 +110,7 @@ fun Prov.createFile(
         val chunkedTest = text.chunked(maxBlockSize)
         for (chunk in chunkedTest) {
             // todo: consider usage of function addTextToFile
-            cmd(
+            cmdNoLog(
                 "printf '%s' " + chunk
                     .escapeAndEncloseByDoubleQuoteForShell() + " | $withSudo tee -a $fullyQualifiedFilename > /dev/null"
             )
@@ -145,28 +146,71 @@ fun Prov.deleteFile(file: String, path: String? = null, sudo: Boolean = false):
 
 
 fun Prov.fileContainsText(file: String, content: String, sudo: Boolean = false): Boolean {
-    // todo consider grep e.g. for content without newlines
+    if (!checkFile(file, sudo)) {
-    //        return cmdNoEval(prefixWithSudo("grep -- '${content.escapeSingleQuote()}' $file", sudo)).success
+        return false
-    val fileContent = fileContent(file, sudo = sudo)
+    }
-    return if (fileContent == null) {
+
-        false
+    // use grep for a single line or for a single line enclosed by a newline
+    if (!content.contains("\n") || (content.length >= 3 && !content.drop(1).dropLast(1).contains("\n"))) {
+        return cmdNoEval(prefixWithSudo("grep -- '${content.escapeSingleQuote().trim('\n')}' $file", sudo)).success
     } else {
-        fileContent.contains(content)
+        val fileContent = fileContent(file, sudo = sudo)
+        return fileContent?.contains(content) ?: false
     }
 }
 
 
 fun Prov.fileContent(file: String, sudo: Boolean = false): String? {
+    val largeFileSize = 40000
+
+    val size = fileSize(file, sudo)
+    if (size == null || size > largeFileSize) {
+        return fileContentLargeFile(file, sudo)
+    } else {
         return cmdNoEval(prefixWithSudo("cat $file", sudo)).out
     }
+}
 
+fun Prov.fileContentLargeFile(file: String, sudo: Boolean = false, chunkSize: Int = 10000): String? {
+    require(chunkSize <= 40000) { "Chunk size must be < 40000" }
+    val maxSizeLargeFileContent = 10000000  // 10 MB
+    val size = fileSize(file, sudo)
+    if (size != null && size > maxSizeLargeFileContent) {
+        throw IllegalArgumentException("Cannot retrieve file content of files larger than: $maxSizeLargeFileContent bytes")
+    }
 
-fun Prov.addTextToFile(
+    var offset = 0
-    text: String,
+
-    file: String,
+    var resultString: String? = null
-    doNotAddIfExisting: Boolean = true,
+    do {
-    sudo: Boolean = false
+        // todo : file paths starting with ~/ are not yet supported
-): ProvResult = addTextToFile(text, File(file), doNotAddIfExisting, sudo)
+        val chunkResult =
+            cmdNoEval(prefixWithSudo("dd if=\"$file\" iflag=skip_bytes,count_bytes,fullblock bs=\"$chunkSize\" skip=\"$offset\" count=\"$chunkSize\" status=none | base64", sudo))
+
+        // check first chunk
+        if (resultString == null) {
+            if (!chunkResult.success) {
+                return resultString
+            } else {
+                resultString = ""
+            }
+        }
+
+        val b = chunkResult.out?.trim() ?: ""
+        offset += chunkSize
+
+        if (b.isEmpty() || b == "0") {
+            break
+        }
+
+        // Use MimeDecoder to ignore newlines (\n)
+        val decodedBytes: ByteArray = Base64.getMimeDecoder().decode( b )
+        val dec = String(decodedBytes)
+        resultString += dec
+    } while (true)
+
+    return resultString
+}
 
 
 fun Prov.addTextToFile(
@@ -303,8 +347,8 @@ fun Prov.userHome(): String {
 /**
  * Returns number of bytes of a file or null if size could not be determined
  */
-fun Prov.fileSize(filename: String): Int? {
+fun Prov.fileSize(filename: String, sudo: Boolean = false): Int? {
-    val result = cmd("wc -c < $filename")
+    val result = cmdNoEval("wc -c < $filename", sudo = sudo)
     return result.out?.trim()?.toIntOrNull()
 }
 

src/test/kotlin/org/domaindrivenarchitecture/provs/framework/ubuntu/filesystem/base/FilesystemKtTest.kt

@@ -2,6 +2,7 @@ package org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base
 
 import org.domaindrivenarchitecture.provs.test.defaultTestContainer
 import org.domaindrivenarchitecture.provs.test.tags.ContainerTest
+import org.domaindrivenarchitecture.provs.test.tags.ExtensiveContainerTest
 import org.domaindrivenarchitecture.provs.test.testLocal
 import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.Test
@@ -275,18 +276,88 @@ internal class FilesystemKtTest {
     @ContainerTest
     fun fileContainsText() {
         // given
-        defaultTestContainer().createFile("testfilecontainingtext", "abc\n- def\nefg")
+        val file = "file_with_text"
+        defaultTestContainer().createFile(file, "\n\nabc\n- def\nefg\nhij\nklm\no\npq", sudo = true)
 
         // when
-        val res = defaultTestContainer().fileContainsText("testfilecontainingtext", "abc")
+        val res = defaultTestContainer().fileContainsText(file, "abc")
-        val res2 = defaultTestContainer().fileContainsText("testfilecontainingtext", "de")
+        val res2 = defaultTestContainer().fileContainsText(file, "de")
-        val res3 = defaultTestContainer().fileContainsText("testfilecontainingtext", "- def")
+        val res3 = defaultTestContainer().fileContainsText(file, "- def")
-        val res4 = defaultTestContainer().fileContainsText("testfilecontainingtext", "xyy")
+        val res4 = defaultTestContainer().fileContainsText(file, "xyy")
+        val res5 = defaultTestContainer().fileContainsText(file, "c\n- def\nefg\nhi")
+        val res6 = defaultTestContainer().fileContainsText(file, "\n\n")
+        val res7 = defaultTestContainer().fileContainsText(file, "\n\n\n")
+        val res8 = defaultTestContainer().fileContainsText(file, "\no\n")
+        val res10 = defaultTestContainer().fileContainsText(file, "\n\nabc")
 
         // then
         assertTrue(res)
         assertTrue(res2)
         assertTrue(res3)
         assertFalse(res4)
+        assertTrue(res5)
+        assertTrue(res6)
+        assertFalse(res7)
+        assertTrue(res8)
+        assertTrue(res10)
+    }
+
+    @Test
+    @ContainerTest
+    fun fileContainsText_with_sudo() {
+        // given
+        val file = "sudotestfilecontainingtext"
+        defaultTestContainer().createFile(file, "abc\n- def\nefg\nhij\nklm\nop", sudo = true)
+
+        // when
+        val res = defaultTestContainer().fileContainsText(file, "abc", sudo = true)
+        val res2 = defaultTestContainer().fileContainsText(file, "de", sudo = true)
+        val res3 = defaultTestContainer().fileContainsText(file, "- def", sudo = true)
+        val res4 = defaultTestContainer().fileContainsText(file, "xyy", sudo = true)
+        // test if newlines are recognized
+        val res5 = defaultTestContainer().fileContainsText(file, "c\n- def\nefg\nhi", sudo = true)
+
+        // then
+        assertTrue(res)
+        assertTrue(res2)
+        assertTrue(res3)
+        assertFalse(res4)
+        assertTrue(res5)
+    }
+
+    @ExtensiveContainerTest
+    fun fileContentLargeFile_success() {
+        // given
+        val prov = defaultTestContainer()
+        val filename = "largetestfile"
+        val content = "012345äöüß".repeat(100000)
+
+        // when
+        val res = prov.createFile(filename, content, overwriteIfExisting = true)
+        val size = prov.fileSize(filename)
+        val actualContent = prov.fileContentLargeFile(filename, chunkSize = 40000)
+
+        // then
+        assertTrue(res.success)
+        assertEquals(content, actualContent)
+        assertEquals(1400000, size)
+    }
+
+    @ExtensiveContainerTest
+    fun fileContentLargeFile_with_sudo_success() {
+        // given
+        val prov = defaultTestContainer()
+        val filename = "largetestfile"
+        val content = "012345äöüß".repeat(100000)
+
+        // when
+        val res = prov.createFile(filename, content, overwriteIfExisting = true, sudo = true)
+        val size = prov.fileSize(filename, sudo = true)
+        val actualContent = prov.fileContentLargeFile(filename, chunkSize = 40000, sudo = true)
+
+        // then
+        assertTrue(res.success)
+        assertEquals(content, actualContent)
+        assertEquals(1400000, size)
     }
 }