add fileContent for large files
This commit is contained in:
parent
db33edf0f7
commit
906deb4ac7
2 changed files with 137 additions and 22 deletions
|
|
@ -4,6 +4,7 @@ import org.domaindrivenarchitecture.provs.framework.core.platforms.SHELL
|
|||
import org.domaindrivenarchitecture.provs.framework.core.*
|
||||
import org.domaindrivenarchitecture.provs.framework.core.getLocalFileContent
|
||||
import java.io.File
|
||||
import java.util.*
|
||||
|
||||
|
||||
/**
|
||||
|
|
@ -109,7 +110,7 @@ fun Prov.createFile(
|
|||
val chunkedTest = text.chunked(maxBlockSize)
|
||||
for (chunk in chunkedTest) {
|
||||
// todo: consider usage of function addTextToFile
|
||||
cmd(
|
||||
cmdNoLog(
|
||||
"printf '%s' " + chunk
|
||||
.escapeAndEncloseByDoubleQuoteForShell() + " | $withSudo tee -a $fullyQualifiedFilename > /dev/null"
|
||||
)
|
||||
|
|
@ -145,28 +146,71 @@ fun Prov.deleteFile(file: String, path: String? = null, sudo: Boolean = false):
|
|||
|
||||
|
||||
fun Prov.fileContainsText(file: String, content: String, sudo: Boolean = false): Boolean {
|
||||
// todo consider grep e.g. for content without newlines
|
||||
// return cmdNoEval(prefixWithSudo("grep -- '${content.escapeSingleQuote()}' $file", sudo)).success
|
||||
val fileContent = fileContent(file, sudo = sudo)
|
||||
return if (fileContent == null) {
|
||||
false
|
||||
if (!checkFile(file, sudo)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// use grep for a single line or for a single line enclosed by a newline
|
||||
if (!content.contains("\n") || (content.length >= 3 && !content.drop(1).dropLast(1).contains("\n"))) {
|
||||
return cmdNoEval(prefixWithSudo("grep -- '${content.escapeSingleQuote().trim('\n')}' $file", sudo)).success
|
||||
} else {
|
||||
fileContent.contains(content)
|
||||
val fileContent = fileContent(file, sudo = sudo)
|
||||
return fileContent?.contains(content) ?: false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
fun Prov.fileContent(file: String, sudo: Boolean = false): String? {
|
||||
val largeFileSize = 40000
|
||||
|
||||
val size = fileSize(file, sudo)
|
||||
if (size == null || size > largeFileSize) {
|
||||
return fileContentLargeFile(file, sudo)
|
||||
} else {
|
||||
return cmdNoEval(prefixWithSudo("cat $file", sudo)).out
|
||||
}
|
||||
}
|
||||
|
||||
fun Prov.fileContentLargeFile(file: String, sudo: Boolean = false, chunkSize: Int = 10000): String? {
|
||||
require(chunkSize <= 40000) { "Chunk size must be < 40000" }
|
||||
val maxSizeLargeFileContent = 10000000 // 10 MB
|
||||
val size = fileSize(file, sudo)
|
||||
if (size != null && size > maxSizeLargeFileContent) {
|
||||
throw IllegalArgumentException("Cannot retrieve file content of files larger than: $maxSizeLargeFileContent bytes")
|
||||
}
|
||||
|
||||
fun Prov.addTextToFile(
|
||||
text: String,
|
||||
file: String,
|
||||
doNotAddIfExisting: Boolean = true,
|
||||
sudo: Boolean = false
|
||||
): ProvResult = addTextToFile(text, File(file), doNotAddIfExisting, sudo)
|
||||
var offset = 0
|
||||
|
||||
var resultString: String? = null
|
||||
do {
|
||||
// todo : file paths starting with ~/ are not yet supported
|
||||
val chunkResult =
|
||||
cmdNoEval(prefixWithSudo("dd if=\"$file\" iflag=skip_bytes,count_bytes,fullblock bs=\"$chunkSize\" skip=\"$offset\" count=\"$chunkSize\" status=none | base64", sudo))
|
||||
|
||||
// check first chunk
|
||||
if (resultString == null) {
|
||||
if (!chunkResult.success) {
|
||||
return resultString
|
||||
} else {
|
||||
resultString = ""
|
||||
}
|
||||
}
|
||||
|
||||
val b = chunkResult.out?.trim() ?: ""
|
||||
offset += chunkSize
|
||||
|
||||
if (b.isEmpty() || b == "0") {
|
||||
break
|
||||
}
|
||||
|
||||
// Use MimeDecoder to ignore newlines (\n)
|
||||
val decodedBytes: ByteArray = Base64.getMimeDecoder().decode( b )
|
||||
val dec = String(decodedBytes)
|
||||
resultString += dec
|
||||
} while (true)
|
||||
|
||||
return resultString
|
||||
}
|
||||
|
||||
|
||||
fun Prov.addTextToFile(
|
||||
|
|
@ -303,8 +347,8 @@ fun Prov.userHome(): String {
|
|||
/**
|
||||
* Returns number of bytes of a file or null if size could not be determined
|
||||
*/
|
||||
fun Prov.fileSize(filename: String): Int? {
|
||||
val result = cmd("wc -c < $filename")
|
||||
fun Prov.fileSize(filename: String, sudo: Boolean = false): Int? {
|
||||
val result = cmdNoEval("wc -c < $filename", sudo = sudo)
|
||||
return result.out?.trim()?.toIntOrNull()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ package org.domaindrivenarchitecture.provs.framework.ubuntu.filesystem.base
|
|||
|
||||
import org.domaindrivenarchitecture.provs.test.defaultTestContainer
|
||||
import org.domaindrivenarchitecture.provs.test.tags.ContainerTest
|
||||
import org.domaindrivenarchitecture.provs.test.tags.ExtensiveContainerTest
|
||||
import org.domaindrivenarchitecture.provs.test.testLocal
|
||||
import org.junit.jupiter.api.Assertions.*
|
||||
import org.junit.jupiter.api.Test
|
||||
|
|
@ -267,7 +268,7 @@ internal class FilesystemKtTest {
|
|||
defaultTestContainer().copyFileFromLocal("copiedFileFromLocal", "$resourcesDirectory/resource-test")
|
||||
|
||||
// then
|
||||
val content = defaultTestContainer().fileContent( "copiedFileFromLocal")
|
||||
val content = defaultTestContainer().fileContent("copiedFileFromLocal")
|
||||
assertEquals("resource text\n", content)
|
||||
}
|
||||
|
||||
|
|
@ -275,18 +276,88 @@ internal class FilesystemKtTest {
|
|||
@ContainerTest
|
||||
fun fileContainsText() {
|
||||
// given
|
||||
defaultTestContainer().createFile("testfilecontainingtext", "abc\n- def\nefg")
|
||||
val file = "file_with_text"
|
||||
defaultTestContainer().createFile(file, "\n\nabc\n- def\nefg\nhij\nklm\no\npq", sudo = true)
|
||||
|
||||
// when
|
||||
val res = defaultTestContainer().fileContainsText("testfilecontainingtext", "abc")
|
||||
val res2 = defaultTestContainer().fileContainsText("testfilecontainingtext", "de")
|
||||
val res3 = defaultTestContainer().fileContainsText("testfilecontainingtext", "- def")
|
||||
val res4 = defaultTestContainer().fileContainsText("testfilecontainingtext", "xyy")
|
||||
val res = defaultTestContainer().fileContainsText(file, "abc")
|
||||
val res2 = defaultTestContainer().fileContainsText(file, "de")
|
||||
val res3 = defaultTestContainer().fileContainsText(file, "- def")
|
||||
val res4 = defaultTestContainer().fileContainsText(file, "xyy")
|
||||
val res5 = defaultTestContainer().fileContainsText(file, "c\n- def\nefg\nhi")
|
||||
val res6 = defaultTestContainer().fileContainsText(file, "\n\n")
|
||||
val res7 = defaultTestContainer().fileContainsText(file, "\n\n\n")
|
||||
val res8 = defaultTestContainer().fileContainsText(file, "\no\n")
|
||||
val res10 = defaultTestContainer().fileContainsText(file, "\n\nabc")
|
||||
|
||||
// then
|
||||
assertTrue(res)
|
||||
assertTrue(res2)
|
||||
assertTrue(res3)
|
||||
assertFalse(res4)
|
||||
assertTrue(res5)
|
||||
assertTrue(res6)
|
||||
assertFalse(res7)
|
||||
assertTrue(res8)
|
||||
assertTrue(res10)
|
||||
}
|
||||
|
||||
@Test
|
||||
@ContainerTest
|
||||
fun fileContainsText_with_sudo() {
|
||||
// given
|
||||
val file = "sudotestfilecontainingtext"
|
||||
defaultTestContainer().createFile(file, "abc\n- def\nefg\nhij\nklm\nop", sudo = true)
|
||||
|
||||
// when
|
||||
val res = defaultTestContainer().fileContainsText(file, "abc", sudo = true)
|
||||
val res2 = defaultTestContainer().fileContainsText(file, "de", sudo = true)
|
||||
val res3 = defaultTestContainer().fileContainsText(file, "- def", sudo = true)
|
||||
val res4 = defaultTestContainer().fileContainsText(file, "xyy", sudo = true)
|
||||
// test if newlines are recognized
|
||||
val res5 = defaultTestContainer().fileContainsText(file, "c\n- def\nefg\nhi", sudo = true)
|
||||
|
||||
// then
|
||||
assertTrue(res)
|
||||
assertTrue(res2)
|
||||
assertTrue(res3)
|
||||
assertFalse(res4)
|
||||
assertTrue(res5)
|
||||
}
|
||||
|
||||
@ExtensiveContainerTest
|
||||
fun fileContentLargeFile_success() {
|
||||
// given
|
||||
val prov = defaultTestContainer()
|
||||
val filename = "largetestfile"
|
||||
val content = "012345äöüß".repeat(100000)
|
||||
|
||||
// when
|
||||
val res = prov.createFile(filename, content, overwriteIfExisting = true)
|
||||
val size = prov.fileSize(filename)
|
||||
val actualContent = prov.fileContentLargeFile(filename, chunkSize = 40000)
|
||||
|
||||
// then
|
||||
assertTrue(res.success)
|
||||
assertEquals(content, actualContent)
|
||||
assertEquals(1400000, size)
|
||||
}
|
||||
|
||||
@ExtensiveContainerTest
|
||||
fun fileContentLargeFile_with_sudo_success() {
|
||||
// given
|
||||
val prov = defaultTestContainer()
|
||||
val filename = "largetestfile"
|
||||
val content = "012345äöüß".repeat(100000)
|
||||
|
||||
// when
|
||||
val res = prov.createFile(filename, content, overwriteIfExisting = true, sudo = true)
|
||||
val size = prov.fileSize(filename, sudo = true)
|
||||
val actualContent = prov.fileContentLargeFile(filename, chunkSize = 40000, sudo = true)
|
||||
|
||||
// then
|
||||
assertTrue(res.success)
|
||||
assertEquals(content, actualContent)
|
||||
assertEquals(1400000, size)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue
