From b5c823d6b519f67ace20883d42c6b5ec7a3bdb65 Mon Sep 17 00:00:00 2001 From: Yann Hamon Date: Sun, 26 Sep 2021 17:45:27 +0200 Subject: [PATCH] Also read up to 256MB from stdin --- acceptance.bats | 8 +++++++- pkg/resource/files.go | 2 +- pkg/resource/stream.go | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/acceptance.bats b/acceptance.bats index f69c897..f178a53 100755 --- a/acceptance.bats +++ b/acceptance.bats @@ -293,4 +293,10 @@ resetCacheFolder() { run bin/kubeconform -summary fixtures/valid_large.yaml [ "$status" -eq 0 ] [ "$output" = 'Summary: 100000 resources found in 1 file - Valid: 100000, Invalid: 0, Errors: 0, Skipped: 0' ] -} \ No newline at end of file +} + +@test "Should support very long streams from stdin" { + run bash -c "cat fixtures/valid_large.yaml | bin/kubeconform -summary" + [ "$status" -eq 0 ] + [ "$output" = 'Summary: 100000 resources found parsing stdin - Valid: 100000, Invalid: 0, Errors: 0, Skipped: 0' ] +} diff --git a/pkg/resource/files.go b/pkg/resource/files.go index a2f3bfa..f5eb893 100644 --- a/pkg/resource/files.go +++ b/pkg/resource/files.go @@ -91,7 +91,7 @@ func findResourcesInReader(p string, f io.Reader, resources chan<- Resource, err scanner := bufio.NewScanner(f) // We start with a buf that is 4MB, scanner will resize it up to 256MB if needed // https://github.com/golang/go/blob/aeea5bacbf79fb945edbeac6cd7630dd70c4d9ce/src/bufio/scan.go#L191 - scanner.Buffer(buf, 256*1024*1024*1024) + scanner.Buffer(buf, 256*1024*1024) scanner.Split(SplitYAMLDocument) nRes := 0 for scanner.Scan() { diff --git a/pkg/resource/stream.go b/pkg/resource/stream.go index 5cfbeb1..5e5bef4 100644 --- a/pkg/resource/stream.go +++ b/pkg/resource/stream.go @@ -48,9 +48,9 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource, go func() { scanner := bufio.NewScanner(r) - const maxResourceSize = 4 * 1024 * 1024 // 4MB ought to be enough for everybody + const maxResourceSize = 4 * 1024 * 1024 // Start with 4MB buf := make([]byte, maxResourceSize) - scanner.Buffer(buf, maxResourceSize) + scanner.Buffer(buf, 256*1024*1024) // Resize up to 256MB scanner.Split(SplitYAMLDocument) SCAN: