diff --git a/acceptance.bats b/acceptance.bats index 1a976fe..6b9917b 100755 --- a/acceptance.bats +++ b/acceptance.bats @@ -244,7 +244,7 @@ resetCacheFolder() { [ "${lines[2]}" == '1..1' ] } -@test "Pass when parsing a file containing multiple a List" { +@test "Pass when parsing a file containing a List" { run bin/kubeconform -summary fixtures/list_valid.yaml [ "$status" -eq 0 ] [ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0, Skipped: 0" ] diff --git a/pkg/resource/files.go b/pkg/resource/files.go index f5eb893..37330d1 100644 --- a/pkg/resource/files.go +++ b/pkg/resource/files.go @@ -88,10 +88,11 @@ func findFilesInFolders(ctx context.Context, paths []string, ignoreFilePatterns } func findResourcesInReader(p string, f io.Reader, resources chan<- Resource, errors chan<- error, buf []byte) { + maxBufSize := 256 * 1024 * 1024 scanner := bufio.NewScanner(f) // We start with a buf that is 4MB, scanner will resize it up to 256MB if needed // https://github.com/golang/go/blob/aeea5bacbf79fb945edbeac6cd7630dd70c4d9ce/src/bufio/scan.go#L191 - scanner.Buffer(buf, 256*1024*1024) + scanner.Buffer(buf, maxBufSize) scanner.Split(SplitYAMLDocument) nRes := 0 for scanner.Scan() { @@ -129,8 +130,8 @@ func FromFiles(ctx context.Context, paths []string, ignoreFilePatterns []string) files, errors := findFilesInFolders(ctx, paths, ignoreFilePatterns) go func() { - maxResourceSize := 4 * 1024 * 1024 // This is the initial size - scanner will resize if needed - buf := make([]byte, maxResourceSize) // We reuse the same buffer to avoid multiple large memory allocations + initialBufSize := 4 * 1024 * 1024 // This is the initial size - scanner will resize if needed + buf := make([]byte, initialBufSize) // We reuse the same buffer to avoid multiple large memory allocations for p := range files { findResourcesInFile(p, resources, errors, buf) diff --git a/pkg/resource/stream.go b/pkg/resource/stream.go index 5e5bef4..8186d53 100644 --- a/pkg/resource/stream.go +++ b/pkg/resource/stream.go @@ -47,10 +47,12 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource, errors := make(chan error) go func() { + const initialBufSize = 4 * 1024 * 1024 // Start with 4MB + const maxBufSize = 256 * 1024 * 1024 // Start with 4MB + scanner := bufio.NewScanner(r) - const maxResourceSize = 4 * 1024 * 1024 // Start with 4MB - buf := make([]byte, maxResourceSize) - scanner.Buffer(buf, 256*1024*1024) // Resize up to 256MB + buf := make([]byte, initialBufSize) + scanner.Buffer(buf, maxBufSize) // Resize up to 256MB scanner.Split(SplitYAMLDocument) SCAN: