actions/kubeconform
9
0
Fork 0
mirror of https://github.com/yannh/kubeconform.git synced 2026-02-11 14:09:21 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions

Merge pull request #76 from yannh/support-for-larger-files

Browse source 
Support for larger files
This commit is contained in:
Yann Hamon 2021-09-26 17:49:44 +02:00 committed by GitHub
commit 563106ede1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 600022 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
acceptance.bats
View file

@ -288,3 +288,15 @@ resetCacheFolder() {
[ "$status" -eq 1 ]
[[ "$output" == *"proxyconnect tcp: dial tcp 127.0.0.1:1234: connect: connection refused"* ]]
}
@test "Should support large files" {
run bin/kubeconform -summary fixtures/valid_large.yaml
[ "$status" -eq 0 ]
[ "$output" = 'Summary: 100000 resources found in 1 file - Valid: 100000, Invalid: 0, Errors: 0, Skipped: 0' ]
}
@test "Should support very long streams from stdin" {
run bash -c "cat fixtures/valid_large.yaml | bin/kubeconform -summary"
[ "$status" -eq 0 ]
[ "$output" = 'Summary: 100000 resources found parsing stdin - Valid: 100000, Invalid: 0, Errors: 0, Skipped: 0' ]
}

600003
fixtures/valid_large.yaml Normal file
View file

File diff suppressed because it is too large Load diff

8
pkg/resource/files.go
View file

@ -89,7 +89,9 @@ func findFilesInFolders(ctx context.Context, paths []string, ignoreFilePatterns
func findResourcesInReader(p string, f io.Reader, resources chan<- Resource, errors chan<- error, buf []byte) {
scanner := bufio.NewScanner(f)
scanner.Buffer(buf, len(buf))
// We start with a buf that is 4MB, scanner will resize it up to 256MB if needed
// https://github.com/golang/go/blob/aeea5bacbf79fb945edbeac6cd7630dd70c4d9ce/src/bufio/scan.go#L191
scanner.Buffer(buf, 256*1024*1024)
scanner.Split(SplitYAMLDocument)
nRes := 0
for scanner.Scan() {
@ -127,8 +129,8 @@ func FromFiles(ctx context.Context, paths []string, ignoreFilePatterns []string)
files, errors := findFilesInFolders(ctx, paths, ignoreFilePatterns)
go func() {
maxResourceSize := 4 * 1024 * 1024 // 4MB ought to be enough for everybody
buf := make([]byte, maxResourceSize) // We reuse this to avoid multiple large memory allocations
maxResourceSize := 4 * 1024 * 1024 // This is the initial size - scanner will resize if needed
buf := make([]byte, maxResourceSize) // We reuse the same buffer to avoid multiple large memory allocations
for p := range files {
findResourcesInFile(p, resources, errors, buf)

4
pkg/resource/stream.go
View file

@ -48,9 +48,9 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource,
go func() {
scanner := bufio.NewScanner(r)
const maxResourceSize = 4 * 1024 * 1024 // 4MB ought to be enough for everybody
const maxResourceSize = 4 * 1024 * 1024 // Start with 4MB
buf := make([]byte, maxResourceSize)
scanner.Buffer(buf, maxResourceSize)
scanner.Buffer(buf, 256*1024*1024) // Resize up to 256MB
scanner.Split(SplitYAMLDocument)
SCAN: