diff --git a/cmd/kubeconform/main.go b/cmd/kubeconform/main.go index d1ae4b3..e047085 100644 --- a/cmd/kubeconform/main.go +++ b/cmd/kubeconform/main.go @@ -102,7 +102,7 @@ func kubeconform(cfg config.Config) int { if useStdin { resourcesChan, errors = resource.FromStream(ctx, "stdin", os.Stdin) } else { - resourcesChan, errors = resource.FromFiles(ctx, cfg.Files, cfg.IgnoreFilenamePatterns) + resourcesChan, errors = resource.FromFiles(ctx, cfg.Files, cfg.IgnoreFilenamePatterns, cfg.NumberOfWorkers) } // Process discovered resources across multiple workers diff --git a/pkg/resource/files.go b/pkg/resource/files.go index 37330d1..795fc46 100644 --- a/pkg/resource/files.go +++ b/pkg/resource/files.go @@ -8,6 +8,7 @@ import ( "path/filepath" "regexp" "strings" + "sync" ) func isYAMLFile(info os.FileInfo) bool { @@ -124,19 +125,33 @@ func findResourcesInFile(p string, resources chan<- Resource, errors chan<- erro findResourcesInReader(p, f, resources, errors, buf) } -func FromFiles(ctx context.Context, paths []string, ignoreFilePatterns []string) (<-chan Resource, <-chan error) { +func FromFiles(ctx context.Context, paths []string, ignoreFilePatterns []string, nReaders int) (<-chan Resource, <-chan error) { resources := make(chan Resource) files, errors := findFilesInFolders(ctx, paths, ignoreFilePatterns) + // Read and split files in parallel: file I/O + YAML scanning is the bottleneck + // when validating large trees of small manifests. + if nReaders < 1 { + nReaders = 1 + } + + wg := sync.WaitGroup{} + for i := 0; i < nReaders; i++ { + wg.Add(1) + go func() { + defer wg.Done() + initialBufSize := 4 * 1024 * 1024 + buf := make([]byte, initialBufSize) // per-reader buffer, reused across files + + for p := range files { + findResourcesInFile(p, resources, errors, buf) + } + }() + } + go func() { - initialBufSize := 4 * 1024 * 1024 // This is the initial size - scanner will resize if needed - buf := make([]byte, initialBufSize) // We reuse the same buffer to avoid multiple large memory allocations - - for p := range files { - findResourcesInFile(p, resources, errors, buf) - } - + wg.Wait() close(errors) close(resources) }()