Small clean-up

This commit is contained in:
Yann Hamon 2021-09-26 22:55:01 +02:00
parent b571b18f8d
commit ab4745ddf0
3 changed files with 10 additions and 7 deletions

View file

@ -88,10 +88,11 @@ func findFilesInFolders(ctx context.Context, paths []string, ignoreFilePatterns
}
func findResourcesInReader(p string, f io.Reader, resources chan<- Resource, errors chan<- error, buf []byte) {
maxBufSize := 256 * 1024 * 1024
scanner := bufio.NewScanner(f)
// We start with a buf that is 4MB, scanner will resize it up to 256MB if needed
// https://github.com/golang/go/blob/aeea5bacbf79fb945edbeac6cd7630dd70c4d9ce/src/bufio/scan.go#L191
scanner.Buffer(buf, 256*1024*1024)
scanner.Buffer(buf, maxBufSize)
scanner.Split(SplitYAMLDocument)
nRes := 0
for scanner.Scan() {
@ -129,8 +130,8 @@ func FromFiles(ctx context.Context, paths []string, ignoreFilePatterns []string)
files, errors := findFilesInFolders(ctx, paths, ignoreFilePatterns)
go func() {
maxResourceSize := 4 * 1024 * 1024 // This is the initial size - scanner will resize if needed
buf := make([]byte, maxResourceSize) // We reuse the same buffer to avoid multiple large memory allocations
initialBufSize := 4 * 1024 * 1024 // This is the initial size - scanner will resize if needed
buf := make([]byte, initialBufSize) // We reuse the same buffer to avoid multiple large memory allocations
for p := range files {
findResourcesInFile(p, resources, errors, buf)

View file

@ -47,10 +47,12 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource,
errors := make(chan error)
go func() {
const initialBufSize = 4 * 1024 * 1024 // Start with 4MB
const maxBufSize = 256 * 1024 * 1024 // Start with 4MB
scanner := bufio.NewScanner(r)
const maxResourceSize = 4 * 1024 * 1024 // Start with 4MB
buf := make([]byte, maxResourceSize)
scanner.Buffer(buf, 256*1024*1024) // Resize up to 256MB
buf := make([]byte, initialBufSize)
scanner.Buffer(buf, maxBufSize) // Resize up to 256MB
scanner.Split(SplitYAMLDocument)
SCAN: