do not buffer all of stdin before starting to process resources

pkg/resource/stream.go

@@ -1,12 +1,30 @@
 package resource
 
 import (
-	"bytes"
+	"bufio"
 	"context"
 	"io"
-	"io/ioutil"
+	"strings"
 )
 
+func yamlSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	// Return nothing if at end of file and no data passed
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	if i := strings.Index(string(data), "---\n"); i >= 0 {
+		return i + 4, data[0:i], nil
+	}
+
+	// If at end of file with data return the data
+	if atEOF {
+		return len(data), data, nil
+	}
+
+	return
+}
+
 // FromStream reads resources from a byte stream, usually here stdin
 func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource, <-chan error) {
 	resources := make(chan Resource)
@@ -19,17 +37,14 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource,
 	}()
 
 	go func() {
-		data, err := ioutil.ReadAll(r)
+		scanner := bufio.NewScanner(r)
-		if err != nil {
+		scanner.Split(yamlSplit)
-			errors <- DiscoveryError{path, err}
-		}
 
-		rawResources := bytes.Split(data, []byte("---\n"))
+		for res := scanner.Scan(); res != false; res = scanner.Scan() {
-		for _, rawResource := range rawResources {
 			if stop == true {
 				break
 			}
-			resources <- Resource{Path: path, Bytes: rawResource}
+			resources <- Resource{Path: path, Bytes: scanner.Bytes()}
 		}
 
 		close(resources)