diff --git a/acceptance.bats b/acceptance.bats index 8f7ebd6..df19fd2 100755 --- a/acceptance.bats +++ b/acceptance.bats @@ -3,13 +3,13 @@ @test "Pass when parsing a valid Kubernetes config YAML file" { run bin/kubeconform -summary fixtures/valid.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when parsing a folder containing valid YAML files" { run bin/kubeconform -summary fixtures/folder [ "$status" -eq 0 ] - [ "$output" = "Summary: 7 resources found in 2 files - Valid: 7, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 7 resources found in 2 files - Valid: 7, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when parsing a valid Kubernetes config file with int_to_string vars" { @@ -21,7 +21,7 @@ @test "Pass when parsing a valid Kubernetes config JSON file" { run bin/kubeconform -kubernetes-version 1.17.1 -summary fixtures/valid.json [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when parsing a valid Kubernetes config YAML file with generate name" { @@ -45,13 +45,13 @@ @test "Pass when parsing a valid Kubernetes config file with null strings" { run bin/kubeconform -summary fixtures/null_string.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when parsing a multi-document config file" { run bin/kubeconform -summary fixtures/multi_valid.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Fail when parsing a multi-document config file with one invalid resource" { @@ -73,13 +73,13 @@ @test "Pass when parsing a blank config file" { run bin/kubeconform -summary fixtures/blank.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when parsing a blank config file with a comment" { run bin/kubeconform -summary fixtures/comment.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Fail when parsing a config with additional properties and strict set" { @@ -105,7 +105,7 @@ @test "Pass when parsing a config with additional properties" { run bin/kubeconform -summary fixtures/extra_property.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when using a valid, preset -schema-location" { @@ -141,13 +141,13 @@ @test "Pass when parsing a valid Kubernetes config YAML file on stdin" { run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary" [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Pass when parsing a valid Kubernetes config YAML file explicitly on stdin" { run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary" [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } @test "Fail when parsing an invalid Kubernetes config file on stdin" { @@ -170,5 +170,5 @@ @test "Ignores file that match the --ignore-filename-pattern given" { run bin/kubeconform -summary --ignore-filename-pattern 'crd' --ignore-filename-pattern '.*invalid.*' fixtures/multi_invalid.yaml fixtures/list_invalid.yaml fixtures/quantity.yaml fixtures/crd_schema.yaml [ "$status" -eq 0 ] - [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] + [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ] } diff --git a/pkg/output/text.go b/pkg/output/text.go index 6e16757..c38181a 100644 --- a/pkg/output/text.go +++ b/pkg/output/text.go @@ -82,9 +82,9 @@ func (o *texto) Flush() error { filesPlural = "s" } if o.isStdin { - _, err = fmt.Fprintf(o.w, "Summary: %d resource%s found parsing stdin - Valid: %d, Invalid: %d, Errors: %d Skipped: %d\n", nResources, resourcesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped) + _, err = fmt.Fprintf(o.w, "Summary: %d resource%s found parsing stdin - Valid: %d, Invalid: %d, Errors: %d, Skipped: %d\n", nResources, resourcesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped) } else { - _, err = fmt.Fprintf(o.w, "Summary: %d resource%s found in %d file%s - Valid: %d, Invalid: %d, Errors: %d Skipped: %d\n", nResources, resourcesPlural, nFiles, filesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped) + _, err = fmt.Fprintf(o.w, "Summary: %d resource%s found in %d file%s - Valid: %d, Invalid: %d, Errors: %d, Skipped: %d\n", nResources, resourcesPlural, nFiles, filesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped) } } diff --git a/pkg/output/text_test.go b/pkg/output/text_test.go index 6d83ec4..8976d85 100644 --- a/pkg/output/text_test.go +++ b/pkg/output/text_test.go @@ -43,7 +43,7 @@ metadata: Err: nil, }, }, - "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0\n", + "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0\n", }, { "a single deployment, verbose, with summary", @@ -65,7 +65,7 @@ metadata: }, }, `deployment.yml - Deployment my-app is valid -Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0 +Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0 `, }, } { diff --git a/pkg/resource/files.go b/pkg/resource/files.go index f953400..df55183 100644 --- a/pkg/resource/files.go +++ b/pkg/resource/files.go @@ -1,10 +1,9 @@ package resource import ( - "bytes" + "bufio" "context" "io" - "io/ioutil" "os" "path/filepath" "regexp" @@ -67,7 +66,7 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string return nil } - ignored, err := isIgnored(path, ignoreFilePatterns) + ignored, err := isIgnored(p, ignoreFilePatterns) if err != nil { return err } @@ -80,13 +79,18 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string return err } - b, err := ioutil.ReadAll(f) - if err != nil { - return err + scanner := bufio.NewScanner(f) + scanner.Split(SplitYAMLDocument) + nRes := 0 + for res := scanner.Scan(); res != false; res = scanner.Scan() { + resources <- Resource{Path: p, Bytes: scanner.Bytes()} + nRes++ } - - for _, r := range bytes.Split(b, []byte("---\n")) { - resources <- Resource{Path: p, Bytes: r} + if err := scanner.Err(); err != nil { + errors <- DiscoveryError{p, err} + } + if nRes == 0 { + resources <- Resource{Path: p, Bytes: []byte{}} } return nil diff --git a/pkg/resource/stream.go b/pkg/resource/stream.go index b0e87d9..ca50f80 100644 --- a/pkg/resource/stream.go +++ b/pkg/resource/stream.go @@ -2,27 +2,40 @@ package resource import ( "bufio" + "bytes" "context" "io" - "strings" ) -func yamlSplit(data []byte, atEOF bool) (advance int, token []byte, err error) { - // Return nothing if at end of file and no data passed +// Thank you https://github.com/helm/helm-classic/blob/master/codec/yaml.go#L90 +func SplitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) { + const yamlSeparator = "\n---" if atEOF && len(data) == 0 { return 0, nil, nil } - - if i := strings.Index(string(data), "---\n"); i >= 0 { - return i + 4, data[0:i], nil + sep := len([]byte(yamlSeparator)) + if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 { + // We have a potential document terminator + i += sep + after := data[i:] + if len(after) == 0 { + // we can't read any more characters + if atEOF { + return len(data), data[:len(data)-sep], nil + } + return 0, nil, nil + } + if j := bytes.IndexByte(after, '\n'); j >= 0 { + return i + j + 1, data[0 : i-sep], nil + } + return 0, nil, nil } - - // If at end of file with data return the data + // If we're at EOF, we have a final, non-terminated line. Return it. if atEOF { return len(data), data, nil } - - return + // Request more data. + return 0, nil, nil } // FromStream reads resources from a byte stream, usually here stdin @@ -38,7 +51,7 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource, go func() { scanner := bufio.NewScanner(r) - scanner.Split(yamlSplit) + scanner.Split(SplitYAMLDocument) for res := scanner.Scan(); res != false; res = scanner.Scan() { if stop == true { diff --git a/pkg/resource/stream_test.go b/pkg/resource/stream_test.go index 0da6ba3..5754571 100644 --- a/pkg/resource/stream_test.go +++ b/pkg/resource/stream_test.go @@ -29,7 +29,7 @@ func TestFromStream(t *testing.T) { { Have: have{ Path: "myfile", - Reader: strings.NewReader(` + Reader: strings.NewReader(`--- apiVersion: v1 kind: ReplicationController `), @@ -38,7 +38,7 @@ kind: ReplicationController Resources: []resource.Resource{ { Path: "myfile", - Bytes: []byte(` + Bytes: []byte(`--- apiVersion: v1 kind: ReplicationController `), @@ -52,20 +52,14 @@ kind: ReplicationController Path: "myfile", Reader: strings.NewReader(`apiVersion: v1 --- ---- apiVersion: v2 `), }, Want: want{ Resources: []resource.Resource{ - { - Path: "myfile", - Bytes: []byte(`apiVersion: v1 -`), - }, { Path: "myfile", - Bytes: []byte(``), + Bytes: []byte(`apiVersion: v1`), }, { Path: "myfile", @@ -94,14 +88,12 @@ kind: CronJob { Path: "myfile", Bytes: []byte(`apiVersion: v1 -kind: ReplicationController -`), +kind: ReplicationController`), }, { Path: "myfile", Bytes: []byte(`apiVersion: v1 -kind: Deployment -`), +kind: Deployment`), }, { Path: "myfile", @@ -128,8 +120,7 @@ kind: Deployment { Path: "myfile", Bytes: []byte(`apiVersion: v1 -kind: ReplicationController -`), +kind: ReplicationController`), }, { Path: "myfile", @@ -143,7 +134,7 @@ kind: Deployment }, } - for _, testCase := range testCases { + for testi, testCase := range testCases { ctx := context.Background() resChan, errChan := resource.FromStream(ctx, testCase.Have.Path, testCase.Have.Reader) var wg sync.WaitGroup @@ -156,11 +147,11 @@ kind: Deployment } if len(testCase.Want.Resources) != len(res) { - t.Errorf("expected %d resources, got %d", len(testCase.Want.Resources), len(res)) + t.Errorf("test %d - expected %d resources, got %d", testi, len(testCase.Want.Resources), len(res)) } for i, v := range res { if bytes.Compare(v.Bytes, testCase.Want.Resources[i].Bytes) != 0 { - t.Errorf("for resource %d, got '%s', expected '%s'", i, string(res[i].Bytes), string(testCase.Want.Resources[i].Bytes)) + t.Errorf("test %d - for resource %d, got '%s', expected '%s'", testi, i, string(res[i].Bytes), string(testCase.Want.Resources[i].Bytes)) } }