actions/kubeconform
9
0
Fork 0
mirror of https://github.com/yannh/kubeconform.git synced 2026-02-24 04:07:02 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions

better yaml splitting - not perfect but similar to k8s - fix path logging

Browse source
This commit is contained in:
Yann Hamon 2020-11-08 23:48:02 +01:00
parent d64a376779
commit 3a2d4705f5
6 changed files with 61 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
acceptance.bats
View file

@ -3,13 +3,13 @@
@test "Pass when parsing a valid Kubernetes config YAML file" { @test "Pass when parsing a valid Kubernetes config YAML file" {
run bin/kubeconform -summary fixtures/valid.yaml run bin/kubeconform -summary fixtures/valid.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when parsing a folder containing valid YAML files" { @test "Pass when parsing a folder containing valid YAML files" {
run bin/kubeconform -summary fixtures/folder run bin/kubeconform -summary fixtures/folder
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 7 resources found in 2 files - Valid: 7, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 7 resources found in 2 files - Valid: 7, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when parsing a valid Kubernetes config file with int_to_string vars" { @test "Pass when parsing a valid Kubernetes config file with int_to_string vars" {
@ -21,7 +21,7 @@
@test "Pass when parsing a valid Kubernetes config JSON file" { @test "Pass when parsing a valid Kubernetes config JSON file" {
run bin/kubeconform -kubernetes-version 1.17.1 -summary fixtures/valid.json run bin/kubeconform -kubernetes-version 1.17.1 -summary fixtures/valid.json
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when parsing a valid Kubernetes config YAML file with generate name" { @test "Pass when parsing a valid Kubernetes config YAML file with generate name" {
@ -45,13 +45,13 @@
@test "Pass when parsing a valid Kubernetes config file with null strings" { @test "Pass when parsing a valid Kubernetes config file with null strings" {
run bin/kubeconform -summary fixtures/null_string.yaml run bin/kubeconform -summary fixtures/null_string.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when parsing a multi-document config file" { @test "Pass when parsing a multi-document config file" {
run bin/kubeconform -summary fixtures/multi_valid.yaml run bin/kubeconform -summary fixtures/multi_valid.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Fail when parsing a multi-document config file with one invalid resource" { @test "Fail when parsing a multi-document config file with one invalid resource" {
@ -73,13 +73,13 @@
@test "Pass when parsing a blank config file" { @test "Pass when parsing a blank config file" {
run bin/kubeconform -summary fixtures/blank.yaml run bin/kubeconform -summary fixtures/blank.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when parsing a blank config file with a comment" { @test "Pass when parsing a blank config file with a comment" {
run bin/kubeconform -summary fixtures/comment.yaml run bin/kubeconform -summary fixtures/comment.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Fail when parsing a config with additional properties and strict set" { @test "Fail when parsing a config with additional properties and strict set" {
@ -105,7 +105,7 @@
@test "Pass when parsing a config with additional properties" { @test "Pass when parsing a config with additional properties" {
run bin/kubeconform -summary fixtures/extra_property.yaml run bin/kubeconform -summary fixtures/extra_property.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when using a valid, preset -schema-location" { @test "Pass when using a valid, preset -schema-location" {
@ -141,13 +141,13 @@
@test "Pass when parsing a valid Kubernetes config YAML file on stdin" { @test "Pass when parsing a valid Kubernetes config YAML file on stdin" {
run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary" run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary"
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Pass when parsing a valid Kubernetes config YAML file explicitly on stdin" { @test "Pass when parsing a valid Kubernetes config YAML file explicitly on stdin" {
run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary" run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary"
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }
@test "Fail when parsing an invalid Kubernetes config file on stdin" { @test "Fail when parsing an invalid Kubernetes config file on stdin" {
@ -170,5 +170,5 @@
@test "Ignores file that match the --ignore-filename-pattern given" { @test "Ignores file that match the --ignore-filename-pattern given" {
run bin/kubeconform -summary --ignore-filename-pattern 'crd' --ignore-filename-pattern '.*invalid.*' fixtures/multi_invalid.yaml fixtures/list_invalid.yaml fixtures/quantity.yaml fixtures/crd_schema.yaml run bin/kubeconform -summary --ignore-filename-pattern 'crd' --ignore-filename-pattern '.*invalid.*' fixtures/multi_invalid.yaml fixtures/list_invalid.yaml fixtures/quantity.yaml fixtures/crd_schema.yaml
[ "$status" -eq 0 ] [ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ] [ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
} }

4
pkg/output/text.go
View file

@ -82,9 +82,9 @@ func (o *texto) Flush() error {
filesPlural = "s" filesPlural = "s"
} }
if o.isStdin { if o.isStdin {
_, err = fmt.Fprintf(o.w, "Summary: %d resource%s found parsing stdin - Valid: %d, Invalid: %d, Errors: %d Skipped: %d\n", nResources, resourcesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped) _, err = fmt.Fprintf(o.w, "Summary: %d resource%s found parsing stdin - Valid: %d, Invalid: %d, Errors: %d, Skipped: %d\n", nResources, resourcesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped)
} else { } else {
_, err = fmt.Fprintf(o.w, "Summary: %d resource%s found in %d file%s - Valid: %d, Invalid: %d, Errors: %d Skipped: %d\n", nResources, resourcesPlural, nFiles, filesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped) _, err = fmt.Fprintf(o.w, "Summary: %d resource%s found in %d file%s - Valid: %d, Invalid: %d, Errors: %d, Skipped: %d\n", nResources, resourcesPlural, nFiles, filesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped)
} }
} }

4
pkg/output/text_test.go
View file

@ -43,7 +43,7 @@ metadata:
Err: nil, Err: nil,
}, },
}, },
"Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0\n", "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0\n",
}, },
{ {
"a single deployment, verbose, with summary", "a single deployment, verbose, with summary",
@ -65,7 +65,7 @@ metadata:
}, },
}, },
`deployment.yml - Deployment my-app is valid `deployment.yml - Deployment my-app is valid
Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0 Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0
`, `,
}, },
} { } {

22
pkg/resource/files.go
View file

@ -1,10 +1,9 @@
package resource package resource
import ( import (
"bytes" "bufio"
"context" "context"
"io" "io"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
@ -67,7 +66,7 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string
return nil return nil
} }
ignored, err := isIgnored(path, ignoreFilePatterns) ignored, err := isIgnored(p, ignoreFilePatterns)
if err != nil { if err != nil {
return err return err
} }
@ -80,13 +79,18 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string
return err return err
} }
b, err := ioutil.ReadAll(f) scanner := bufio.NewScanner(f)
if err != nil { scanner.Split(SplitYAMLDocument)
return err nRes := 0
for res := scanner.Scan(); res != false; res = scanner.Scan() {
resources <- Resource{Path: p, Bytes: scanner.Bytes()}
nRes++
} }
if err := scanner.Err(); err != nil {
for _, r := range bytes.Split(b, []byte("---\n")) { errors <- DiscoveryError{p, err}
resources <- Resource{Path: p, Bytes: r} }
if nRes == 0 {
resources <- Resource{Path: p, Bytes: []byte{}}
} }
return nil return nil

35
pkg/resource/stream.go
View file

@ -2,27 +2,40 @@ package resource
import ( import (
"bufio" "bufio"
"bytes"
"context" "context"
"io" "io"
"strings"
) )
func yamlSplit(data []byte, atEOF bool) (advance int, token []byte, err error) { // Thank you https://github.com/helm/helm-classic/blob/master/codec/yaml.go#L90
// Return nothing if at end of file and no data passed func SplitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
const yamlSeparator = "\n---"
if atEOF && len(data) == 0 { if atEOF && len(data) == 0 {
return 0, nil, nil return 0, nil, nil
} }
sep := len([]byte(yamlSeparator))
if i := strings.Index(string(data), "---\n"); i >= 0 { if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
return i + 4, data[0:i], nil // We have a potential document terminator
i += sep
after := data[i:]
if len(after) == 0 {
// we can't read any more characters
if atEOF {
return len(data), data[:len(data)-sep], nil
}
return 0, nil, nil
}
if j := bytes.IndexByte(after, '\n'); j >= 0 {
return i + j + 1, data[0 : i-sep], nil
}
return 0, nil, nil
} }
// If we're at EOF, we have a final, non-terminated line. Return it.
// If at end of file with data return the data
if atEOF { if atEOF {
return len(data), data, nil return len(data), data, nil
} }
// Request more data.
return return 0, nil, nil
} }
// FromStream reads resources from a byte stream, usually here stdin // FromStream reads resources from a byte stream, usually here stdin
@ -38,7 +51,7 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource,
go func() { go func() {
scanner := bufio.NewScanner(r) scanner := bufio.NewScanner(r)
scanner.Split(yamlSplit) scanner.Split(SplitYAMLDocument)
for res := scanner.Scan(); res != false; res = scanner.Scan() { for res := scanner.Scan(); res != false; res = scanner.Scan() {
if stop == true { if stop == true {

27
pkg/resource/stream_test.go
View file

@ -29,7 +29,7 @@ func TestFromStream(t *testing.T) {
{ {
Have: have{ Have: have{
Path: "myfile", Path: "myfile",
Reader: strings.NewReader(` Reader: strings.NewReader(`---
apiVersion: v1 apiVersion: v1
kind: ReplicationController kind: ReplicationController
`), `),
@ -38,7 +38,7 @@ kind: ReplicationController
Resources: []resource.Resource{ Resources: []resource.Resource{
{ {
Path: "myfile", Path: "myfile",
Bytes: []byte(` Bytes: []byte(`---
apiVersion: v1 apiVersion: v1
kind: ReplicationController kind: ReplicationController
`), `),
@ -52,20 +52,14 @@ kind: ReplicationController
Path: "myfile", Path: "myfile",
Reader: strings.NewReader(`apiVersion: v1 Reader: strings.NewReader(`apiVersion: v1
--- ---
---
apiVersion: v2 apiVersion: v2
`), `),
}, },
Want: want{ Want: want{
Resources: []resource.Resource{ Resources: []resource.Resource{
{
Path: "myfile",
Bytes: []byte(`apiVersion: v1
`),
},
{ {
Path: "myfile", Path: "myfile",
Bytes: []byte(``), Bytes: []byte(`apiVersion: v1`),
}, },
{ {
Path: "myfile", Path: "myfile",
@ -94,14 +88,12 @@ kind: CronJob
{ {
Path: "myfile", Path: "myfile",
Bytes: []byte(`apiVersion: v1 Bytes: []byte(`apiVersion: v1
kind: ReplicationController kind: ReplicationController`),
`),
}, },
{ {
Path: "myfile", Path: "myfile",
Bytes: []byte(`apiVersion: v1 Bytes: []byte(`apiVersion: v1
kind: Deployment kind: Deployment`),
`),
}, },
{ {
Path: "myfile", Path: "myfile",
@ -128,8 +120,7 @@ kind: Deployment
{ {
Path: "myfile", Path: "myfile",
Bytes: []byte(`apiVersion: v1 Bytes: []byte(`apiVersion: v1
kind: ReplicationController kind: ReplicationController`),
`),
}, },
{ {
Path: "myfile", Path: "myfile",
@ -143,7 +134,7 @@ kind: Deployment
}, },
} }
for _, testCase := range testCases { for testi, testCase := range testCases {
ctx := context.Background() ctx := context.Background()
resChan, errChan := resource.FromStream(ctx, testCase.Have.Path, testCase.Have.Reader) resChan, errChan := resource.FromStream(ctx, testCase.Have.Path, testCase.Have.Reader)
var wg sync.WaitGroup var wg sync.WaitGroup
@ -156,11 +147,11 @@ kind: Deployment
} }
if len(testCase.Want.Resources) != len(res) { if len(testCase.Want.Resources) != len(res) {
t.Errorf("expected %d resources, got %d", len(testCase.Want.Resources), len(res)) t.Errorf("test %d - expected %d resources, got %d", testi, len(testCase.Want.Resources), len(res))
} }
for i, v := range res { for i, v := range res {
if bytes.Compare(v.Bytes, testCase.Want.Resources[i].Bytes) != 0 { if bytes.Compare(v.Bytes, testCase.Want.Resources[i].Bytes) != 0 {
t.Errorf("for resource %d, got '%s', expected '%s'", i, string(res[i].Bytes), string(testCase.Want.Resources[i].Bytes)) t.Errorf("test %d - for resource %d, got '%s', expected '%s'", testi, i, string(res[i].Bytes), string(testCase.Want.Resources[i].Bytes))
} }
} }