actions/kubeconform
9
0
Fork 0
mirror of https://github.com/yannh/kubeconform.git synced 2026-02-11 14:09:21 +00:00
Code Issues Projects Releases 1 Packages Wiki Activity Actions

better yaml splitting - not perfect but similar to k8s - fix path logging

Browse source
This commit is contained in:
Yann Hamon 2020-11-08 23:48:02 +01:00
parent d64a376779
commit 3a2d4705f5
6 changed files with 61 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
acceptance.bats
View file

@ -3,13 +3,13 @@
@test "Pass when parsing a valid Kubernetes config YAML file" {
run bin/kubeconform -summary fixtures/valid.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when parsing a folder containing valid YAML files" {
run bin/kubeconform -summary fixtures/folder
[ "$status" -eq 0 ]
[ "$output" = "Summary: 7 resources found in 2 files - Valid: 7, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 7 resources found in 2 files - Valid: 7, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when parsing a valid Kubernetes config file with int_to_string vars" {
@ -21,7 +21,7 @@
@test "Pass when parsing a valid Kubernetes config JSON file" {
run bin/kubeconform -kubernetes-version 1.17.1 -summary fixtures/valid.json
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when parsing a valid Kubernetes config YAML file with generate name" {
@ -45,13 +45,13 @@
@test "Pass when parsing a valid Kubernetes config file with null strings" {
run bin/kubeconform -summary fixtures/null_string.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when parsing a multi-document config file" {
run bin/kubeconform -summary fixtures/multi_valid.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 6 resources found in 1 file - Valid: 6, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Fail when parsing a multi-document config file with one invalid resource" {
@ -73,13 +73,13 @@
@test "Pass when parsing a blank config file" {
run bin/kubeconform -summary fixtures/blank.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when parsing a blank config file with a comment" {
run bin/kubeconform -summary fixtures/comment.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 0 resource found in 1 file - Valid: 0, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Fail when parsing a config with additional properties and strict set" {
@ -105,7 +105,7 @@
@test "Pass when parsing a config with additional properties" {
run bin/kubeconform -summary fixtures/extra_property.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when using a valid, preset -schema-location" {
@ -141,13 +141,13 @@
@test "Pass when parsing a valid Kubernetes config YAML file on stdin" {
run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary"
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Pass when parsing a valid Kubernetes config YAML file explicitly on stdin" {
run bash -c "cat fixtures/valid.yaml | bin/kubeconform -summary"
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found parsing stdin - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}
@test "Fail when parsing an invalid Kubernetes config file on stdin" {
@ -170,5 +170,5 @@
@test "Ignores file that match the --ignore-filename-pattern given" {
run bin/kubeconform -summary --ignore-filename-pattern 'crd' --ignore-filename-pattern '.*invalid.*' fixtures/multi_invalid.yaml fixtures/list_invalid.yaml fixtures/quantity.yaml fixtures/crd_schema.yaml
[ "$status" -eq 0 ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0" ]
[ "$output" = "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0" ]
}

4
pkg/output/text.go
View file

@ -82,9 +82,9 @@ func (o *texto) Flush() error {
filesPlural = "s"
}
if o.isStdin {
_, err = fmt.Fprintf(o.w, "Summary: %d resource%s found parsing stdin - Valid: %d, Invalid: %d, Errors: %d Skipped: %d\n", nResources, resourcesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped)
_, err = fmt.Fprintf(o.w, "Summary: %d resource%s found parsing stdin - Valid: %d, Invalid: %d, Errors: %d, Skipped: %d\n", nResources, resourcesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped)
} else {
_, err = fmt.Fprintf(o.w, "Summary: %d resource%s found in %d file%s - Valid: %d, Invalid: %d, Errors: %d Skipped: %d\n", nResources, resourcesPlural, nFiles, filesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped)
_, err = fmt.Fprintf(o.w, "Summary: %d resource%s found in %d file%s - Valid: %d, Invalid: %d, Errors: %d, Skipped: %d\n", nResources, resourcesPlural, nFiles, filesPlural, o.nValid, o.nInvalid, o.nErrors, o.nSkipped)
}
}

4
pkg/output/text_test.go
View file

@ -43,7 +43,7 @@ metadata:
Err: nil,
},
},
"Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0\n",
"Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0\n",
},
{
"a single deployment, verbose, with summary",
@ -65,7 +65,7 @@ metadata:
},
},
`deployment.yml - Deployment my-app is valid
Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0
Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0, Skipped: 0
`,
},
} {

22
pkg/resource/files.go
View file

@ -1,10 +1,9 @@
package resource
import (
"bytes"
"bufio"
"context"
"io"
"io/ioutil"
"os"
"path/filepath"
"regexp"
@ -67,7 +66,7 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string
return nil
}
ignored, err := isIgnored(path, ignoreFilePatterns)
ignored, err := isIgnored(p, ignoreFilePatterns)
if err != nil {
return err
}
@ -80,13 +79,18 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string
return err
}
b, err := ioutil.ReadAll(f)
if err != nil {
return err
scanner := bufio.NewScanner(f)
scanner.Split(SplitYAMLDocument)
nRes := 0
for res := scanner.Scan(); res != false; res = scanner.Scan() {
resources <- Resource{Path: p, Bytes: scanner.Bytes()}
nRes++
}
for _, r := range bytes.Split(b, []byte("---\n")) {
resources <- Resource{Path: p, Bytes: r}
if err := scanner.Err(); err != nil {
errors <- DiscoveryError{p, err}
}
if nRes == 0 {
resources <- Resource{Path: p, Bytes: []byte{}}
}
return nil

35
pkg/resource/stream.go
View file

@ -2,27 +2,40 @@ package resource
import (
"bufio"
"bytes"
"context"
"io"
"strings"
)
func yamlSplit(data []byte, atEOF bool) (advance int, token []byte, err error) {
// Return nothing if at end of file and no data passed
// Thank you https://github.com/helm/helm-classic/blob/master/codec/yaml.go#L90
func SplitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
const yamlSeparator = "\n---"
if atEOF && len(data) == 0 {
return 0, nil, nil
}
if i := strings.Index(string(data), "---\n"); i >= 0 {
return i + 4, data[0:i], nil
sep := len([]byte(yamlSeparator))
if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
// We have a potential document terminator
i += sep
after := data[i:]
if len(after) == 0 {
// we can't read any more characters
if atEOF {
return len(data), data[:len(data)-sep], nil
}
return 0, nil, nil
}
if j := bytes.IndexByte(after, '\n'); j >= 0 {
return i + j + 1, data[0 : i-sep], nil
}
return 0, nil, nil
}
// If at end of file with data return the data
// If we're at EOF, we have a final, non-terminated line. Return it.
if atEOF {
return len(data), data, nil
}
return
// Request more data.
return 0, nil, nil
}
// FromStream reads resources from a byte stream, usually here stdin
@ -38,7 +51,7 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource,
go func() {
scanner := bufio.NewScanner(r)
scanner.Split(yamlSplit)
scanner.Split(SplitYAMLDocument)
for res := scanner.Scan(); res != false; res = scanner.Scan() {
if stop == true {

27
pkg/resource/stream_test.go
View file

@ -29,7 +29,7 @@ func TestFromStream(t *testing.T) {
{
Have: have{
Path: "myfile",
Reader: strings.NewReader(`
Reader: strings.NewReader(`---
apiVersion: v1
kind: ReplicationController
`),
@ -38,7 +38,7 @@ kind: ReplicationController
Resources: []resource.Resource{
{
Path: "myfile",
Bytes: []byte(`
Bytes: []byte(`---
apiVersion: v1
kind: ReplicationController
`),
@ -52,20 +52,14 @@ kind: ReplicationController
Path: "myfile",
Reader: strings.NewReader(`apiVersion: v1
---
---
apiVersion: v2
`),
},
Want: want{
Resources: []resource.Resource{
{
Path: "myfile",
Bytes: []byte(`apiVersion: v1
`),
},
{
Path: "myfile",
Bytes: []byte(``),
Bytes: []byte(`apiVersion: v1`),
},
{
Path: "myfile",
@ -94,14 +88,12 @@ kind: CronJob
{
Path: "myfile",
Bytes: []byte(`apiVersion: v1
kind: ReplicationController
`),
kind: ReplicationController`),
},
{
Path: "myfile",
Bytes: []byte(`apiVersion: v1
kind: Deployment
`),
kind: Deployment`),
},
{
Path: "myfile",
@ -128,8 +120,7 @@ kind: Deployment
{
Path: "myfile",
Bytes: []byte(`apiVersion: v1
kind: ReplicationController
`),
kind: ReplicationController`),
},
{
Path: "myfile",
@ -143,7 +134,7 @@ kind: Deployment
},
}
for _, testCase := range testCases {
for testi, testCase := range testCases {
ctx := context.Background()
resChan, errChan := resource.FromStream(ctx, testCase.Have.Path, testCase.Have.Reader)
var wg sync.WaitGroup
@ -156,11 +147,11 @@ kind: Deployment
}
if len(testCase.Want.Resources) != len(res) {
t.Errorf("expected %d resources, got %d", len(testCase.Want.Resources), len(res))
t.Errorf("test %d - expected %d resources, got %d", testi, len(testCase.Want.Resources), len(res))
}
for i, v := range res {
if bytes.Compare(v.Bytes, testCase.Want.Resources[i].Bytes) != 0 {
t.Errorf("for resource %d, got '%s', expected '%s'", i, string(res[i].Bytes), string(testCase.Want.Resources[i].Bytes))
t.Errorf("test %d - for resource %d, got '%s', expected '%s'", testi, i, string(res[i].Bytes), string(testCase.Want.Resources[i].Bytes))
}
}