From 1bc92832402817ada2385deab230c038f27bdb37 Mon Sep 17 00:00:00 2001 From: Yann Hamon Date: Sun, 1 Nov 2020 13:00:02 +0100 Subject: [PATCH] refactor --- cmd/kubeconform/main.go | 284 ++++++++++++--------------- pkg/fsutils/findyaml.go | 14 +- pkg/output/json.go | 36 ++-- pkg/output/json_test.go | 114 ----------- pkg/output/output.go | 30 +-- pkg/output/text.go | 31 +-- pkg/output/text_test.go | 86 -------- pkg/registry/kubernetesjsonschema.go | 2 +- pkg/registry/local.go | 2 +- pkg/registry/registry.go | 12 ++ pkg/resource/signature.go | 19 +- pkg/resource/signature_test.go | 4 +- pkg/resource/stream.go | 22 +++ pkg/validator/validator.go | 37 ++-- pkg/validator/validator_test.go | 6 +- 15 files changed, 241 insertions(+), 458 deletions(-) delete mode 100644 pkg/output/json_test.go delete mode 100644 pkg/output/text_test.go create mode 100644 pkg/resource/stream.go diff --git a/cmd/kubeconform/main.go b/cmd/kubeconform/main.go index 8383b52..14e4b26 100644 --- a/cmd/kubeconform/main.go +++ b/cmd/kubeconform/main.go @@ -1,17 +1,13 @@ package main import ( - "bytes" "fmt" "github.com/xeipuuv/gojsonschema" - "github.com/yannh/kubeconform/pkg/config" - "io" - "io/ioutil" "os" - "strings" "sync" "github.com/yannh/kubeconform/pkg/cache" + "github.com/yannh/kubeconform/pkg/config" "github.com/yannh/kubeconform/pkg/fsutils" "github.com/yannh/kubeconform/pkg/output" "github.com/yannh/kubeconform/pkg/registry" @@ -19,23 +15,6 @@ import ( "github.com/yannh/kubeconform/pkg/validator" ) -type validationResult struct { - filename, kind, version, Name string - err error - skipped bool -} - -func resourcesFromReader(r io.Reader) ([][]byte, error) { - data, err := ioutil.ReadAll(r) - if err != nil { - return [][]byte{}, err - } - - resources := bytes.Split(data, []byte("---\n")) - - return resources, nil -} - func downloadSchema(registries []registry.Registry, kind, version, k8sVersion string) (*gojsonschema.Schema, error) { var err error var schemaBytes []byte @@ -57,99 +36,82 @@ func downloadSchema(registries []registry.Registry, kind, version, k8sVersion st return nil, nil // No schema found - we don't consider it an error, resource will be skipped } -// filter returns true if the file should be skipped -// Returning an array, this Reader might container multiple resources -func ValidateStream(r io.Reader, regs []registry.Registry, k8sVersion string, c *cache.SchemaCache, skip func(signature resource.Signature) bool, ignoreMissingSchemas bool) []validationResult { - rawResources, err := resourcesFromReader(r) - if err != nil { - return []validationResult{{err: fmt.Errorf("failed reading file: %s", err)}} - } - - validationResults := []validationResult{} - if len(rawResources) == 0 { - // In case a file has no resources, we want to capture that the file was parsed - and therefore send a message with an empty resource and no error - validationResults = append(validationResults, validationResult{kind: "", version: "", Name: "", err: nil, skipped: false}) - } - - for _, rawResource := range rawResources { - var sig resource.Signature - if sig, err = resource.SignatureFromBytes(rawResource); err != nil { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: fmt.Errorf("error while parsing: %s", err)}) - continue - } - - if sig.Kind == "" { - validationResults = append(validationResults, validationResult{kind: "", version: "", Name: "", err: nil, skipped: false}) - continue // We skip resoures that don't have a Kind defined - } - - if skip(sig) { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: nil, skipped: true}) - continue - } - - ok := false - var schema *gojsonschema.Schema - cacheKey := "" - - if c != nil { - cacheKey = cache.Key(sig.Kind, sig.Version, k8sVersion) - schema, ok = c.Get(cacheKey) - } - - if !ok { - schema, err = downloadSchema(regs, sig.Kind, sig.Version, k8sVersion) +func ValidateResources(resources <-chan []resource.Resource, validationResults chan<- validator.Result, regs []registry.Registry, k8sVersion string, c *cache.SchemaCache, skip func(signature resource.Signature) bool, ignoreMissingSchemas bool) { + for resBatch := range resources { + for _, res := range resBatch { + sig, err := res.Signature() if err != nil { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: err, skipped: false}) + validationResults <- validator.Result{Resource: res, Err: fmt.Errorf("error while parsing: %s", err), Status: validator.Error} continue } + if sig.Kind == "" { + validationResults <- validator.Result{Resource: res, Err: nil, Status: validator.Empty} + continue // We skip resoures that don't have a Kind defined + } + + if skip(*sig) { + validationResults <- validator.Result{Resource: res, Err: nil, Status: validator.Skipped} + continue + } + + ok := false + var schema *gojsonschema.Schema + cacheKey := "" + if c != nil { - c.Set(cacheKey, schema) + cacheKey = cache.Key(sig.Kind, sig.Version, k8sVersion) + schema, ok = c.Get(cacheKey) } - } - if schema == nil { - if ignoreMissingSchemas { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: nil, skipped: true}) - } else { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: fmt.Errorf("could not find schema for %s", sig.Kind), skipped: false}) + if !ok { + schema, err = downloadSchema(regs, sig.Kind, sig.Version, k8sVersion) + if err != nil { + validationResults <- validator.Result{Resource: res, Err: err, Status: validator.Error} + continue + } + + if c != nil { + c.Set(cacheKey, schema) + } } - } - err = validator.Validate(rawResource, schema) - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: err}) + if schema == nil { + if ignoreMissingSchemas { + validationResults <- validator.Result{Resource: res, Err: nil, Status: validator.Skipped} + } else { + validationResults <- validator.Result{Resource: res, Err: fmt.Errorf("could not find schema for %s", sig.Kind), Status: validator.Error} + } + } + + validationResults <- validator.Validate(res, schema) + } } - - return validationResults } -func processResults(o output.Output, validationResults chan []validationResult, result chan<- bool) { +func processResults(o output.Output, validationResults chan validator.Result, result chan<- bool) { success := true - for results := range validationResults { - for _, result := range results { - if result.err != nil { - success = false - } - - if err := o.Write(result.filename, result.kind, result.Name, result.version, result.err, result.skipped); err != nil { - fmt.Fprint(os.Stderr, "failed writing log\n") - } + for res := range validationResults { + if res.Err != nil { + success = false + } + if err := o.Write(res); err != nil { + fmt.Fprint(os.Stderr, "failed writing log\n") } } result <- success } -func getFiles(files []string, fileBatches chan []string, validationResults chan []validationResult) { +func getFiles(files []string, filesChan chan<- string, validationResults chan validator.Result) { for _, filename := range files { file, err := os.Open(filename) if err != nil { - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: false, - }} + validationResults <- validator.Result{ + Resource: resource.Resource{Path: filename}, + Err: err, + Status: validator.Error, + } continue } defer file.Close() @@ -157,23 +119,23 @@ func getFiles(files []string, fileBatches chan []string, validationResults chan fi, err := file.Stat() switch { case err != nil: - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: false, - }} + validationResults <- validator.Result{ + Resource: resource.Resource{Path: filename}, + Err: err, + Status: validator.Error, + } case fi.IsDir(): - if err := fsutils.FindYamlInDir(filename, fileBatches, 10); err != nil { - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: false, - }} + if err := fsutils.FindYamlInDir(filename, filesChan); err != nil { + validationResults <- validator.Result{ + Resource: resource.Resource{Path: filename}, + Err: err, + Status: validator.Error, + } } default: - fileBatches <- []string{filename} + filesChan <- filename } } } @@ -189,7 +151,6 @@ func realMain() int { // Detect whether we have data being piped through stdin stat, _ := os.Stdin.Stat() isStdin := (stat.Mode() & os.ModeCharDevice) == 0 - if len(cfg.Files) == 1 && cfg.Files[0] == "-" { isStdin = true } @@ -201,75 +162,76 @@ func realMain() int { registries := []registry.Registry{} for _, schemaLocation := range cfg.SchemaLocations { - if !strings.HasSuffix(schemaLocation, "json") { // If we dont specify a full templated path, we assume the paths of kubernetesjsonschema.dev - schemaLocation += "/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json" - } - - if strings.HasPrefix(schemaLocation, "http") { - registries = append(registries, registry.NewHTTPRegistry(schemaLocation, cfg.Strict)) - } else { - registries = append(registries, registry.NewLocalRegistry(schemaLocation, cfg.Strict)) - } + registries = append(registries, registry.New(schemaLocation, cfg.Strict)) } - validationResults := make(chan []validationResult) - c := cache.New() - - fileBatches := make(chan []string) - go func() { - getFiles(cfg.Files, fileBatches, validationResults) - close(fileBatches) - }() - var o output.Output if o, err = output.New(cfg.OutputFormat, cfg.Summary, isStdin, cfg.Verbose); err != nil { fmt.Fprintln(os.Stderr, err) return 1 } + validationResults := make(chan validator.Result) res := make(chan bool) go processResults(o, validationResults, res) - if isStdin { - res := ValidateStream(os.Stdin, registries, cfg.KubernetesVersion, c, filter, cfg.IgnoreMissingSchemas) - for i := range res { - res[i].filename = "stdin" - } - validationResults <- res - } else { - var wg sync.WaitGroup - for i := 0; i < cfg.NumberOfWorkers; i++ { - wg.Add(1) - go func() { - defer wg.Done() + files := make(chan string) + go func() { + getFiles(cfg.Files, files, validationResults) + close(files) + }() - for fileBatch := range fileBatches { - for _, filename := range fileBatch { - f, err := os.Open(filename) - if err != nil { - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: true, - }} - continue - } - - res := ValidateStream(f, registries, cfg.KubernetesVersion, c, filter, cfg.IgnoreMissingSchemas) - f.Close() - - for i := range res { - res[i].filename = filename - } - validationResults <- res - } - } - }() - } - - wg.Wait() + resourcesChan := make(chan []resource.Resource) + c := cache.New() + wg := sync.WaitGroup{} + for i := 0; i < cfg.NumberOfWorkers; i++ { + wg.Add(1) + go func() { + ValidateResources(resourcesChan, validationResults, registries, cfg.KubernetesVersion, c, filter, cfg.IgnoreMissingSchemas) + wg.Done() + }() } + if isStdin { + resources, err := resource.FromStream("stdin", os.Stdin) + if err != nil { + validationResults <- validator.Result{ + Resource: resource.Resource{Path: "stdin"}, + Err: err, + Status: validator.Error, + } + } else { + resourcesChan <- resources + } + } else { + for filename := range files { + f, err := os.Open(filename) + if err != nil { + validationResults <- validator.Result{ + Resource: resource.Resource{Path: filename}, + Err: err, + Status: validator.Error, + } + continue + } + + resources, err := resource.FromStream(filename, f) + if err != nil { + validationResults <- validator.Result{ + Resource: resource.Resource{Path: filename}, + Err: err, + Status: validator.Error, + } + continue + } + + resourcesChan <- resources + f.Close() + } + } + + close(resourcesChan) + wg.Wait() close(validationResults) success := <-res o.Flush() diff --git a/pkg/fsutils/findyaml.go b/pkg/fsutils/findyaml.go index f32a57f..3121ebc 100644 --- a/pkg/fsutils/findyaml.go +++ b/pkg/fsutils/findyaml.go @@ -8,27 +8,17 @@ import ( // FindYamlInDir will find yaml files in folder dir, and send their filenames in batches // of size batchSize to channel fileBatches -func FindYamlInDir(dir string, fileBatches chan<- []string, batchSize int) error { - files := []string{} - +func FindYamlInDir(dir string, fileBatches chan<- string) error { err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if !info.IsDir() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) { - files = append(files, path) - if len(files) > batchSize { - fileBatches <- files - files = []string{} - } + fileBatches <- path } return nil }) - if len(files) > 0 { - fileBatches <- files - } - return err } diff --git a/pkg/output/json.go b/pkg/output/json.go index 0654b14..1d42f6c 100644 --- a/pkg/output/json.go +++ b/pkg/output/json.go @@ -3,10 +3,11 @@ package output import ( "encoding/json" "fmt" + "github.com/yannh/kubeconform/pkg/validator" "io" ) -type result struct { +type oresult struct { Filename string `json:"filename"` Kind string `json:"kind"` Name string `json:"name"` @@ -19,7 +20,7 @@ type jsono struct { w io.Writer withSummary bool verbose bool - results []result + results []oresult nValid, nInvalid, nErrors, nSkipped int } @@ -29,7 +30,7 @@ func jsonOutput(w io.Writer, withSummary bool, isStdin, verbose bool) Output { w: w, withSummary: withSummary, verbose: verbose, - results: []result{}, + results: []oresult{}, nValid: 0, nInvalid: 0, nErrors: 0, @@ -38,31 +39,30 @@ func jsonOutput(w io.Writer, withSummary bool, isStdin, verbose bool) Output { } // JSON.Write will only write when JSON.Flush has been called -func (o *jsono) Write(filename, kind, name, version string, err error, skipped bool) error { +func (o *jsono) Write(result validator.Result) error { msg, st := "", "" - s := status(kind, name, err, skipped) - - switch s { - case statusValid: + switch result.Status { + case validator.Valid: st = "statusValid" o.nValid++ - case statusInvalid: + case validator.Invalid: st = "statusInvalid" - msg = err.Error() + msg = result.Err.Error() o.nInvalid++ - case statusError: + case validator.Error: st = "statusError" - msg = err.Error() + msg = result.Err.Error() o.nErrors++ - case statusSkipped: + case validator.Skipped: st = "statusSkipped" o.nSkipped++ - case statusEmpty: + case validator.Empty: } - if o.verbose || (s != statusValid && s != statusSkipped && s != statusEmpty) { - o.results = append(o.results, result{Filename: filename, Kind: kind, Name: name, Version: version, Status: st, Msg: msg}) + if o.verbose || (result.Status != validator.Valid && result.Status != validator.Skipped && result.Status != validator.Empty) { + sig, _ := result.Resource.Signature() + o.results = append(o.results, oresult{Filename: result.Resource.Path, Kind: sig.Kind, Name: sig.Name, Version: sig.Version, Status: st, Msg: msg}) } return nil @@ -75,7 +75,7 @@ func (o *jsono) Flush() error { if o.withSummary { jsonObj := struct { - Resources []result `json:"resources"` + Resources []oresult `json:"resources"` Summary struct { Valid int `json:"valid"` Invalid int `json:"invalid"` @@ -100,7 +100,7 @@ func (o *jsono) Flush() error { res, err = json.MarshalIndent(jsonObj, "", " ") } else { jsonObj := struct { - Resources []result `json:"resources"` + Resources []oresult `json:"resources"` }{ Resources: o.results, } diff --git a/pkg/output/json_test.go b/pkg/output/json_test.go deleted file mode 100644 index f14c912..0000000 --- a/pkg/output/json_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package output - -import ( - "bytes" - "testing" -) - -func TestJSONWrite(t *testing.T) { - type result struct { - fileName, kind, name, version string - err error - skipped bool - } - - for _, testCase := range []struct { - name string - withSummary bool - verbose bool - - res []result - expect string - }{ - { - "a single deployment, no summary, no verbose", - false, - false, - []result{ - { - "deployment.yml", - "Deployment", - "my-app", - "apps/v1", - nil, - false, - }, - }, - `{ - "resources": [] -} -`, - }, - { - "a single deployment, summary, no verbose", - true, - false, - []result{ - { - "deployment.yml", - "Deployment", - "my-app", - "apps/v1", - nil, - false, - }, - }, - `{ - "resources": [], - "summary": { - "valid": 1, - "invalid": 0, - "errors": 0, - "skipped": 0 - } -} -`, - }, - { - "a single deployment, verbose, with summary", - true, - true, - []result{ - { - "deployment.yml", - "Deployment", - "my-app", - "apps/v1", - nil, - false, - }, - }, - `{ - "resources": [ - { - "filename": "deployment.yml", - "kind": "Deployment", - "name": "my-app", - "version": "apps/v1", - "status": "statusValid", - "msg": "" - } - ], - "summary": { - "valid": 1, - "invalid": 0, - "errors": 0, - "skipped": 0 - } -} -`, - }, - } { - w := new(bytes.Buffer) - o := jsonOutput(w, testCase.withSummary, false, testCase.verbose) - - for _, res := range testCase.res { - o.Write(res.fileName, res.kind, res.name, res.version, res.err, res.skipped) - } - o.Flush() - - if w.String() != testCase.expect { - t.Fatalf("%s - expected %s, got %s", testCase.name, testCase.expect, w) - } - } -} diff --git a/pkg/output/output.go b/pkg/output/output.go index 5e0e58e..eaabbbe 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -6,17 +6,8 @@ import ( "os" ) -const ( - _ = iota - statusValid - statusInvalid - statusError - statusSkipped - statusEmpty -) - type Output interface { - Write(filename, kind, name, version string, err error, skipped bool) error + Write(validator.Result) error Flush() error } @@ -32,22 +23,3 @@ func New(outputFormat string, printSummary, isStdin, verbose bool) (Output, erro return nil, fmt.Errorf("`outputFormat` must be 'text' or 'json'") } } - -func status(kind, name string, err error, skipped bool) int { - if name == "" && kind == "" && err == nil && skipped == false { - return statusEmpty - } - - if skipped { - return statusSkipped - } - - if err != nil { - if _, ok := err.(validator.InvalidResourceError); ok { - return statusInvalid - } - return statusError - } - - return statusValid -} diff --git a/pkg/output/text.go b/pkg/output/text.go index cdf462b..6e16757 100644 --- a/pkg/output/text.go +++ b/pkg/output/text.go @@ -2,6 +2,7 @@ package output import ( "fmt" + "github.com/yannh/kubeconform/pkg/validator" "io" "sync" ) @@ -31,35 +32,37 @@ func textOutput(w io.Writer, withSummary, isStdin, verbose bool) Output { } } -func (o *texto) Write(filename, kind, name, version string, reserr error, skipped bool) error { +func (o *texto) Write(result validator.Result) error { o.Lock() defer o.Unlock() var err error - o.files[filename] = true - switch status(kind, name, reserr, skipped) { - case statusValid: + sig, _ := result.Resource.Signature() + + o.files[result.Resource.Path] = true + switch result.Status { + case validator.Valid: if o.verbose { - _, err = fmt.Fprintf(o.w, "%s - %s %s is valid\n", filename, kind, name) + _, err = fmt.Fprintf(o.w, "%s - %s %s is valid\n", result.Resource.Path, sig.Kind, sig.Name) } o.nValid++ - case statusInvalid: - _, err = fmt.Fprintf(o.w, "%s - %s %s is invalid: %s\n", filename, kind, name, reserr) + case validator.Invalid: + _, err = fmt.Fprintf(o.w, "%s - %s %s is invalid: %s\n", result.Resource.Path, sig.Kind, sig.Name, result.Err) o.nInvalid++ - case statusError: - if kind != "" && name != "" { - _, err = fmt.Fprintf(o.w, "%s - %s %s failed validation: %s\n", filename, kind, name, reserr) + case validator.Error: + if sig.Kind != "" && sig.Name != "" { + _, err = fmt.Fprintf(o.w, "%s - %s %s failed validation: %s\n", result.Resource.Path, sig.Kind, sig.Name, result.Err) } else { - _, err = fmt.Fprintf(o.w, "%s - failed validation: %s\n", filename, reserr) + _, err = fmt.Fprintf(o.w, "%s - failed validation: %s\n", result.Resource.Path, result.Err) } o.nErrors++ - case statusSkipped: + case validator.Skipped: if o.verbose { - _, err = fmt.Fprintf(o.w, "%s - %s %s skipped\n", filename, name, kind) + _, err = fmt.Fprintf(o.w, "%s - %s %s skipped\n", result.Resource.Path, sig.Name, sig.Kind) } o.nSkipped++ - case statusEmpty: // sent to ensure we count the filename as parsed + case validator.Empty: // sent to ensure we count the filename as parsed } return err diff --git a/pkg/output/text_test.go b/pkg/output/text_test.go deleted file mode 100644 index ba9d2ea..0000000 --- a/pkg/output/text_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package output - -import ( - "bytes" - "testing" -) - -func TestTextWrite(t *testing.T) { - type result struct { - fileName, kind, name, version string - err error - skipped bool - } - - for _, testCase := range []struct { - name string - withSummary bool - verbose bool - - res []result - expect string - }{ - { - "a single deployment, no summary, no verbose", - false, - false, - []result{ - { - "deployment.yml", - "Deployment", - "my-app", - "apps/v1", - nil, - false, - }, - }, - "", - }, - { - "a single deployment, summary, no verbose", - true, - false, - []result{ - { - "deployment.yml", - "Deployment", - "my-app", - "apps/v1", - nil, - false, - }, - }, - "Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0\n", - }, - { - "a single deployment, verbose, with summary", - true, - true, - []result{ - { - "deployment.yml", - "Deployment", - "my-app", - "apps/v1", - nil, - false, - }, - }, - `deployment.yml - Deployment my-app is valid -Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0 -`, - }, - } { - w := new(bytes.Buffer) - o := textOutput(w, testCase.withSummary, false, testCase.verbose) - - for _, res := range testCase.res { - o.Write(res.fileName, res.kind, res.name, res.version, res.err, res.skipped) - } - o.Flush() - - if w.String() != testCase.expect { - t.Errorf("%s - expected: %s, got: %s", testCase.name, testCase.expect, w) - } - } -} diff --git a/pkg/registry/kubernetesjsonschema.go b/pkg/registry/kubernetesjsonschema.go index e29e314..41fab0c 100644 --- a/pkg/registry/kubernetesjsonschema.go +++ b/pkg/registry/kubernetesjsonschema.go @@ -22,7 +22,7 @@ func newDownloadError(err error, isRetryable bool) *downloadError { func (e *downloadError) IsRetryable() bool { return e.isRetryable } func (e *downloadError) Error() string { return e.err.Error() } -func NewHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry { +func newHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry { return &KubernetesRegistry{ schemaPathTemplate: schemaPathTemplate, strict: strict, diff --git a/pkg/registry/local.go b/pkg/registry/local.go index 67d2460..d746591 100644 --- a/pkg/registry/local.go +++ b/pkg/registry/local.go @@ -23,7 +23,7 @@ func (e *fileNotFoundError) IsRetryable() bool { return e.isRetryable } func (e *fileNotFoundError) Error() string { return e.err.Error() } // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames -func NewLocalRegistry(pathTemplate string, strict bool) *LocalRegistry { +func newLocalRegistry(pathTemplate string, strict bool) *LocalRegistry { return &LocalRegistry{ pathTemplate, strict, diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 415ddf3..44814eb 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -64,3 +64,15 @@ func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict return buf.String(), nil } + +func New(schemaLocation string, strict bool) Registry { + if !strings.HasSuffix(schemaLocation, "json") { // If we dont specify a full templated path, we assume the paths of kubernetesjsonschema.dev + schemaLocation += "/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json" + } + + if strings.HasPrefix(schemaLocation, "http") { + return newHTTPRegistry(schemaLocation, strict) + } else { + return newLocalRegistry(schemaLocation, strict) + } +} diff --git a/pkg/resource/signature.go b/pkg/resource/signature.go index fab315b..30fdc34 100644 --- a/pkg/resource/signature.go +++ b/pkg/resource/signature.go @@ -4,12 +4,21 @@ import ( "sigs.k8s.io/yaml" ) +type Resource struct { + Path string + Bytes []byte + sig *Signature +} + type Signature struct { Kind, Version, Namespace, Name string } -// SignatureFromBytes returns key identifying elements from a []byte representing the resource -func SignatureFromBytes(res []byte) (Signature, error) { +func (res *Resource) Signature() (*Signature, error) { + if res.sig != nil { + return res.sig, nil + } + resource := struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` @@ -19,12 +28,14 @@ func SignatureFromBytes(res []byte) (Signature, error) { GenerateName string `yaml:"generateName"` } `yaml:"Metadata"` }{} - err := yaml.Unmarshal(res, &resource) + err := yaml.Unmarshal(res.Bytes, &resource) name := resource.Metadata.Name if resource.Metadata.GenerateName != "" { name = resource.Metadata.GenerateName + "{{ generateName }}" } - return Signature{Kind: resource.Kind, Version: resource.APIVersion, Namespace: resource.Metadata.Namespace, Name: name}, err + // We cache the result to not unmarshall every time we want to access the signature + res.sig = &Signature{Kind: resource.Kind, Version: resource.APIVersion, Namespace: resource.Metadata.Namespace, Name: name} + return res.sig, err } diff --git a/pkg/resource/signature_test.go b/pkg/resource/signature_test.go index ea638c4..50144e8 100644 --- a/pkg/resource/signature_test.go +++ b/pkg/resource/signature_test.go @@ -35,7 +35,8 @@ spec: } for _, testCase := range testCases { - sig, err := resource.SignatureFromBytes(testCase.have) + res := resource.Resource{Bytes: testCase.have} + sig, err := res.Signature() if err != nil && err.Error() != testCase.err.Error() { t.Errorf("test \"%s\" - received error: %s", testCase.name, err) } @@ -44,6 +45,5 @@ spec: sig.Namespace != testCase.want.Namespace { t.Errorf("test \"%s\": received %+v, expected %+v", testCase.name, sig, testCase.want) } - } } diff --git a/pkg/resource/stream.go b/pkg/resource/stream.go new file mode 100644 index 0000000..d7f6297 --- /dev/null +++ b/pkg/resource/stream.go @@ -0,0 +1,22 @@ +package resource + +import ( + "bytes" + "io" + "io/ioutil" +) + +func FromStream(path string, r io.Reader) ([]Resource, error) { + data, err := ioutil.ReadAll(r) + if err != nil { + return []Resource{}, err + } + + resources := []Resource{} + rawResources := bytes.Split(data, []byte("---\n")) + for _, rawResource := range rawResources { + resources = append(resources, Resource{Path: path, Bytes: rawResource}) + } + + return resources, nil +} diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index f431895..6191eb8 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -2,18 +2,22 @@ package validator import ( "fmt" + "github.com/yannh/kubeconform/pkg/resource" "github.com/xeipuuv/gojsonschema" "sigs.k8s.io/yaml" ) -// InvalidResourceError is returned when a resource does not conform to -// the associated schema -type InvalidResourceError struct{ err string } +type Status int -func (r InvalidResourceError) Error() string { - return r.err -} +const ( + _ Status = iota + Error + Skipped + Valid + Invalid + Empty +) // ValidFormat is a type for quickly forcing // new formats on the gojsonschema loader @@ -33,26 +37,32 @@ func (f ValidFormat) IsFormat(input interface{}) bool { // gojsonschema.FormatCheckers.Add("int-or-string", ValidFormat{}) // } +type Result struct { + Resource resource.Resource + Err error + Status Status +} + // Validate validates a single Kubernetes resource against a Json Schema -func Validate(rawResource []byte, schema *gojsonschema.Schema) error { +func Validate(res resource.Resource, schema *gojsonschema.Schema) Result { if schema == nil { - return nil + return Result{Resource: res, Status: Skipped, Err: nil} } var resource map[string]interface{} - if err := yaml.Unmarshal(rawResource, &resource); err != nil { - return fmt.Errorf("error unmarshalling resource: %s", err) + if err := yaml.Unmarshal(res.Bytes, &resource); err != nil { + return Result{Resource: res, Status: Error, Err: fmt.Errorf("error unmarshalling resource: %s", err)} } resourceLoader := gojsonschema.NewGoLoader(resource) results, err := schema.Validate(resourceLoader) if err != nil { // This error can only happen if the Object to validate is poorly formed. There's no hope of saving this one - return fmt.Errorf("problem validating schema. Check JSON formatting: %s", err) + return Result{Resource: res, Status: Error, Err: fmt.Errorf("problem validating schema. Check JSON formatting: %s", err)} } if results.Valid() { - return nil + return Result{Resource: res, Status: Valid} } msg := "" @@ -62,5 +72,6 @@ func Validate(rawResource []byte, schema *gojsonschema.Schema) error { } msg += errMsg.Description() } - return InvalidResourceError{err: msg} + + return Result{Resource: res, Status: Invalid, Err: fmt.Errorf("%s", msg)} } diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go index f498ba6..c9f2529 100644 --- a/pkg/validator/validator_test.go +++ b/pkg/validator/validator_test.go @@ -2,13 +2,13 @@ package validator import ( "fmt" + "github.com/yannh/kubeconform/pkg/resource" "testing" "github.com/xeipuuv/gojsonschema" ) func TestValidate(t *testing.T) { - for i, testCase := range []struct { name string rawResource, schema []byte @@ -122,8 +122,8 @@ lastName: bar if err != nil { t.Errorf("failed parsing test schema") } - if got := Validate(testCase.rawResource, schema); ((got == nil) != (testCase.expect == nil)) || (got != nil && (got.Error() != testCase.expect.Error())) { - t.Errorf("%d - expected %s, got %s", i, testCase.expect, got) + if got := Validate(resource.Resource{Bytes: testCase.rawResource}, schema); ((got.Err == nil) != (testCase.expect == nil)) || (got.Err != nil && (got.Err.Error() != testCase.expect.Error())) { + t.Errorf("%d - expected %s, got %s", i, testCase.expect, got.Err) } } }