From b35b5a2a117499b8acb24d57ada8c9d24246b269 Mon Sep 17 00:00:00 2001 From: Christoph Mertz Date: Tue, 27 Oct 2020 18:42:13 +0100 Subject: [PATCH] big bad mass commit --- Makefile | 2 +- cmd/kubeconform/main.go | 379 ++++----------------- cmd/kubeconform/main_test.go | 41 --- pkg/cache/schemacache.go | 36 -- pkg/config/config.go | 82 +++++ pkg/fsutils/findyaml.go | 34 -- pkg/output/output.go | 25 +- pkg/registry/kubernetesjsonschema.go | 58 ---- pkg/registry/local.go | 55 --- pkg/registry/registry_test.go | 58 ---- pkg/resource/discover.go | 81 +++++ pkg/resource/resource.go | 17 + pkg/resource/signature.go | 31 +- pkg/resource/signature_test.go | 49 --- pkg/{registry/registry.go => schema/fs.go} | 52 ++- pkg/schema/noop.go | 9 + pkg/schema/remote.go | 49 +++ pkg/schema/repository.go | 69 ++++ pkg/schema/schema.go | 40 +++ pkg/validation/result.go | 23 ++ pkg/validation/validation.go | 96 ++++++ pkg/validator/validator.go | 66 ---- pkg/validator/validator_test.go | 129 ------- 23 files changed, 602 insertions(+), 879 deletions(-) delete mode 100644 cmd/kubeconform/main_test.go delete mode 100644 pkg/cache/schemacache.go create mode 100644 pkg/config/config.go delete mode 100644 pkg/fsutils/findyaml.go delete mode 100644 pkg/registry/kubernetesjsonschema.go delete mode 100644 pkg/registry/local.go delete mode 100644 pkg/registry/registry_test.go create mode 100644 pkg/resource/discover.go create mode 100644 pkg/resource/resource.go delete mode 100644 pkg/resource/signature_test.go rename pkg/{registry/registry.go => schema/fs.go} (53%) create mode 100644 pkg/schema/noop.go create mode 100644 pkg/schema/remote.go create mode 100644 pkg/schema/repository.go create mode 100644 pkg/schema/schema.go create mode 100644 pkg/validation/result.go create mode 100644 pkg/validation/validation.go delete mode 100644 pkg/validator/validator.go delete mode 100644 pkg/validator/validator_test.go diff --git a/Makefile b/Makefile index 51f4ac7..0321050 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ build-bats: docker build -t bats -f Dockerfile.bats . docker-acceptance: build-bats - docker run -t bats acceptance.bats + docker run -it bats --pretty acceptance.bats release: docker run -e GITHUB_TOKEN -t -v $$PWD:/go/src/github.com/yannh/kubeconform -w /go/src/github.com/yannh/kubeconform goreleaser/goreleaser:v0.138 goreleaser release --rm-dist \ No newline at end of file diff --git a/cmd/kubeconform/main.go b/cmd/kubeconform/main.go index a694bbe..045cb1a 100644 --- a/cmd/kubeconform/main.go +++ b/cmd/kubeconform/main.go @@ -1,337 +1,94 @@ package main import ( - "bytes" "flag" "fmt" - "io" - "io/ioutil" "os" + "runtime" "strings" "sync" - "github.com/xeipuuv/gojsonschema" - - "github.com/yannh/kubeconform/pkg/cache" - "github.com/yannh/kubeconform/pkg/fsutils" + "github.com/yannh/kubeconform/pkg/config" "github.com/yannh/kubeconform/pkg/output" - "github.com/yannh/kubeconform/pkg/registry" "github.com/yannh/kubeconform/pkg/resource" - "github.com/yannh/kubeconform/pkg/validator" + "github.com/yannh/kubeconform/pkg/schema" + "github.com/yannh/kubeconform/pkg/validation" ) -type validationResult struct { - filename, kind, version, Name string - err error - skipped bool -} +func processResults(o output.Output, results <-chan validation.Result) <-chan bool { + done := make(chan bool) -func resourcesFromReader(r io.Reader) ([][]byte, error) { - data, err := ioutil.ReadAll(r) - if err != nil { - return [][]byte{}, err - } - - resources := bytes.Split(data, []byte("---\n")) - - return resources, nil -} - -func downloadSchema(registries []registry.Registry, kind, version, k8sVersion string) (*gojsonschema.Schema, error) { - var err error - var schemaBytes []byte - - for _, reg := range registries { - schemaBytes, err = reg.DownloadSchema(kind, version, k8sVersion) - if err == nil { - return gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaBytes)) - } - - // If we get a 404, we try the next registry, but we exit if we get a real failure - if er, retryable := err.(registry.Retryable); retryable && !er.IsRetryable() { - continue - } - - return nil, err - } - - return nil, nil // No schema found - we don't consider it an error, resource will be skipped -} - -// filter returns true if the file should be skipped -// Returning an array, this Reader might container multiple resources -func ValidateStream(r io.Reader, regs []registry.Registry, k8sVersion string, c *cache.SchemaCache, skip func(signature resource.Signature) bool, ignoreMissingSchemas bool) []validationResult { - rawResources, err := resourcesFromReader(r) - if err != nil { - return []validationResult{{err: fmt.Errorf("failed reading file: %s", err)}} - } - - validationResults := []validationResult{} - if len(rawResources) == 0 { - // In case a file has no resources, we want to capture that the file was parsed - and therefore send a message with an empty resource and no error - validationResults = append(validationResults, validationResult{kind: "", version: "", Name: "", err: nil, skipped: false}) - } - - for _, rawResource := range rawResources { - var sig resource.Signature - if sig, err = resource.SignatureFromBytes(rawResource); err != nil { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: fmt.Errorf("error while parsing: %s", err)}) - continue - } - - if sig.Kind == "" { - validationResults = append(validationResults, validationResult{kind: "", version: "", Name: "", err: nil, skipped: false}) - continue // We skip resoures that don't have a Kind defined - } - - if skip(sig) { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: nil, skipped: true}) - continue - } - - ok := false - var schema *gojsonschema.Schema - cacheKey := "" - - if c != nil { - cacheKey = cache.Key(sig.Kind, sig.Version, k8sVersion) - schema, ok = c.Get(cacheKey) - } - - if !ok { - schema, err = downloadSchema(regs, sig.Kind, sig.Version, k8sVersion) + go func() { + ret := true + for result := range results { + sig, err := result.Signature() if err != nil { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: err, skipped: false}) - continue + // fmt.Fprint(os.Stderr, "failed writing log\n") } - - if c != nil { - c.Set(cacheKey, schema) + if result.Status == validation.Invalid { + ret = false } - } - - if schema == nil { - if ignoreMissingSchemas { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: nil, skipped: true}) - } else { - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: fmt.Errorf("could not find schema for %s", sig.Kind), skipped: false}) - } - } - - err = validator.Validate(rawResource, schema) - validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, Name: sig.Name, err: err}) - } - - return validationResults -} - -type arrayParam []string - -func (ap *arrayParam) String() string { - return strings.Join(*ap, " - ") -} - -func (ap *arrayParam) Set(value string) error { - *ap = append(*ap, value) - return nil -} - -func getLogger(outputFormat string, printSummary, verbose bool) (output.Output, error) { - w := os.Stdout - - switch { - case outputFormat == "text": - return output.Text(w, printSummary, verbose), nil - case outputFormat == "json": - return output.JSON(w, printSummary, verbose), nil - default: - return nil, fmt.Errorf("-output must be text or json") - } -} - -func skipKindsMap(skipKindsCSV string) map[string]bool { - splitKinds := strings.Split(skipKindsCSV, ",") - skipKinds := map[string]bool{} - for _, kind := range splitKinds { - if len(kind) > 0 { - skipKinds[kind] = true - } - } - return skipKinds -} - -func processResults(o output.Output, validationResults chan []validationResult, result chan<- bool) { - success := true - for results := range validationResults { - for _, result := range results { - if result.err != nil { - success = false - } - - if err := o.Write(result.filename, result.kind, result.Name, result.version, result.err, result.skipped); err != nil { + err = o.Write(result.Path, sig.Kind, sig.Metadata.Name, sig.APIVersion, result.Err, result.Status == validation.Skipped) + if err != nil { fmt.Fprint(os.Stderr, "failed writing log\n") } } - } - - result <- success -} - -func getFiles(files []string, fileBatches chan []string, validationResults chan []validationResult) { - for _, filename := range files { - file, err := os.Open(filename) - if err != nil { - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: false, - }} - continue - } - defer file.Close() - - fi, err := file.Stat() - switch { - case err != nil: - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: false, - }} - - case fi.IsDir(): - if err := fsutils.FindYamlInDir(filename, fileBatches, 10); err != nil { - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: false, - }} - } - - default: - fileBatches <- []string{filename} - } - } -} - -func realMain() int { - var schemaLocationsParam arrayParam - var skipKindsCSV, k8sVersion, outputFormat string - var summary, strict, verbose, ignoreMissingSchemas, help bool - var nWorkers int - var err error - var files []string - - flag.StringVar(&k8sVersion, "kubernetes-version", "1.18.0", "version of Kubernetes to validate against") - flag.Var(&schemaLocationsParam, "schema-location", "override schemas location search path (can be specified multiple times)") - flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing") - flag.BoolVar(&summary, "summary", false, "print a summary at the end") - flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel") - flag.StringVar(&skipKindsCSV, "skip", "", "comma-separated list of kinds to ignore") - flag.BoolVar(&strict, "strict", false, "disallow additional properties not in schema") - flag.StringVar(&outputFormat, "output", "text", "output format - text, json") - flag.BoolVar(&verbose, "verbose", false, "print results for all resources") - flag.BoolVar(&help, "h", false, "show help information") - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE OR FOLDER]...\n", os.Args[0]) - flag.PrintDefaults() - } - - flag.Parse() - - if help { - flag.Usage() - return 1 - } - - skipKinds := skipKindsMap(skipKindsCSV) - - for _, file := range flag.Args() { - files = append(files, file) - } - - filter := func(signature resource.Signature) bool { - isSkipKind, ok := skipKinds[signature.Kind] - return ok && isSkipKind - } - - if len(schemaLocationsParam) == 0 { - schemaLocationsParam = append(schemaLocationsParam, "https://kubernetesjsonschema.dev") // if not specified, default behaviour is to use kubernetesjson-schema.dev as registry - } - - registries := []registry.Registry{} - for _, schemaLocation := range schemaLocationsParam { - if !strings.HasSuffix(schemaLocation, "json") { // If we dont specify a full templated path, we assume the paths of kubernetesjsonschema.dev - schemaLocation += "/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json" - } - - if strings.HasPrefix(schemaLocation, "http") { - registries = append(registries, registry.NewHTTPRegistry(schemaLocation, strict)) - } else { - registries = append(registries, registry.NewLocalRegistry(schemaLocation, strict)) - } - } - - validationResults := make(chan []validationResult) - - fileBatches := make(chan []string) - go func() { - getFiles(files, fileBatches, validationResults) - close(fileBatches) + done <- ret }() - var o output.Output - if o, err = getLogger(outputFormat, summary, verbose); err != nil { - fmt.Fprintln(os.Stderr, err) - return 1 - } - - res := make(chan bool) - go processResults(o, validationResults, res) - - c := cache.New() - var wg sync.WaitGroup - for i := 0; i < nWorkers; i++ { - wg.Add(1) - go func() { - defer wg.Done() - - for fileBatch := range fileBatches { - for _, filename := range fileBatch { - f, err := os.Open(filename) - if err != nil { - validationResults <- []validationResult{{ - filename: filename, - err: err, - skipped: true, - }} - continue - } - - res := ValidateStream(f, registries, k8sVersion, c, filter, ignoreMissingSchemas) - f.Close() - - for i := range res { - res[i].filename = filename - } - validationResults <- res - } - } - }() - } - - wg.Wait() - close(validationResults) - success := <-res - o.Flush() - - if !success { - return 1 - } - - return 0 + return done } func main() { - os.Exit(realMain()) + conf := config.FromFlags() + if conf.Help { + os.Exit(0) + } + + if conf.NumberOfWorkers == -1 { + conf.NumberOfWorkers = runtime.NumCPU() + } + + var opts []schema.Option + for _, loc := range conf.SchemaLocations { + if strings.HasPrefix(loc, "http://") || strings.HasPrefix(loc, "https://") { + opts = append(opts, schema.FromRemote(loc)) + } else { + opts = append(opts, schema.FromFS(loc)) + } + } + + schemas := schema.New(opts...) + validator := validation.New(schemas, conf) + resources := resource.Discover(flag.Args()...) + + results := make(chan validation.Result) + var wg sync.WaitGroup + wg.Add(conf.NumberOfWorkers) + for i := 0; i < conf.NumberOfWorkers; i++ { + go func() { + for batch := range resources { + for _, res := range batch { + results <- validator.Validate(res) + } + } + wg.Done() + }() + } + + output, err := output.New(conf.OutputFormat, conf.Summary, conf.Verbose) + if err != nil { + fmt.Fprintf(os.Stderr, "error getting output: %s\n", err) + os.Exit(1) + } + + done := processResults(output, results) + + wg.Wait() + close(results) + if <-done { + os.Exit(0) + } + + os.Exit(1) } diff --git a/cmd/kubeconform/main_test.go b/cmd/kubeconform/main_test.go deleted file mode 100644 index 91c0ee5..0000000 --- a/cmd/kubeconform/main_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "reflect" - "testing" -) - -func TestSkipKindMaps(t *testing.T) { - for _, testCase := range []struct { - name string - csvSkipKinds string - expect map[string]bool - }{ - { - "nothing to skip", - "", - map[string]bool{}, - }, - { - "a single kind to skip", - "somekind", - map[string]bool{ - "somekind": true, - }, - }, - { - "multiple kinds to skip", - "somekind,anotherkind,yetsomeotherkind", - map[string]bool{ - "somekind": true, - "anotherkind": true, - "yetsomeotherkind": true, - }, - }, - } { - got := skipKindsMap(testCase.csvSkipKinds) - if !reflect.DeepEqual(got, testCase.expect) { - t.Errorf("%s - got %+v, expected %+v", testCase.name, got, testCase.expect) - } - } -} diff --git a/pkg/cache/schemacache.go b/pkg/cache/schemacache.go deleted file mode 100644 index 2290d4e..0000000 --- a/pkg/cache/schemacache.go +++ /dev/null @@ -1,36 +0,0 @@ -package cache - -import ( - "fmt" - "sync" - - "github.com/xeipuuv/gojsonschema" -) - -type SchemaCache struct { - sync.RWMutex - schemas map[string]*gojsonschema.Schema -} - -func New() *SchemaCache { - return &SchemaCache{ - schemas: map[string]*gojsonschema.Schema{}, - } -} - -func Key(resourceKind, resourceAPIVersion, k8sVersion string) string { - return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion) -} - -func (c *SchemaCache) Get(key string) (*gojsonschema.Schema, bool) { - c.RLock() - defer c.RUnlock() - schema, ok := c.schemas[key] - return schema, ok -} - -func (c *SchemaCache) Set(key string, schema *gojsonschema.Schema) { - c.Lock() - defer c.Unlock() - c.schemas[key] = schema -} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..c002456 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,82 @@ +package config + +import ( + "flag" + "fmt" + "os" + "strings" +) + +// Config TODO +type Config struct { + RootPaths []string + SchemaLocations []string + SkipKinds map[string]struct{} + OutputFormat string + KubernetesVersion string + NumberOfWorkers int + Summary bool + Strict bool + Verbose bool + IgnoreMissingSchemas bool + Help bool +} + +type arrayParam []string + +func (ap *arrayParam) String() string { + return strings.Join(*ap, " - ") +} + +func (ap *arrayParam) Set(value string) error { + *ap = append(*ap, value) + return nil +} + +func skipKinds(skipKindsCSV string) map[string]struct{} { + splitKinds := strings.Split(skipKindsCSV, ",") + skipKinds := map[string]struct{}{} + + for _, kind := range splitKinds { + if len(kind) > 0 { + skipKinds[kind] = struct{}{} + } + } + + return skipKinds +} + +// FromFlags TODO +func FromFlags() Config { + c := Config{} + + flag.StringVar(&c.KubernetesVersion, "kubernetes-version", "1.18.0", "version of Kubernetes to validate against") + + var schemaLocationsParam arrayParam + flag.Var(&schemaLocationsParam, "schema-location", "override schemas location search path (can be specified multiple times)") + c.SchemaLocations = []string(schemaLocationsParam) + + var skipKindsCSV string + flag.StringVar(&skipKindsCSV, "skip", "", "comma-separated list of kinds to ignore") + c.SkipKinds = skipKinds(skipKindsCSV) + + flag.BoolVar(&c.IgnoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing") + flag.BoolVar(&c.Summary, "summary", false, "print a summary at the end") + flag.IntVar(&c.NumberOfWorkers, "n", -1, "number of goroutines to run concurrently") + + flag.BoolVar(&c.Strict, "strict", false, "disallow additional properties not in schema") + flag.StringVar(&c.OutputFormat, "output", "text", "output format - text, json") + flag.BoolVar(&c.Verbose, "verbose", false, "print results for all resources") + flag.BoolVar(&c.Help, "h", false, "show help information") + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE OR FOLDER]...\n", os.Args[0]) + flag.PrintDefaults() + } + + flag.Parse() + if c.Help { + flag.Usage() + } + + return c +} diff --git a/pkg/fsutils/findyaml.go b/pkg/fsutils/findyaml.go deleted file mode 100644 index f32a57f..0000000 --- a/pkg/fsutils/findyaml.go +++ /dev/null @@ -1,34 +0,0 @@ -package fsutils - -import ( - "os" - "path/filepath" - "strings" -) - -// FindYamlInDir will find yaml files in folder dir, and send their filenames in batches -// of size batchSize to channel fileBatches -func FindYamlInDir(dir string, fileBatches chan<- []string, batchSize int) error { - files := []string{} - - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) { - files = append(files, path) - if len(files) > batchSize { - fileBatches <- files - files = []string{} - } - } - return nil - }) - - if len(files) > 0 { - fileBatches <- files - } - - return err -} diff --git a/pkg/output/output.go b/pkg/output/output.go index d5c0a4f..4edf079 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -1,9 +1,11 @@ package output import ( - "github.com/yannh/kubeconform/pkg/validator" + "fmt" + "os" ) +// TODO comment const ( _ = iota VALID @@ -13,11 +15,26 @@ const ( EMPTY ) +// Output TODO type Output interface { Write(filename, kind, name, version string, err error, skipped bool) error Flush() error } +// New TODO +func New(outputFormat string, printSummary, verbose bool) (Output, error) { + w := os.Stdout + + switch { + case outputFormat == "text": + return Text(w, printSummary, verbose), nil + case outputFormat == "json": + return JSON(w, printSummary, verbose), nil + default: + return nil, fmt.Errorf("-output must be text or json") + } +} + func status(kind, name string, err error, skipped bool) int { if name == "" && kind == "" && err == nil && skipped == false { return EMPTY @@ -28,9 +45,9 @@ func status(kind, name string, err error, skipped bool) int { } if err != nil { - if _, ok := err.(validator.InvalidResourceError); ok { - return INVALID - } + // if _, ok := err.(validator.InvalidResourceError); ok { + // return INVALID + // } return ERROR } diff --git a/pkg/registry/kubernetesjsonschema.go b/pkg/registry/kubernetesjsonschema.go deleted file mode 100644 index e29e314..0000000 --- a/pkg/registry/kubernetesjsonschema.go +++ /dev/null @@ -1,58 +0,0 @@ -package registry - -import ( - "fmt" - "io/ioutil" - "net/http" -) - -type KubernetesRegistry struct { - schemaPathTemplate string - strict bool -} - -type downloadError struct { - err error - isRetryable bool -} - -func newDownloadError(err error, isRetryable bool) *downloadError { - return &downloadError{err, isRetryable} -} -func (e *downloadError) IsRetryable() bool { return e.isRetryable } -func (e *downloadError) Error() string { return e.err.Error() } - -func NewHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry { - return &KubernetesRegistry{ - schemaPathTemplate: schemaPathTemplate, - strict: strict, - } -} - -func (r KubernetesRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - url, err := schemaPath(r.schemaPathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict) - if err != nil { - return nil, err - } - - resp, err := http.Get(url) - if err != nil { - return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err) - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusNotFound { - return nil, newDownloadError(fmt.Errorf("no schema found"), false) - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("error while downloading schema - received HTTP status %d", resp.StatusCode) - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err) - } - - return body, nil -} diff --git a/pkg/registry/local.go b/pkg/registry/local.go deleted file mode 100644 index 67d2460..0000000 --- a/pkg/registry/local.go +++ /dev/null @@ -1,55 +0,0 @@ -package registry - -import ( - "fmt" - "io/ioutil" - "os" -) - -type LocalRegistry struct { - pathTemplate string - strict bool -} - -type fileNotFoundError struct { - err error - isRetryable bool -} - -func newFileNotFoundError(err error, isRetryable bool) *fileNotFoundError { - return &fileNotFoundError{err, isRetryable} -} -func (e *fileNotFoundError) IsRetryable() bool { return e.isRetryable } -func (e *fileNotFoundError) Error() string { return e.err.Error() } - -// NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames -func NewLocalRegistry(pathTemplate string, strict bool) *LocalRegistry { - return &LocalRegistry{ - pathTemplate, - strict, - } -} - -// DownloadSchema retrieves the schema from a file for the resource -func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - schemaFile, err := schemaPath(r.pathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict) - if err != nil { - return []byte{}, nil - } - - f, err := os.Open(schemaFile) - if err != nil { - if os.IsNotExist(err) { - return nil, newFileNotFoundError(fmt.Errorf("no schema found"), false) - } - return nil, fmt.Errorf("failed to open schema %s", schemaFile) - } - - defer f.Close() - content, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - return content, nil -} diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go deleted file mode 100644 index 8594c0f..0000000 --- a/pkg/registry/registry_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package registry - -import ( - "testing" -) - -func TestSchemaPath(t *testing.T) { - for i, testCase := range []struct { - tpl, resourceKind, resourceAPIVersion, k8sVersion, expected string - strict bool - errExpected error - }{ - { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", - "Deployment", - "apps/v1", - "1.16.0", - "https://kubernetesjsonschema.dev/v1.16.0-standalone-strict/deployment-apps-v1.json", - true, - nil, - }, - { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", - "Deployment", - "apps/v1", - "1.16.0", - "https://kubernetesjsonschema.dev/v1.16.0-standalone/deployment-apps-v1.json", - false, - nil, - }, - { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", - "Service", - "v1", - "1.18.0", - "https://kubernetesjsonschema.dev/v1.18.0-standalone/service-v1.json", - false, - nil, - }, - { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", - "Service", - "v1", - "master", - "https://kubernetesjsonschema.dev/master-standalone/service-v1.json", - false, - nil, - }, - } { - got, err := schemaPath(testCase.tpl, testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict) - if err != testCase.errExpected { - t.Errorf("%d - got error %s, expected %s", i+1, err, testCase.errExpected) - } - if got != testCase.expected { - t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected) - } - } -} diff --git a/pkg/resource/discover.go b/pkg/resource/discover.go new file mode 100644 index 0000000..5a82b18 --- /dev/null +++ b/pkg/resource/discover.go @@ -0,0 +1,81 @@ +package resource + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +const batchSize = 10 + +func isYAMLFile(info os.FileInfo) bool { + return !info.IsDir() && (strings.HasSuffix(info.Name(), ".yaml") || strings.HasSuffix(info.Name(), ".yml")) +} + +func Discover(paths ...string) <-chan []Resource { + res := make(chan []Resource) + + go func() { + for _, path := range paths { + var batch []Resource + + // we handle errors in the walk function directly + // so it should be safe to discard the outer error + _ = filepath.Walk(path, func(p string, i os.FileInfo, e error) error { + if len(batch) > batchSize { + res <- batch + batch = nil + } + + if e != nil { + batch = append(batch, Resource{ + Path: p, + Err: e, + }) + return e + } + + if !isYAMLFile(i) { + return nil + } + + f, err := os.Open(p) + if err != nil { + batch = append(batch, Resource{ + Path: p, + Err: err, + }) + return err + } + + b, err := ioutil.ReadAll(f) + if err != nil { + batch = append(batch, Resource{ + Path: p, + Err: err, + }) + return err + } + + for _, r := range bytes.Split(b, []byte("---\n")) { + batch = append(batch, Resource{ + Path: p, + Bytes: r, + }) + } + + return nil + }) + + if len(batch) > 0 { + res <- batch + } + } + + close(res) + }() + + return res +} diff --git a/pkg/resource/resource.go b/pkg/resource/resource.go new file mode 100644 index 0000000..15e6989 --- /dev/null +++ b/pkg/resource/resource.go @@ -0,0 +1,17 @@ +package resource + +import ( + "sigs.k8s.io/yaml" +) + +type Resource struct { + Path string + Bytes []byte + Err error +} + +func (r Resource) AsMap() (map[string]interface{}, error) { + var res map[string]interface{} + err := yaml.Unmarshal(r.Bytes, &res) + return res, err +} diff --git a/pkg/resource/signature.go b/pkg/resource/signature.go index fab315b..521acf7 100644 --- a/pkg/resource/signature.go +++ b/pkg/resource/signature.go @@ -5,26 +5,17 @@ import ( ) type Signature struct { - Kind, Version, Namespace, Name string + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + GenerateName string `yaml:"generateName"` + } `yaml:"Metadata"` } -// SignatureFromBytes returns key identifying elements from a []byte representing the resource -func SignatureFromBytes(res []byte) (Signature, error) { - resource := struct { - APIVersion string `yaml:"apiVersion"` - Kind string `yaml:"kind"` - Metadata struct { - Name string `yaml:"name"` - Namespace string `yaml:"namespace"` - GenerateName string `yaml:"generateName"` - } `yaml:"Metadata"` - }{} - err := yaml.Unmarshal(res, &resource) - - name := resource.Metadata.Name - if resource.Metadata.GenerateName != "" { - name = resource.Metadata.GenerateName + "{{ generateName }}" - } - - return Signature{Kind: resource.Kind, Version: resource.APIVersion, Namespace: resource.Metadata.Namespace, Name: name}, err +func (r Resource) Signature() (Signature, error) { + var sig Signature + err := yaml.Unmarshal(r.Bytes, &sig) + return sig, err } diff --git a/pkg/resource/signature_test.go b/pkg/resource/signature_test.go deleted file mode 100644 index ea638c4..0000000 --- a/pkg/resource/signature_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package resource_test - -import ( - "testing" - - "github.com/yannh/kubeconform/pkg/resource" -) - -func TestSignatureFromBytes(t *testing.T) { - testCases := []struct { - name string - have []byte - want resource.Signature - err error - }{ - { - name: "valid deployment", - have: []byte(` -apiVersion: apps/v1 -kind: Deployment -metadata: - name: myService - namespace: default - labels: - app: myService -spec: -`), - want: resource.Signature{ - Kind: "Deployment", - Version: "apps/v1", - Namespace: "default", - }, - err: nil, - }, - } - - for _, testCase := range testCases { - sig, err := resource.SignatureFromBytes(testCase.have) - if err != nil && err.Error() != testCase.err.Error() { - t.Errorf("test \"%s\" - received error: %s", testCase.name, err) - } - if sig.Version != testCase.want.Version || - sig.Kind != testCase.want.Kind || - sig.Namespace != testCase.want.Namespace { - t.Errorf("test \"%s\": received %+v, expected %+v", testCase.name, sig, testCase.want) - } - - } -} diff --git a/pkg/registry/registry.go b/pkg/schema/fs.go similarity index 53% rename from pkg/registry/registry.go rename to pkg/schema/fs.go index 415ddf3..1c20db3 100644 --- a/pkg/registry/registry.go +++ b/pkg/schema/fs.go @@ -1,26 +1,18 @@ -package registry +package schema import ( "bytes" + "io/ioutil" + "os" "strings" "text/template" + + "github.com/xeipuuv/gojsonschema" ) -type Manifest struct { - Kind, Version string -} +type fs string -// Registry is an interface that should be implemented by any source of Kubernetes schemas -type Registry interface { - DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) -} - -// Retryable indicates whether an error is a temporary or a permanent failure -type Retryable interface { - IsRetryable() bool -} - -func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict bool) (string, error) { +func path(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict bool) (string, error) { normalisedVersion := k8sVersion if normalisedVersion != "master" { normalisedVersion = "v" + normalisedVersion @@ -58,9 +50,35 @@ func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict var buf bytes.Buffer err = tmpl.Execute(&buf, tplData) + + return buf.String(), err +} + +func (f fs) Get(kind string, version string, kubernetesVersion string) (*Schema, error) { + p, err := path(string(f), kind, version, kubernetesVersion, true) if err != nil { - return "", err + return nil, err } - return buf.String(), nil + file, err := os.Open(p) + if err != nil { + return nil, err + } + defer file.Close() + + b, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(b)) + + return (*Schema)(schema), err +} + +// FromFS TODO +func FromFS(path string) Option { + return func(r *Repository) { + r.fetcher = append(r.fetcher, fs(path)) + } } diff --git a/pkg/schema/noop.go b/pkg/schema/noop.go new file mode 100644 index 0000000..aaa7a15 --- /dev/null +++ b/pkg/schema/noop.go @@ -0,0 +1,9 @@ +package schema + +import "errors" + +type noop struct{} + +func (n noop) Get(kind string, version, kubernetesVersion string) (*Schema, error) { + return nil, errors.New("schema not found") +} diff --git a/pkg/schema/remote.go b/pkg/schema/remote.go new file mode 100644 index 0000000..ef7840f --- /dev/null +++ b/pkg/schema/remote.go @@ -0,0 +1,49 @@ +package schema + +import ( + "fmt" + "io/ioutil" + "net/http" + + "github.com/xeipuuv/gojsonschema" +) + +const kubernetesJSONSchemaURLTmpl = "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json" + +type remote string + +func (r remote) Get(kind string, version string, kubernetesVersion string) (*Schema, error) { + url, err := path(string(r), kind, version, kubernetesVersion, true) + if err != nil { + return nil, err + } + + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("no schema found") + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("error while downloading schema - received HTTP status %d", resp.StatusCode) + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err) + } + + schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(b)) + return (*Schema)(schema), err +} + +// FromRemote TODO +func FromRemote(url string) Option { + return func(r *Repository) { + r.fetcher = append(r.fetcher, remote(url)) + } +} diff --git a/pkg/schema/repository.go b/pkg/schema/repository.go new file mode 100644 index 0000000..7e24427 --- /dev/null +++ b/pkg/schema/repository.go @@ -0,0 +1,69 @@ +package schema + +import ( + "errors" + "fmt" + "sync" +) + +// Fetcher TODO +type Fetcher interface { + Get(kind string, version string, kubernetesVersion string) (*Schema, error) +} + +// Repository TODO +type Repository struct { + schemas map[string]*Schema + schemasLock sync.RWMutex + + fetcher []Fetcher +} + +func key(resourceKind, resourceAPIVersion, k8sVersion string) string { + return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion) +} + +// Get TODO +func (r *Repository) Get(kind string, version string, kubernetesVersion string) (*Schema, error) { + r.schemasLock.RLock() + defer r.schemasLock.RUnlock() + + schema, ok := r.schemas[key(kind, version, kubernetesVersion)] + if ok { + return schema, nil + } + + for _, fetcher := range r.fetcher { + schema, err := fetcher.Get(kind, version, kubernetesVersion) + + if err != nil { + continue + } + + r.schemas[key(kind, version, kubernetesVersion)] = schema + return schema, nil + } + + return nil, errors.New("schema not found") +} + +// Option TODO +type Option func(*Repository) + +// New TODO +func New(opts ...Option) *Repository { + r := &Repository{ + schemas: map[string]*Schema{}, + schemasLock: sync.RWMutex{}, + fetcher: []Fetcher{}, + } + + for _, opt := range opts { + opt(r) + } + + // add kubernetesjsonschema.dev as last fetcher + FromRemote(kubernetesJSONSchemaURLTmpl)(r) + + return r +} diff --git a/pkg/schema/schema.go b/pkg/schema/schema.go new file mode 100644 index 0000000..4a4f603 --- /dev/null +++ b/pkg/schema/schema.go @@ -0,0 +1,40 @@ +package schema + +import ( + "errors" + "strings" + + "github.com/xeipuuv/gojsonschema" +) + +type multiError []error + +func (m multiError) Error() string { + var r []string + for _, e := range ([]error)(m) { + r = append(r, e.Error()) + } + return strings.Join(r, "\n") +} + +// Schema TODO +type Schema gojsonschema.Schema + +// Validate TODO +func (s *Schema) Validate(resource map[string]interface{}) error { + results, err := (*gojsonschema.Schema)(s).Validate(gojsonschema.NewGoLoader(resource)) + if err != nil { + return err + } + + if results.Valid() { + return nil + } + + var errs []error + for _, e := range results.Errors() { + errs = append(errs, errors.New(e.Description())) + } + + return multiError(errs) +} diff --git a/pkg/validation/result.go b/pkg/validation/result.go new file mode 100644 index 0000000..eaa8de8 --- /dev/null +++ b/pkg/validation/result.go @@ -0,0 +1,23 @@ +package validation + +import "github.com/yannh/kubeconform/pkg/resource" + +// Status TODO +type Status int + +// TODO +const ( + _ Status = iota + Error + Skipped + Valid + Invalid + Empty +) + +// Result TODO +type Result struct { + resource.Resource + Err error + Status Status +} diff --git a/pkg/validation/validation.go b/pkg/validation/validation.go new file mode 100644 index 0000000..a49a575 --- /dev/null +++ b/pkg/validation/validation.go @@ -0,0 +1,96 @@ +package validation + +import ( + "fmt" + + "github.com/yannh/kubeconform/pkg/config" + "github.com/yannh/kubeconform/pkg/resource" + "github.com/yannh/kubeconform/pkg/schema" +) + +// Validator TODO +type Validator interface { + Validate(res resource.Resource) Result +} + +type validator struct { + repo *schema.Repository + conf config.Config +} + +// func (v validator) Validate(res resource.Resource) Result { +// return Result{} +// } + +// New TODO +func New(repo *schema.Repository, c config.Config) Validator { + return &validator{ + repo: repo, + conf: c, + } +} + +// Single TODO +func (v validator) Validate(res resource.Resource) Result { + sig, err := res.Signature() + if err != nil { + return Result{ + Resource: res, + Err: err, + Status: Error, + } + } + + if sig.Kind == "" { + // We skip resoures that don't have a Kind defined + return Result{ + Resource: res, + Err: nil, + Status: Skipped, + } + } + if _, ok := v.conf.SkipKinds[sig.Kind]; ok { + return Result{ + Resource: res, + Err: nil, + Status: Skipped, + } + } + + s, err := v.repo.Get(sig.Kind, sig.APIVersion, v.conf.KubernetesVersion) + if err != nil { + if v.conf.IgnoreMissingSchemas { + return Result{ + Resource: res, + Err: nil, + Status: Skipped, + } + } + + return Result{ + Resource: res, + Err: fmt.Errorf("could not find schema for %s", sig.Kind), + Status: Error, + } + } + + re, err := res.AsMap() + if err != nil { + return Result{ + Resource: res, + Err: err, + Status: Error, + } + } + + err = s.Validate(re) + status := Valid + if err != nil { + status = Error + } + return Result{ + Resource: res, + Err: err, + Status: status, + } +} diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go deleted file mode 100644 index f431895..0000000 --- a/pkg/validator/validator.go +++ /dev/null @@ -1,66 +0,0 @@ -package validator - -import ( - "fmt" - - "github.com/xeipuuv/gojsonschema" - "sigs.k8s.io/yaml" -) - -// InvalidResourceError is returned when a resource does not conform to -// the associated schema -type InvalidResourceError struct{ err string } - -func (r InvalidResourceError) Error() string { - return r.err -} - -// ValidFormat is a type for quickly forcing -// new formats on the gojsonschema loader -type ValidFormat struct{} - -// ValidFormat is a type for quickly forcing -// new formats on the gojsonschema loader -func (f ValidFormat) IsFormat(input interface{}) bool { - return true -} - -// From kubeval - let's see if absolutely necessary -// func init () { -// gojsonschema.FormatCheckers.Add("int64", ValidFormat{}) -// gojsonschema.FormatCheckers.Add("byte", ValidFormat{}) -// gojsonschema.FormatCheckers.Add("int32", ValidFormat{}) -// gojsonschema.FormatCheckers.Add("int-or-string", ValidFormat{}) -// } - -// Validate validates a single Kubernetes resource against a Json Schema -func Validate(rawResource []byte, schema *gojsonschema.Schema) error { - if schema == nil { - return nil - } - - var resource map[string]interface{} - if err := yaml.Unmarshal(rawResource, &resource); err != nil { - return fmt.Errorf("error unmarshalling resource: %s", err) - } - resourceLoader := gojsonschema.NewGoLoader(resource) - - results, err := schema.Validate(resourceLoader) - if err != nil { - // This error can only happen if the Object to validate is poorly formed. There's no hope of saving this one - return fmt.Errorf("problem validating schema. Check JSON formatting: %s", err) - } - - if results.Valid() { - return nil - } - - msg := "" - for _, errMsg := range results.Errors() { - if msg != "" { - msg += " - " - } - msg += errMsg.Description() - } - return InvalidResourceError{err: msg} -} diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go deleted file mode 100644 index f498ba6..0000000 --- a/pkg/validator/validator_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package validator - -import ( - "fmt" - "testing" - - "github.com/xeipuuv/gojsonschema" -) - -func TestValidate(t *testing.T) { - - for i, testCase := range []struct { - name string - rawResource, schema []byte - expect error - }{ - { - "valid resource", - []byte(` -firstName: foo -lastName: bar -`), - []byte(`{ - "title": "Example Schema", - "type": "object", - "properties": { - "firstName": { - "type": "string" - }, - "lastName": { - "type": "string" - }, - "age": { - "description": "Age in years", - "type": "integer", - "minimum": 0 - } - }, - "required": ["firstName", "lastName"] -}`), - nil, - }, - { - "invalid resource", - []byte(` -firstName: foo -lastName: bar -`), - []byte(`{ - "title": "Example Schema", - "type": "object", - "properties": { - "firstName": { - "type": "number" - }, - "lastName": { - "type": "string" - }, - "age": { - "description": "Age in years", - "type": "integer", - "minimum": 0 - } - }, - "required": ["firstName", "lastName"] -}`), - fmt.Errorf("Invalid type. Expected: number, given: string"), - }, - { - "missing required field", - []byte(` -firstName: foo -`), - []byte(`{ - "title": "Example Schema", - "type": "object", - "properties": { - "firstName": { - "type": "string" - }, - "lastName": { - "type": "string" - }, - "age": { - "description": "Age in years", - "type": "integer", - "minimum": 0 - } - }, - "required": ["firstName", "lastName"] -}`), - fmt.Errorf("lastName is required"), - }, - { - "resource has invalid yaml", - []byte(` -firstName foo -lastName: bar -`), - []byte(`{ - "title": "Example Schema", - "type": "object", - "properties": { - "firstName": { - "type": "number" - }, - "lastName": { - "type": "string" - }, - "age": { - "description": "Age in years", - "type": "integer", - "minimum": 0 - } - }, - "required": ["firstName", "lastName"] -}`), - fmt.Errorf("error unmarshalling resource: error converting YAML to JSON: yaml: line 3: mapping values are not allowed in this context"), - }, - } { - schema, err := gojsonschema.NewSchema(gojsonschema.NewBytesLoader(testCase.schema)) - if err != nil { - t.Errorf("failed parsing test schema") - } - if got := Validate(testCase.rawResource, schema); ((got == nil) != (testCase.expect == nil)) || (got != nil && (got.Error() != testCase.expect.Error())) { - t.Errorf("%d - expected %s, got %s", i, testCase.expect, got) - } - } -}