Support for CRDs

cmd/kubeconform/main.go

@@ -0,0 +1,318 @@
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"github.com/xeipuuv/gojsonschema"
+	"github.com/yannh/kubeconform/pkg/fsutils"
+	"github.com/yannh/kubeconform/pkg/output"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/yannh/kubeconform/pkg/cache"
+	"github.com/yannh/kubeconform/pkg/registry"
+	"github.com/yannh/kubeconform/pkg/resource"
+	"github.com/yannh/kubeconform/pkg/validator"
+)
+
+type validationResult struct {
+	filename, kind, version string
+	err                     error
+	skipped                 bool
+}
+
+func resourcesFromReader(r io.Reader) ([][]byte, error) {
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return [][]byte{}, err
+	}
+
+	resources := bytes.Split(data, []byte("---\n"))
+
+	return resources, nil
+}
+
+func downloadSchema(registries []registry.Registry, kind, version, k8sVersion string) (*gojsonschema.Schema, error) {
+	var err error
+	var schemaBytes []byte
+
+	for _, reg := range registries {
+		schemaBytes, err = reg.DownloadSchema(kind, version, k8sVersion)
+		if err == nil {
+			return gojsonschema.NewSchema(gojsonschema.NewBytesLoader(schemaBytes))
+		}
+
+		// If we get a 404, we try the next registry, but we exit if we get a real failure
+		if er, retryable := err.(registry.Retryable); retryable && !er.IsRetryable() {
+			continue
+		}
+
+		return nil, err
+	}
+
+	return nil, nil // No schema found - we don't consider it an error, resource will be skipped
+}
+
+// filter returns true if the file should be skipped
+// Returning an array, this Reader might container multiple resources
+func ValidateStream(r io.Reader, regs []registry.Registry, k8sVersion string, c *cache.SchemaCache, skip func(signature resource.Signature) bool, ignoreMissingSchemas bool) []validationResult {
+	rawResources, err := resourcesFromReader(r)
+	if err != nil {
+		return []validationResult{{err: fmt.Errorf("failed reading file: %s", err)}}
+	}
+
+	validationResults := []validationResult{}
+
+	for _, rawResource := range rawResources {
+		var sig resource.Signature
+		if sig, err = resource.SignatureFromBytes(rawResource); err != nil {
+			validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, err: fmt.Errorf("error while parsing: %s", err)})
+			continue
+		}
+
+		if sig.Kind == "" {
+			continue // We skip resoures that don't have a Kind defined
+		}
+
+		if skip(sig) {
+			validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, err: nil, skipped: true})
+			continue
+		}
+
+		ok := false
+		var schema *gojsonschema.Schema
+		cacheKey := ""
+
+		if c != nil {
+			cacheKey = cache.Key(sig.Kind, sig.Version, k8sVersion)
+			schema, ok = c.Get(cacheKey)
+		}
+
+		if !ok {
+			schema, err = downloadSchema(regs, sig.Kind, sig.Version, k8sVersion)
+			if err != nil {
+				validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, err: err, skipped: false})
+				continue
+			}
+
+			if c != nil {
+				c.Set(cacheKey, schema)
+			}
+		}
+
+		if schema == nil {
+			if ignoreMissingSchemas {
+				validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, err: nil, skipped: true})
+			} else {
+				validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, err: fmt.Errorf("could not find schema for %s", sig.Kind), skipped: false})
+			}
+		}
+
+		err = validator.Validate(rawResource, schema)
+		validationResults = append(validationResults, validationResult{kind: sig.Kind, version: sig.Version, err: err})
+	}
+
+	return validationResults
+}
+
+type arrayParam []string
+
+func (ap *arrayParam) String() string {
+	return strings.Join(*ap, " - ")
+}
+
+func (ap *arrayParam) Set(value string) error {
+	*ap = append(*ap, value)
+	return nil
+}
+
+func getLogger(outputFormat string, printSummary, verbose bool) (output.Output, error) {
+	w := os.Stdout
+
+	switch {
+	case outputFormat == "text":
+		return output.Text(w, printSummary, verbose), nil
+	case outputFormat == "json":
+		return output.JSON(w, printSummary, verbose), nil
+	default:
+		return nil, fmt.Errorf("-output must be text or json")
+	}
+}
+
+func skipKindsMap(skipKindsCSV string) map[string]bool {
+	splitKinds := strings.Split(skipKindsCSV, ",")
+	skipKinds := map[string]bool{}
+	for _, kind := range splitKinds {
+		if len(kind) > 0 {
+			skipKinds[kind] = true
+		}
+	}
+	return skipKinds
+}
+
+func processResults(o output.Output, validationResults chan []validationResult, result chan<- bool) {
+	success := true
+	for results := range validationResults {
+		for _, result := range results {
+			if result.err != nil {
+				success = false
+			}
+
+			if err := o.Write(result.filename, result.kind, result.version, result.err, result.skipped); err != nil {
+				fmt.Fprint(os.Stderr, "failed writing log\n")
+			}
+		}
+	}
+
+	result <- success
+}
+
+func getFiles(files []string, fileBatches chan []string, validationResults chan []validationResult) {
+	for _, filename := range files {
+		file, err := os.Open(filename)
+		if err != nil {
+			validationResults <- []validationResult{{
+				filename: filename,
+				err:      err,
+				skipped:  false,
+			}}
+			continue
+		}
+		defer file.Close()
+
+		fi, err := file.Stat()
+		switch {
+		case err != nil:
+			validationResults <- []validationResult{{
+				filename: filename,
+				err:      err,
+				skipped:  false,
+			}}
+
+		case fi.IsDir():
+			if err := fsutils.FindYamlInDir(filename, fileBatches, 10); err != nil {
+				validationResults <- []validationResult{{
+					filename: filename,
+					err:      err,
+					skipped:  false,
+				}}
+			}
+
+		default:
+			fileBatches <- []string{filename}
+		}
+	}
+}
+
+func realMain() int {
+	var regs arrayParam
+	var skipKindsCSV, k8sVersion, outputFormat string
+	var summary, strict, verbose, ignoreMissingSchemas bool
+	var nWorkers int
+	var err error
+	var files []string
+
+	flag.StringVar(&k8sVersion, "k8sversion", "1.18.0", "version of Kubernetes to test against")
+	flag.Var(&regs, "registry", "override schemas registry path (can be specified multiple times)")
+	flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing")
+	flag.BoolVar(&summary, "summary", false, "print a summary at the end")
+	flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel")
+	flag.StringVar(&skipKindsCSV, "skip", "", "comma-separated list of kinds to ignore")
+	flag.BoolVar(&strict, "strict", false, "disallow additional properties not in schema")
+	flag.StringVar(&outputFormat, "output", "text", "output format - text, json")
+	flag.BoolVar(&verbose, "verbose", false, "print results for all resources")
+	flag.Parse()
+
+	skipKinds := skipKindsMap(skipKindsCSV)
+
+	for _, file := range flag.Args() {
+		files = append(files, file)
+	}
+
+	filter := func(signature resource.Signature) bool {
+		isSkipKind, ok := skipKinds[signature.Kind]
+		return ok && isSkipKind
+	}
+
+	registries := []registry.Registry{}
+	if len(regs) == 0 {
+		regs = append(regs, "kubernetesjsonschema.dev") // if not specified, default behaviour is to use kubernetesjson-schema.dev as registry
+	}
+
+	for _, reg := range regs {
+		if reg == "kubernetesjsonschema.dev" {
+			registries = append(registries, registry.NewHTTPRegistry("https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", strict))
+		} else if strings.HasPrefix(reg, "http") {
+			registries = append(registries, registry.NewHTTPRegistry(reg, strict))
+		} else {
+			registries = append(registries, registry.NewLocalRegistry(reg, strict))
+		}
+	}
+
+	validationResults := make(chan []validationResult)
+
+	fileBatches := make(chan []string)
+	go func() {
+		getFiles(files, fileBatches, validationResults)
+		close(fileBatches)
+	}()
+
+	var o output.Output
+	if o, err = getLogger(outputFormat, summary, verbose); err != nil {
+		fmt.Println(err)
+		return 1
+	}
+
+	res := make(chan bool)
+	go processResults(o, validationResults, res)
+
+	c := cache.New()
+	var wg sync.WaitGroup
+	for i := 0; i < nWorkers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			for fileBatch := range fileBatches {
+				for _, filename := range fileBatch {
+					f, err := os.Open(filename)
+					if err != nil {
+						validationResults <- []validationResult{{
+							filename: filename,
+							err:      err,
+							skipped:  true,
+						}}
+						continue
+					}
+
+					res := ValidateStream(f, registries, k8sVersion, c, filter, ignoreMissingSchemas)
+					f.Close()
+
+					for i := range res {
+						res[i].filename = filename
+					}
+					validationResults <- res
+				}
+			}
+		}()
+	}
+
+	wg.Wait()
+	close(validationResults)
+	success := <-res
+	o.Flush()
+
+	if !success {
+		return 1
+	}
+
+	return 0
+}
+
+func main() {
+	os.Exit(realMain())
+}

cmd/kubeconform/main_test.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestSkipKindMaps(t *testing.T) {
+	for _, testCase := range []struct {
+		name         string
+		csvSkipKinds string
+		expect       map[string]bool
+	}{
+		{
+			"nothing to skip",
+			"",
+			map[string]bool{},
+		},
+		{
+			"a single kind to skip",
+			"somekind",
+			map[string]bool{
+				"somekind": true,
+			},
+		},
+		{
+			"multiple kinds to skip",
+			"somekind,anotherkind,yetsomeotherkind",
+			map[string]bool{
+				"somekind":         true,
+				"anotherkind":      true,
+				"yetsomeotherkind": true,
+			},
+		},
+	} {
+		got := skipKindsMap(testCase.csvSkipKinds)
+		if !reflect.DeepEqual(got, testCase.expect) {
+			t.Errorf("%s - got %+v, expected %+v", testCase.name, got, testCase.expect)
+		}
+	}
+}

cmd/openapi2jsonschema/main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+import yaml
+import json
+
+def iteritems(d):
+    if hasattr(dict, "iteritems"):
+        return d.iteritems()
+    else:
+        return iter(d.items())
+
+
+def additional_properties(data):
+    "This recreates the behaviour of kubectl at https://github.com/kubernetes/kubernetes/blob/225b9119d6a8f03fcbe3cc3d590c261965d928d0/pkg/kubectl/validation/schema.go#L312"
+    new = {}
+    try:
+        for k, v in iteritems(data):
+            new_v = v
+            if isinstance(v, dict):
+                if "properties" in v:
+                    if "additionalProperties" not in v:
+                        v["additionalProperties"] = False
+                new_v = additional_properties(v)
+            else:
+                new_v = v
+            new[k] = new_v
+        return new
+    except AttributeError:
+        return data
+
+
+def replace_int_or_string(data):
+    new = {}
+    try:
+        for k, v in iteritems(data):
+            new_v = v
+            if isinstance(v, dict):
+                if "format" in v and v["format"] == "int-or-string":
+                    new_v = {"oneOf": [{"type": "string"}, {"type": "integer"}]}
+                else:
+                    new_v = replace_int_or_string(v)
+            elif isinstance(v, list):
+                new_v = list()
+                for x in v:
+                    new_v.append(replace_int_or_string(x))
+            else:
+                new_v = v
+            new[k] = new_v
+        return new
+    except AttributeError:
+        return data
+
+
+def allow_null_optional_fields(data, parent=None, grand_parent=None, key=None):
+    new = {}
+    try:
+        for k, v in iteritems(data):
+            new_v = v
+            if isinstance(v, dict):
+                new_v = allow_null_optional_fields(v, data, parent, k)
+            elif isinstance(v, list):
+                new_v = list()
+                for x in v:
+                    new_v.append(allow_null_optional_fields(x, v, parent, k))
+            elif isinstance(v, str):
+                is_non_null_type = k == "type" and v != "null"
+                has_required_fields = grand_parent and "required" in grand_parent
+                if is_non_null_type and not has_required_field:
+                    new_v = [v, "null"]
+            new[k] = new_v
+        return new
+    except AttributeError:
+        return data
+
+
+def append_no_duplicates(obj, key, value):
+    """
+    Given a dictionary, lookup the given key, if it doesn't exist create a new array.
+    Then check if the given value already exists in the array, if it doesn't add it.
+    """
+    if key not in obj:
+        obj[key] = []
+    if value not in obj[key]:
+        obj[key].append(value)
+
+with open(r'synced_secrets.yaml') as f:
+    y = yaml.load(f, Loader=yaml.SafeLoader)
+    schema = y["spec"]["validation"]["openAPIV3Schema"]
+    schema = additional_properties(schema)
+    schema = replace_int_or_string(schema)
+    print(json.dumps(schema))