Merge pull request #5 from yannh/crd-support

Support for CRDs

.gitignore

@@ -1 +1,2 @@
 dist/
+bin/

Makefile

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 
-.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image
+.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image release
 
 test-build: test build
 
@@ -8,13 +8,13 @@ test:
 	go test ./...
 
 build:
-	go build -o bin/kubeconform
+	go build -o bin/ ./...
 
 docker-image:
 	docker build -t kubeconform .
 
 build-static:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o bin/kubeconform
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o bin/ ./...
 
 docker-test:
 	docker run -t -v $$PWD:/go/src/github.com/yannh/kubeconform -w /go/src/github.com/yannh/kubeconform golang:1.14 make test

Readme.md

@@ -37,12 +37,12 @@ Usage of ./bin/kubeconform:
         skip files with missing schemas instead of failing
   -k8sversion string
         version of Kubernetes to test against (default "1.18.0")
-  -local-registry value
-        folder containing additional schemas (can be specified multiple times)
   -n int
         number of routines to run in parallel (default 4)
   -output string
         output format - text, json (default "text")
+  -registry value
+        override schemas registry path (can be specified multiple times)
   -skip string
         comma-separated list of kinds to ignore
   -strict
@@ -95,6 +95,28 @@ fixtures/invalid.yaml - ReplicationController is invalid: Invalid type. Expected
 Summary: 48 resources found in 25 files - Valid: 39, Invalid: 2, Errors: 7 Skipped: 0
 ```
 
+### Overriding schemas registries lookup order - CRD support
+
+When the `-registry` file is not used, kubeconform will default to downloading schemas from
+`kubernetesjsonschema.dev`. Kubeconform however supports the use of one, or multiple, custom schemas
+registries - with access over HTTP or local filesystem. Kubeconform will lookup for schema definitions
+in each of them, in order, stopping as soon as a matching file is found.
+
+All 3 following command lines are equivalent:
+```
+$ ./bin/kubeconform fixtures/valid.yaml
+$ ./bin/kubeconform -registry kubernetesjsonschema.dev fixtures/valid.yaml
+$ ./bin/kubeconform -registry 'https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/valid.yaml
+```
+
+To support validating CRDs, we need to convert OpenAPI files to JSON schema, storing the JSON schemas
+in a local folder - for example schemas. Then we specify this folder as an additional registry to lookup:
+
+```
+# If the resource Kind is not found in kubernetesjsonschema.dev, also lookup in the schemas/ folder for a matching file
+$ ./bin/kubeconform -registry kubernetesjsonschema.dev -registry 'schemas/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/custom-resource.yaml
+```
+
 ### Credits
 
  * @garethr for the [Kubeval](https://github.com/instrumenta/kubeval) and

cmd/kubeconform/main.go

@@ -9,7 +9,6 @@ import (
 	"github.com/yannh/kubeconform/pkg/output"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"sync"
@@ -210,7 +209,7 @@ func getFiles(files []string, fileBatches chan []string, validationResults chan
 }
 
 func realMain() int {
-	var localRegistryFolders arrayParam
+	var regs arrayParam
 	var skipKindsCSV, k8sVersion, outputFormat string
 	var summary, strict, verbose, ignoreMissingSchemas bool
 	var nWorkers int
@@ -218,7 +217,7 @@ func realMain() int {
 	var files []string
 
 	flag.StringVar(&k8sVersion, "k8sversion", "1.18.0", "version of Kubernetes to test against")
-	flag.Var(&localRegistryFolders, "local-registry", "folder containing additional schemas (can be specified multiple times)")
+	flag.Var(&regs, "registry", "override schemas registry path (can be specified multiple times)")
 	flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing")
 	flag.BoolVar(&summary, "summary", false, "print a summary at the end")
 	flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel")
@@ -240,14 +239,17 @@ func realMain() int {
 	}
 
 	registries := []registry.Registry{}
-	registries = append(registries, registry.NewKubernetesRegistry(strict))
-	if len(localRegistryFolders) > 0 {
-		for _, localRegistryFolder := range localRegistryFolders {
-			localRegistry, err := registry.NewLocalRegistry(localRegistryFolder, strict)
-			if err != nil {
-				log.Fatalf("%s", err)
-			}
-			registries = append(registries, localRegistry)
+	if len(regs) == 0 {
+		regs = append(regs, "kubernetesjsonschema.dev") // if not specified, default behaviour is to use kubernetesjson-schema.dev as registry
+	}
+
+	for _, reg := range regs {
+		if reg == "kubernetesjsonschema.dev" {
+			registries = append(registries, registry.NewHTTPRegistry("https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", strict))
+		} else if strings.HasPrefix(reg, "http") {
+			registries = append(registries, registry.NewHTTPRegistry(reg, strict))
+		} else {
+			registries = append(registries, registry.NewLocalRegistry(reg, strict))
 		}
 	}
 

cmd/openapi2jsonschema/main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+import yaml
+import json
+
+def iteritems(d):
+    if hasattr(dict, "iteritems"):
+        return d.iteritems()
+    else:
+        return iter(d.items())
+
+
+def additional_properties(data):
+    "This recreates the behaviour of kubectl at https://github.com/kubernetes/kubernetes/blob/225b9119d6a8f03fcbe3cc3d590c261965d928d0/pkg/kubectl/validation/schema.go#L312"
+    new = {}
+    try:
+        for k, v in iteritems(data):
+            new_v = v
+            if isinstance(v, dict):
+                if "properties" in v:
+                    if "additionalProperties" not in v:
+                        v["additionalProperties"] = False
+                new_v = additional_properties(v)
+            else:
+                new_v = v
+            new[k] = new_v
+        return new
+    except AttributeError:
+        return data
+
+
+def replace_int_or_string(data):
+    new = {}
+    try:
+        for k, v in iteritems(data):
+            new_v = v
+            if isinstance(v, dict):
+                if "format" in v and v["format"] == "int-or-string":
+                    new_v = {"oneOf": [{"type": "string"}, {"type": "integer"}]}
+                else:
+                    new_v = replace_int_or_string(v)
+            elif isinstance(v, list):
+                new_v = list()
+                for x in v:
+                    new_v.append(replace_int_or_string(x))
+            else:
+                new_v = v
+            new[k] = new_v
+        return new
+    except AttributeError:
+        return data
+
+
+def allow_null_optional_fields(data, parent=None, grand_parent=None, key=None):
+    new = {}
+    try:
+        for k, v in iteritems(data):
+            new_v = v
+            if isinstance(v, dict):
+                new_v = allow_null_optional_fields(v, data, parent, k)
+            elif isinstance(v, list):
+                new_v = list()
+                for x in v:
+                    new_v.append(allow_null_optional_fields(x, v, parent, k))
+            elif isinstance(v, str):
+                is_non_null_type = k == "type" and v != "null"
+                has_required_fields = grand_parent and "required" in grand_parent
+                if is_non_null_type and not has_required_field:
+                    new_v = [v, "null"]
+            new[k] = new_v
+        return new
+    except AttributeError:
+        return data
+
+
+def append_no_duplicates(obj, key, value):
+    """
+    Given a dictionary, lookup the given key, if it doesn't exist create a new array.
+    Then check if the given value already exists in the array, if it doesn't add it.
+    """
+    if key not in obj:
+        obj[key] = []
+    if value not in obj[key]:
+        obj[key].append(value)
+
+with open(r'synced_secrets.yaml') as f:
+    y = yaml.load(f, Loader=yaml.SafeLoader)
+    schema = y["spec"]["validation"]["openAPIV3Schema"]
+    schema = additional_properties(schema)
+    schema = replace_int_or_string(schema)
+    print(json.dumps(schema))

pkg/registry/kubernetesjsonschema.go

@@ -7,8 +7,8 @@ import (
 )
 
 type KubernetesRegistry struct {
-	baseURL string
-	strict  bool
+	schemaPathTemplate string
+	strict             bool
 }
 
 type downloadError struct {
@@ -22,15 +22,18 @@ func newDownloadError(err error, isRetryable bool) *downloadError {
 func (e *downloadError) IsRetryable() bool { return e.isRetryable }
 func (e *downloadError) Error() string     { return e.err.Error() }
 
-func NewKubernetesRegistry(strict bool) *KubernetesRegistry {
+func NewHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry {
 	return &KubernetesRegistry{
-		baseURL: "https://kubernetesjsonschema.dev",
-		strict:  strict,
+		schemaPathTemplate: schemaPathTemplate,
+		strict:             strict,
 	}
 }
 
 func (r KubernetesRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) {
-	url := r.baseURL + "/" + schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict)
+	url, err := schemaPath(r.schemaPathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict)
+	if err != nil {
+		return nil, err
+	}
 
 	resp, err := http.Get(url)
 	if err != nil {

pkg/registry/local.go

@@ -4,30 +4,47 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
 )
 
 type LocalRegistry struct {
-	folder string
-	strict bool
+	pathTemplate string
+	strict       bool
 }
 
+type fileNotFoundError struct {
+	err         error
+	isRetryable bool
+}
+
+func newFileNotFoundError(err error, isRetryable bool) *fileNotFoundError {
+	return &fileNotFoundError{err, isRetryable}
+}
+func (e *fileNotFoundError) IsRetryable() bool { return e.isRetryable }
+func (e *fileNotFoundError) Error() string     { return e.err.Error() }
+
 // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames
-func NewLocalRegistry(folder string, strict bool) (*LocalRegistry, error) {
+func NewLocalRegistry(pathTemplate string, strict bool) *LocalRegistry {
 	return &LocalRegistry{
-		folder,
+		pathTemplate,
 		strict,
-	}, nil
+	}
 }
 
 // DownloadSchema retrieves the schema from a file for the resource
 func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) {
-	schemaFile := path.Join(r.folder, schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict))
+	schemaFile, err := schemaPath(r.pathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict)
+	if err != nil {
+		return []byte{}, nil
+	}
 
 	f, err := os.Open(schemaFile)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, newFileNotFoundError(fmt.Errorf("no schema found"), false)
+		}
 		return nil, fmt.Errorf("failed to open schema %s", schemaFile)
 	}
+
 	defer f.Close()
 	content, err := ioutil.ReadAll(f)
 	if err != nil {

pkg/registry/registry.go

@@ -1,8 +1,9 @@
 package registry
 
 import (
-	"fmt"
+	"bytes"
 	"strings"
+	"text/template"
 )
 
 type Manifest struct {
@@ -19,7 +20,7 @@ type Retryable interface {
 	IsRetryable() bool
 }
 
-func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool) string {
+func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict bool) (string, error) {
 	normalisedVersion := k8sVersion
 	if normalisedVersion != "master" {
 		normalisedVersion = "v" + normalisedVersion
@@ -38,5 +39,28 @@ func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool
 		kindSuffix += "-" + strings.ToLower(groupParts[1])
 	}
 
-	return fmt.Sprintf("%s-standalone%s/%s%s.json", normalisedVersion, strictSuffix, strings.ToLower(resourceKind), kindSuffix)
+	tmpl, err := template.New("tpl").Parse(tpl)
+	if err != nil {
+		return "", err
+	}
+
+	tplData := struct {
+		NormalizedVersion string
+		StrictSuffix      string
+		ResourceKind      string
+		KindSuffix        string
+	}{
+		normalisedVersion,
+		strictSuffix,
+		strings.ToLower(resourceKind),
+		kindSuffix,
+	}
+
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, tplData)
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
 }

pkg/registry/registry_test.go

@@ -6,39 +6,52 @@ import (
 
 func TestSchemaPath(t *testing.T) {
 	for i, testCase := range []struct {
-		resourceKind, resourceAPIVersion, k8sVersion, expected string
-		strict                                                 bool
+		tpl, resourceKind, resourceAPIVersion, k8sVersion, expected string
+		strict                                                      bool
+		errExpected                                                 error
 	}{
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Deployment",
 			"apps/v1",
 			"1.16.0",
-			"v1.16.0-standalone-strict/deployment-apps-v1.json",
+			"https://kubernetesjsonschema.dev/v1.16.0-standalone-strict/deployment-apps-v1.json",
 			true,
+			nil,
 		},
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Deployment",
 			"apps/v1",
 			"1.16.0",
-			"v1.16.0-standalone/deployment-apps-v1.json",
+			"https://kubernetesjsonschema.dev/v1.16.0-standalone/deployment-apps-v1.json",
 			false,
+			nil,
 		},
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Service",
 			"v1",
 			"1.18.0",
-			"v1.18.0-standalone/service-v1.json",
+			"https://kubernetesjsonschema.dev/v1.18.0-standalone/service-v1.json",
 			false,
+			nil,
 		},
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Service",
 			"v1",
 			"master",
-			"master-standalone/service-v1.json",
+			"https://kubernetesjsonschema.dev/master-standalone/service-v1.json",
 			false,
+			nil,
 		},
 	} {
-		if got := schemaPath(testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict); got != testCase.expected {
+		got, err := schemaPath(testCase.tpl, testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict)
+		if err != testCase.errExpected {
+			t.Errorf("%d - got error %s, expected %s", i+1, err, testCase.errExpected)
+		}
+		if got != testCase.expected {
 			t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected)
 		}
 	}

pkg/validator/validator_test.go

@@ -65,6 +65,31 @@ lastName: bar
 }`),
 			fmt.Errorf("Invalid type. Expected: number, given: string"),
 		},
+		{
+			"missing required field",
+			[]byte(`
+firstName: foo
+`),
+			[]byte(`{
+  "title": "Example Schema",
+  "type": "object",
+  "properties": {
+    "firstName": {
+      "type": "string"
+    },
+    "lastName": {
+      "type": "string"
+    },
+    "age": {
+      "description": "Age in years",
+      "type": "integer",
+      "minimum": 0
+    }
+  },
+  "required": ["firstName", "lastName"]
+}`),
+			fmt.Errorf("lastName is required"),
+		},
 		{
 			"resource has invalid yaml",
 			[]byte(`