support templated schema registries

Makefile

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 
-.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image
+.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image release
 
 test-build: test build
 

cmd/kubeconform/main.go

@@ -9,7 +9,6 @@ import (
 	"github.com/yannh/kubeconform/pkg/output"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"strings"
 	"sync"
@@ -210,7 +209,7 @@ func getFiles(files []string, fileBatches chan []string, validationResults chan
 }
 
 func realMain() int {
-	var localRegistryFolders arrayParam
+	var regs arrayParam
 	var skipKindsCSV, k8sVersion, outputFormat string
 	var summary, strict, verbose, ignoreMissingSchemas bool
 	var nWorkers int
@@ -218,7 +217,7 @@ func realMain() int {
 	var files []string
 
 	flag.StringVar(&k8sVersion, "k8sversion", "1.18.0", "version of Kubernetes to test against")
-	flag.Var(&localRegistryFolders, "local-registry", "folder containing additional schemas (can be specified multiple times)")
+	flag.Var(&regs, "registry", "filepath template for registry")
 	flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing")
 	flag.BoolVar(&summary, "summary", false, "print a summary at the end")
 	flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel")
@@ -240,14 +239,13 @@ func realMain() int {
 	}
 
 	registries := []registry.Registry{}
-	registries = append(registries, registry.NewKubernetesRegistry(strict))
+	for _, reg := range regs {
-	if len(localRegistryFolders) > 0 {
+		if reg == "kubernetesjsonschema.dev" {
-		for _, localRegistryFolder := range localRegistryFolders {
+			registries = append(registries, registry.NewHTTPRegistry("https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", strict))
-			localRegistry, err := registry.NewLocalRegistry(localRegistryFolder, strict)
+		} else if strings.HasPrefix(reg, "http") {
-			if err != nil {
+			registries = append(registries, registry.NewHTTPRegistry(reg, strict))
-				log.Fatalf("%s", err)
+		} else {
-			}
+			registries = append(registries, registry.NewLocalRegistry(reg, strict))
-			registries = append(registries, localRegistry)
 		}
 	}
 

cmd/openapi2jsonschema/main.py

@@ -65,10 +65,7 @@ def allow_null_optional_fields(data, parent=None, grand_parent=None, key=None):
             elif isinstance(v, str):
                 is_non_null_type = k == "type" and v != "null"
                 has_required_fields = grand_parent and "required" in grand_parent
-                is_required_field = (
+                if is_non_null_type and not has_required_field:
-                    has_required_fields and key in grand_parent["required"]
-                )
-                if is_non_null_type and not is_required_field:
                     new_v = [v, "null"]
             new[k] = new_v
         return new
@@ -91,5 +88,4 @@ with open(r'synced_secrets.yaml') as f:
     schema = y["spec"]["validation"]["openAPIV3Schema"]
     schema = additional_properties(schema)
     schema = replace_int_or_string(schema)
-    schema = allow_null_optional_fields(schema)
     print(json.dumps(schema))

pkg/registry/kubernetesjsonschema.go

@@ -7,8 +7,8 @@ import (
 )
 
 type KubernetesRegistry struct {
-	baseURL string
+	schemaPathTemplate string
-	strict  bool
+	strict             bool
 }
 
 type downloadError struct {
@@ -22,15 +22,18 @@ func newDownloadError(err error, isRetryable bool) *downloadError {
 func (e *downloadError) IsRetryable() bool { return e.isRetryable }
 func (e *downloadError) Error() string     { return e.err.Error() }
 
-func NewKubernetesRegistry(strict bool) *KubernetesRegistry {
+func NewHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry {
 	return &KubernetesRegistry{
-		baseURL: "https://kubernetesjsonschema.dev",
+		schemaPathTemplate: schemaPathTemplate,
-		strict:  strict,
+		strict:             strict,
 	}
 }
 
 func (r KubernetesRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) {
-	url := r.baseURL + "/" + schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict)
+	url, err := schemaPath(r.schemaPathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict)
+	if err != nil {
+		return nil, err
+	}
 
 	resp, err := http.Get(url)
 	if err != nil {

pkg/registry/local.go

@@ -4,30 +4,47 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
 )
 
 type LocalRegistry struct {
-	folder string
+	pathTemplate string
-	strict bool
+	strict       bool
 }
 
+type fileNotFoundError struct {
+	err         error
+	isRetryable bool
+}
+
+func newFileNotFoundError(err error, isRetryable bool) *fileNotFoundError {
+	return &fileNotFoundError{err, isRetryable}
+}
+func (e *fileNotFoundError) IsRetryable() bool { return e.isRetryable }
+func (e *fileNotFoundError) Error() string     { return e.err.Error() }
+
 // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames
-func NewLocalRegistry(folder string, strict bool) (*LocalRegistry, error) {
+func NewLocalRegistry(pathTemplate string, strict bool) *LocalRegistry {
 	return &LocalRegistry{
-		folder,
+		pathTemplate,
 		strict,
-	}, nil
+	}
 }
 
 // DownloadSchema retrieves the schema from a file for the resource
 func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) {
-	schemaFile := path.Join(r.folder, schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict))
+	schemaFile, err := schemaPath(r.pathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict)
+	if err != nil {
+		return []byte{}, nil
+	}
 
 	f, err := os.Open(schemaFile)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, newFileNotFoundError(fmt.Errorf("no schema found"), false)
+		}
 		return nil, fmt.Errorf("failed to open schema %s", schemaFile)
 	}
+
 	defer f.Close()
 	content, err := ioutil.ReadAll(f)
 	if err != nil {

pkg/registry/registry.go

@@ -1,8 +1,9 @@
 package registry
 
 import (
-	"fmt"
+	"bytes"
 	"strings"
+	"text/template"
 )
 
 type Manifest struct {
@@ -19,7 +20,7 @@ type Retryable interface {
 	IsRetryable() bool
 }
 
-func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool) string {
+func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict bool) (string, error) {
 	normalisedVersion := k8sVersion
 	if normalisedVersion != "master" {
 		normalisedVersion = "v" + normalisedVersion
@@ -38,5 +39,28 @@ func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool
 		kindSuffix += "-" + strings.ToLower(groupParts[1])
 	}
 
-	return fmt.Sprintf("%s-standalone%s/%s%s.json", normalisedVersion, strictSuffix, strings.ToLower(resourceKind), kindSuffix)
+	tmpl, err := template.New("tpl").Parse(tpl)
+	if err != nil {
+		return "", err
+	}
+
+	tplData := struct {
+		NormalizedVersion string
+		StrictSuffix      string
+		ResourceKind      string
+		KindSuffix        string
+	}{
+		normalisedVersion,
+		strictSuffix,
+		strings.ToLower(resourceKind),
+		kindSuffix,
+	}
+
+	var buf bytes.Buffer
+	err = tmpl.Execute(&buf, tplData)
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
 }

pkg/registry/registry_test.go

@@ -6,39 +6,52 @@ import (
 
 func TestSchemaPath(t *testing.T) {
 	for i, testCase := range []struct {
-		resourceKind, resourceAPIVersion, k8sVersion, expected string
+		tpl, resourceKind, resourceAPIVersion, k8sVersion, expected string
-		strict                                                 bool
+		strict                                                      bool
+		errExpected                                                 error
 	}{
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Deployment",
 			"apps/v1",
 			"1.16.0",
-			"v1.16.0-standalone-strict/deployment-apps-v1.json",
+			"https://kubernetesjsonschema.dev/v1.16.0-standalone-strict/deployment-apps-v1.json",
 			true,
+			nil,
 		},
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Deployment",
 			"apps/v1",
 			"1.16.0",
-			"v1.16.0-standalone/deployment-apps-v1.json",
+			"https://kubernetesjsonschema.dev/v1.16.0-standalone/deployment-apps-v1.json",
 			false,
+			nil,
 		},
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Service",
 			"v1",
 			"1.18.0",
-			"v1.18.0-standalone/service-v1.json",
+			"https://kubernetesjsonschema.dev/v1.18.0-standalone/service-v1.json",
 			false,
+			nil,
 		},
 		{
+			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Service",
 			"v1",
 			"master",
-			"master-standalone/service-v1.json",
+			"https://kubernetesjsonschema.dev/master-standalone/service-v1.json",
 			false,
+			nil,
 		},
 	} {
-		if got := schemaPath(testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict); got != testCase.expected {
+		got, err := schemaPath(testCase.tpl, testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict)
+		if err != testCase.errExpected {
+			t.Errorf("%d - got error %s, expected %s", i+1, err, testCase.errExpected)
+		}
+		if got != testCase.expected {
 			t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected)
 		}
 	}

pkg/validator/validator_test.go

@@ -65,6 +65,31 @@ lastName: bar
 }`),
 			fmt.Errorf("Invalid type. Expected: number, given: string"),
 		},
+		{
+			"missing required field",
+			[]byte(`
+firstName: foo
+`),
+			[]byte(`{
+  "title": "Example Schema",
+  "type": "object",
+  "properties": {
+    "firstName": {
+      "type": "string"
+    },
+    "lastName": {
+      "type": "string"
+    },
+    "age": {
+      "description": "Age in years",
+      "type": "integer",
+      "minimum": 0
+    }
+  },
+  "required": ["firstName", "lastName"]
+}`),
+			fmt.Errorf("lastName is required"),
+		},
 		{
 			"resource has invalid yaml",
 			[]byte(`