diff --git a/.gitignore b/.gitignore index 849ddff..62cb519 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ dist/ +bin/ diff --git a/Makefile b/Makefile index 381c7f6..51f4ac7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f -.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image +.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image release test-build: test build @@ -8,13 +8,13 @@ test: go test ./... build: - go build -o bin/kubeconform + go build -o bin/ ./... docker-image: docker build -t kubeconform . build-static: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o bin/kubeconform + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o bin/ ./... docker-test: docker run -t -v $$PWD:/go/src/github.com/yannh/kubeconform -w /go/src/github.com/yannh/kubeconform golang:1.14 make test diff --git a/Readme.md b/Readme.md index e61ed88..f419427 100644 --- a/Readme.md +++ b/Readme.md @@ -37,12 +37,12 @@ Usage of ./bin/kubeconform: skip files with missing schemas instead of failing -k8sversion string version of Kubernetes to test against (default "1.18.0") - -local-registry value - folder containing additional schemas (can be specified multiple times) -n int number of routines to run in parallel (default 4) -output string output format - text, json (default "text") + -registry value + override schemas registry path (can be specified multiple times) -skip string comma-separated list of kinds to ignore -strict @@ -95,6 +95,28 @@ fixtures/invalid.yaml - ReplicationController is invalid: Invalid type. Expected Summary: 48 resources found in 25 files - Valid: 39, Invalid: 2, Errors: 7 Skipped: 0 ``` +### Overriding schemas registries lookup order - CRD support + +When the `-registry` file is not used, kubeconform will default to downloading schemas from +`kubernetesjsonschema.dev`. Kubeconform however supports the use of one, or multiple, custom schemas +registries - with access over HTTP or local filesystem. Kubeconform will lookup for schema definitions +in each of them, in order, stopping as soon as a matching file is found. + +All 3 following command lines are equivalent: +``` +$ ./bin/kubeconform fixtures/valid.yaml +$ ./bin/kubeconform -registry kubernetesjsonschema.dev fixtures/valid.yaml +$ ./bin/kubeconform -registry 'https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/valid.yaml +``` + +To support validating CRDs, we need to convert OpenAPI files to JSON schema, storing the JSON schemas +in a local folder - for example schemas. Then we specify this folder as an additional registry to lookup: + +``` +# If the resource Kind is not found in kubernetesjsonschema.dev, also lookup in the schemas/ folder for a matching file +$ ./bin/kubeconform -registry kubernetesjsonschema.dev -registry 'schemas/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/custom-resource.yaml +``` + ### Credits * @garethr for the [Kubeval](https://github.com/instrumenta/kubeval) and diff --git a/main.go b/cmd/kubeconform/main.go similarity index 91% rename from main.go rename to cmd/kubeconform/main.go index 31d10e6..a9e2a7a 100644 --- a/main.go +++ b/cmd/kubeconform/main.go @@ -9,7 +9,6 @@ import ( "github.com/yannh/kubeconform/pkg/output" "io" "io/ioutil" - "log" "os" "strings" "sync" @@ -210,7 +209,7 @@ func getFiles(files []string, fileBatches chan []string, validationResults chan } func realMain() int { - var localRegistryFolders arrayParam + var regs arrayParam var skipKindsCSV, k8sVersion, outputFormat string var summary, strict, verbose, ignoreMissingSchemas bool var nWorkers int @@ -218,7 +217,7 @@ func realMain() int { var files []string flag.StringVar(&k8sVersion, "k8sversion", "1.18.0", "version of Kubernetes to test against") - flag.Var(&localRegistryFolders, "local-registry", "folder containing additional schemas (can be specified multiple times)") + flag.Var(®s, "registry", "override schemas registry path (can be specified multiple times)") flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing") flag.BoolVar(&summary, "summary", false, "print a summary at the end") flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel") @@ -240,14 +239,17 @@ func realMain() int { } registries := []registry.Registry{} - registries = append(registries, registry.NewKubernetesRegistry(strict)) - if len(localRegistryFolders) > 0 { - for _, localRegistryFolder := range localRegistryFolders { - localRegistry, err := registry.NewLocalRegistry(localRegistryFolder, strict) - if err != nil { - log.Fatalf("%s", err) - } - registries = append(registries, localRegistry) + if len(regs) == 0 { + regs = append(regs, "kubernetesjsonschema.dev") // if not specified, default behaviour is to use kubernetesjson-schema.dev as registry + } + + for _, reg := range regs { + if reg == "kubernetesjsonschema.dev" { + registries = append(registries, registry.NewHTTPRegistry("https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", strict)) + } else if strings.HasPrefix(reg, "http") { + registries = append(registries, registry.NewHTTPRegistry(reg, strict)) + } else { + registries = append(registries, registry.NewLocalRegistry(reg, strict)) } } diff --git a/main_test.go b/cmd/kubeconform/main_test.go similarity index 100% rename from main_test.go rename to cmd/kubeconform/main_test.go diff --git a/cmd/openapi2jsonschema/main.py b/cmd/openapi2jsonschema/main.py new file mode 100755 index 0000000..b0716d8 --- /dev/null +++ b/cmd/openapi2jsonschema/main.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +import yaml +import json + +def iteritems(d): + if hasattr(dict, "iteritems"): + return d.iteritems() + else: + return iter(d.items()) + + +def additional_properties(data): + "This recreates the behaviour of kubectl at https://github.com/kubernetes/kubernetes/blob/225b9119d6a8f03fcbe3cc3d590c261965d928d0/pkg/kubectl/validation/schema.go#L312" + new = {} + try: + for k, v in iteritems(data): + new_v = v + if isinstance(v, dict): + if "properties" in v: + if "additionalProperties" not in v: + v["additionalProperties"] = False + new_v = additional_properties(v) + else: + new_v = v + new[k] = new_v + return new + except AttributeError: + return data + + +def replace_int_or_string(data): + new = {} + try: + for k, v in iteritems(data): + new_v = v + if isinstance(v, dict): + if "format" in v and v["format"] == "int-or-string": + new_v = {"oneOf": [{"type": "string"}, {"type": "integer"}]} + else: + new_v = replace_int_or_string(v) + elif isinstance(v, list): + new_v = list() + for x in v: + new_v.append(replace_int_or_string(x)) + else: + new_v = v + new[k] = new_v + return new + except AttributeError: + return data + + +def allow_null_optional_fields(data, parent=None, grand_parent=None, key=None): + new = {} + try: + for k, v in iteritems(data): + new_v = v + if isinstance(v, dict): + new_v = allow_null_optional_fields(v, data, parent, k) + elif isinstance(v, list): + new_v = list() + for x in v: + new_v.append(allow_null_optional_fields(x, v, parent, k)) + elif isinstance(v, str): + is_non_null_type = k == "type" and v != "null" + has_required_fields = grand_parent and "required" in grand_parent + if is_non_null_type and not has_required_field: + new_v = [v, "null"] + new[k] = new_v + return new + except AttributeError: + return data + + +def append_no_duplicates(obj, key, value): + """ + Given a dictionary, lookup the given key, if it doesn't exist create a new array. + Then check if the given value already exists in the array, if it doesn't add it. + """ + if key not in obj: + obj[key] = [] + if value not in obj[key]: + obj[key].append(value) + +with open(r'synced_secrets.yaml') as f: + y = yaml.load(f, Loader=yaml.SafeLoader) + schema = y["spec"]["validation"]["openAPIV3Schema"] + schema = additional_properties(schema) + schema = replace_int_or_string(schema) + print(json.dumps(schema)) \ No newline at end of file diff --git a/pkg/registry/kubernetesjsonschema.go b/pkg/registry/kubernetesjsonschema.go index 9ac8ed1..e29e314 100644 --- a/pkg/registry/kubernetesjsonschema.go +++ b/pkg/registry/kubernetesjsonschema.go @@ -7,8 +7,8 @@ import ( ) type KubernetesRegistry struct { - baseURL string - strict bool + schemaPathTemplate string + strict bool } type downloadError struct { @@ -22,15 +22,18 @@ func newDownloadError(err error, isRetryable bool) *downloadError { func (e *downloadError) IsRetryable() bool { return e.isRetryable } func (e *downloadError) Error() string { return e.err.Error() } -func NewKubernetesRegistry(strict bool) *KubernetesRegistry { +func NewHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry { return &KubernetesRegistry{ - baseURL: "https://kubernetesjsonschema.dev", - strict: strict, + schemaPathTemplate: schemaPathTemplate, + strict: strict, } } func (r KubernetesRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - url := r.baseURL + "/" + schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict) + url, err := schemaPath(r.schemaPathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict) + if err != nil { + return nil, err + } resp, err := http.Get(url) if err != nil { diff --git a/pkg/registry/local.go b/pkg/registry/local.go index 5c4561d..67d2460 100644 --- a/pkg/registry/local.go +++ b/pkg/registry/local.go @@ -4,30 +4,47 @@ import ( "fmt" "io/ioutil" "os" - "path" ) type LocalRegistry struct { - folder string - strict bool + pathTemplate string + strict bool } +type fileNotFoundError struct { + err error + isRetryable bool +} + +func newFileNotFoundError(err error, isRetryable bool) *fileNotFoundError { + return &fileNotFoundError{err, isRetryable} +} +func (e *fileNotFoundError) IsRetryable() bool { return e.isRetryable } +func (e *fileNotFoundError) Error() string { return e.err.Error() } + // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames -func NewLocalRegistry(folder string, strict bool) (*LocalRegistry, error) { +func NewLocalRegistry(pathTemplate string, strict bool) *LocalRegistry { return &LocalRegistry{ - folder, + pathTemplate, strict, - }, nil + } } // DownloadSchema retrieves the schema from a file for the resource func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - schemaFile := path.Join(r.folder, schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict)) + schemaFile, err := schemaPath(r.pathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict) + if err != nil { + return []byte{}, nil + } f, err := os.Open(schemaFile) if err != nil { + if os.IsNotExist(err) { + return nil, newFileNotFoundError(fmt.Errorf("no schema found"), false) + } return nil, fmt.Errorf("failed to open schema %s", schemaFile) } + defer f.Close() content, err := ioutil.ReadAll(f) if err != nil { diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 58ecde6..415ddf3 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -1,8 +1,9 @@ package registry import ( - "fmt" + "bytes" "strings" + "text/template" ) type Manifest struct { @@ -19,7 +20,7 @@ type Retryable interface { IsRetryable() bool } -func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool) string { +func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict bool) (string, error) { normalisedVersion := k8sVersion if normalisedVersion != "master" { normalisedVersion = "v" + normalisedVersion @@ -38,5 +39,28 @@ func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool kindSuffix += "-" + strings.ToLower(groupParts[1]) } - return fmt.Sprintf("%s-standalone%s/%s%s.json", normalisedVersion, strictSuffix, strings.ToLower(resourceKind), kindSuffix) + tmpl, err := template.New("tpl").Parse(tpl) + if err != nil { + return "", err + } + + tplData := struct { + NormalizedVersion string + StrictSuffix string + ResourceKind string + KindSuffix string + }{ + normalisedVersion, + strictSuffix, + strings.ToLower(resourceKind), + kindSuffix, + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, tplData) + if err != nil { + return "", err + } + + return buf.String(), nil } diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go index 9140f7d..8594c0f 100644 --- a/pkg/registry/registry_test.go +++ b/pkg/registry/registry_test.go @@ -6,39 +6,52 @@ import ( func TestSchemaPath(t *testing.T) { for i, testCase := range []struct { - resourceKind, resourceAPIVersion, k8sVersion, expected string - strict bool + tpl, resourceKind, resourceAPIVersion, k8sVersion, expected string + strict bool + errExpected error }{ { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Deployment", "apps/v1", "1.16.0", - "v1.16.0-standalone-strict/deployment-apps-v1.json", + "https://kubernetesjsonschema.dev/v1.16.0-standalone-strict/deployment-apps-v1.json", true, + nil, }, { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Deployment", "apps/v1", "1.16.0", - "v1.16.0-standalone/deployment-apps-v1.json", + "https://kubernetesjsonschema.dev/v1.16.0-standalone/deployment-apps-v1.json", false, + nil, }, { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Service", "v1", "1.18.0", - "v1.18.0-standalone/service-v1.json", + "https://kubernetesjsonschema.dev/v1.18.0-standalone/service-v1.json", false, + nil, }, { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Service", "v1", "master", - "master-standalone/service-v1.json", + "https://kubernetesjsonschema.dev/master-standalone/service-v1.json", false, + nil, }, } { - if got := schemaPath(testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict); got != testCase.expected { + got, err := schemaPath(testCase.tpl, testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict) + if err != testCase.errExpected { + t.Errorf("%d - got error %s, expected %s", i+1, err, testCase.errExpected) + } + if got != testCase.expected { t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected) } } diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go index 2fe690f..128460a 100644 --- a/pkg/validator/validator_test.go +++ b/pkg/validator/validator_test.go @@ -65,6 +65,31 @@ lastName: bar }`), fmt.Errorf("Invalid type. Expected: number, given: string"), }, + { + "missing required field", + []byte(` +firstName: foo +`), + []byte(`{ + "title": "Example Schema", + "type": "object", + "properties": { + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "age": { + "description": "Age in years", + "type": "integer", + "minimum": 0 + } + }, + "required": ["firstName", "lastName"] +}`), + fmt.Errorf("lastName is required"), + }, { "resource has invalid yaml", []byte(`