diff --git a/Makefile b/Makefile index 64b9164..51f4ac7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ #!/usr/bin/make -f -.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image +.PHONY: test-build test build build-static docker-test docker-build-static build-bats docker-acceptance docker-image release test-build: test build diff --git a/cmd/kubeconform/main.go b/cmd/kubeconform/main.go index 31d10e6..34859c6 100644 --- a/cmd/kubeconform/main.go +++ b/cmd/kubeconform/main.go @@ -9,7 +9,6 @@ import ( "github.com/yannh/kubeconform/pkg/output" "io" "io/ioutil" - "log" "os" "strings" "sync" @@ -210,7 +209,7 @@ func getFiles(files []string, fileBatches chan []string, validationResults chan } func realMain() int { - var localRegistryFolders arrayParam + var regs arrayParam var skipKindsCSV, k8sVersion, outputFormat string var summary, strict, verbose, ignoreMissingSchemas bool var nWorkers int @@ -218,7 +217,7 @@ func realMain() int { var files []string flag.StringVar(&k8sVersion, "k8sversion", "1.18.0", "version of Kubernetes to test against") - flag.Var(&localRegistryFolders, "local-registry", "folder containing additional schemas (can be specified multiple times)") + flag.Var(®s, "registry", "filepath template for registry") flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing") flag.BoolVar(&summary, "summary", false, "print a summary at the end") flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel") @@ -240,14 +239,13 @@ func realMain() int { } registries := []registry.Registry{} - registries = append(registries, registry.NewKubernetesRegistry(strict)) - if len(localRegistryFolders) > 0 { - for _, localRegistryFolder := range localRegistryFolders { - localRegistry, err := registry.NewLocalRegistry(localRegistryFolder, strict) - if err != nil { - log.Fatalf("%s", err) - } - registries = append(registries, localRegistry) + for _, reg := range regs { + if reg == "kubernetesjsonschema.dev" { + registries = append(registries, registry.NewHTTPRegistry("https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", strict)) + } else if strings.HasPrefix(reg, "http") { + registries = append(registries, registry.NewHTTPRegistry(reg, strict)) + } else { + registries = append(registries, registry.NewLocalRegistry(reg, strict)) } } diff --git a/cmd/openapi2jsonschema/main.py b/cmd/openapi2jsonschema/main.py index 724a2a0..b0716d8 100755 --- a/cmd/openapi2jsonschema/main.py +++ b/cmd/openapi2jsonschema/main.py @@ -65,10 +65,7 @@ def allow_null_optional_fields(data, parent=None, grand_parent=None, key=None): elif isinstance(v, str): is_non_null_type = k == "type" and v != "null" has_required_fields = grand_parent and "required" in grand_parent - is_required_field = ( - has_required_fields and key in grand_parent["required"] - ) - if is_non_null_type and not is_required_field: + if is_non_null_type and not has_required_field: new_v = [v, "null"] new[k] = new_v return new @@ -91,5 +88,4 @@ with open(r'synced_secrets.yaml') as f: schema = y["spec"]["validation"]["openAPIV3Schema"] schema = additional_properties(schema) schema = replace_int_or_string(schema) - schema = allow_null_optional_fields(schema) print(json.dumps(schema)) \ No newline at end of file diff --git a/pkg/registry/kubernetesjsonschema.go b/pkg/registry/kubernetesjsonschema.go index 9ac8ed1..e29e314 100644 --- a/pkg/registry/kubernetesjsonschema.go +++ b/pkg/registry/kubernetesjsonschema.go @@ -7,8 +7,8 @@ import ( ) type KubernetesRegistry struct { - baseURL string - strict bool + schemaPathTemplate string + strict bool } type downloadError struct { @@ -22,15 +22,18 @@ func newDownloadError(err error, isRetryable bool) *downloadError { func (e *downloadError) IsRetryable() bool { return e.isRetryable } func (e *downloadError) Error() string { return e.err.Error() } -func NewKubernetesRegistry(strict bool) *KubernetesRegistry { +func NewHTTPRegistry(schemaPathTemplate string, strict bool) *KubernetesRegistry { return &KubernetesRegistry{ - baseURL: "https://kubernetesjsonschema.dev", - strict: strict, + schemaPathTemplate: schemaPathTemplate, + strict: strict, } } func (r KubernetesRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - url := r.baseURL + "/" + schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict) + url, err := schemaPath(r.schemaPathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict) + if err != nil { + return nil, err + } resp, err := http.Get(url) if err != nil { diff --git a/pkg/registry/local.go b/pkg/registry/local.go index 5c4561d..67d2460 100644 --- a/pkg/registry/local.go +++ b/pkg/registry/local.go @@ -4,30 +4,47 @@ import ( "fmt" "io/ioutil" "os" - "path" ) type LocalRegistry struct { - folder string - strict bool + pathTemplate string + strict bool } +type fileNotFoundError struct { + err error + isRetryable bool +} + +func newFileNotFoundError(err error, isRetryable bool) *fileNotFoundError { + return &fileNotFoundError{err, isRetryable} +} +func (e *fileNotFoundError) IsRetryable() bool { return e.isRetryable } +func (e *fileNotFoundError) Error() string { return e.err.Error() } + // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames -func NewLocalRegistry(folder string, strict bool) (*LocalRegistry, error) { +func NewLocalRegistry(pathTemplate string, strict bool) *LocalRegistry { return &LocalRegistry{ - folder, + pathTemplate, strict, - }, nil + } } // DownloadSchema retrieves the schema from a file for the resource func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - schemaFile := path.Join(r.folder, schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict)) + schemaFile, err := schemaPath(r.pathTemplate, resourceKind, resourceAPIVersion, k8sVersion, r.strict) + if err != nil { + return []byte{}, nil + } f, err := os.Open(schemaFile) if err != nil { + if os.IsNotExist(err) { + return nil, newFileNotFoundError(fmt.Errorf("no schema found"), false) + } return nil, fmt.Errorf("failed to open schema %s", schemaFile) } + defer f.Close() content, err := ioutil.ReadAll(f) if err != nil { diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 58ecde6..415ddf3 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -1,8 +1,9 @@ package registry import ( - "fmt" + "bytes" "strings" + "text/template" ) type Manifest struct { @@ -19,7 +20,7 @@ type Retryable interface { IsRetryable() bool } -func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool) string { +func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict bool) (string, error) { normalisedVersion := k8sVersion if normalisedVersion != "master" { normalisedVersion = "v" + normalisedVersion @@ -38,5 +39,28 @@ func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool kindSuffix += "-" + strings.ToLower(groupParts[1]) } - return fmt.Sprintf("%s-standalone%s/%s%s.json", normalisedVersion, strictSuffix, strings.ToLower(resourceKind), kindSuffix) + tmpl, err := template.New("tpl").Parse(tpl) + if err != nil { + return "", err + } + + tplData := struct { + NormalizedVersion string + StrictSuffix string + ResourceKind string + KindSuffix string + }{ + normalisedVersion, + strictSuffix, + strings.ToLower(resourceKind), + kindSuffix, + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, tplData) + if err != nil { + return "", err + } + + return buf.String(), nil } diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go index 9140f7d..8594c0f 100644 --- a/pkg/registry/registry_test.go +++ b/pkg/registry/registry_test.go @@ -6,39 +6,52 @@ import ( func TestSchemaPath(t *testing.T) { for i, testCase := range []struct { - resourceKind, resourceAPIVersion, k8sVersion, expected string - strict bool + tpl, resourceKind, resourceAPIVersion, k8sVersion, expected string + strict bool + errExpected error }{ { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Deployment", "apps/v1", "1.16.0", - "v1.16.0-standalone-strict/deployment-apps-v1.json", + "https://kubernetesjsonschema.dev/v1.16.0-standalone-strict/deployment-apps-v1.json", true, + nil, }, { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Deployment", "apps/v1", "1.16.0", - "v1.16.0-standalone/deployment-apps-v1.json", + "https://kubernetesjsonschema.dev/v1.16.0-standalone/deployment-apps-v1.json", false, + nil, }, { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Service", "v1", "1.18.0", - "v1.18.0-standalone/service-v1.json", + "https://kubernetesjsonschema.dev/v1.18.0-standalone/service-v1.json", false, + nil, }, { + "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Service", "v1", "master", - "master-standalone/service-v1.json", + "https://kubernetesjsonschema.dev/master-standalone/service-v1.json", false, + nil, }, } { - if got := schemaPath(testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict); got != testCase.expected { + got, err := schemaPath(testCase.tpl, testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict) + if err != testCase.errExpected { + t.Errorf("%d - got error %s, expected %s", i+1, err, testCase.errExpected) + } + if got != testCase.expected { t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected) } } diff --git a/pkg/validator/validator_test.go b/pkg/validator/validator_test.go index 2fe690f..128460a 100644 --- a/pkg/validator/validator_test.go +++ b/pkg/validator/validator_test.go @@ -65,6 +65,31 @@ lastName: bar }`), fmt.Errorf("Invalid type. Expected: number, given: string"), }, + { + "missing required field", + []byte(` +firstName: foo +`), + []byte(`{ + "title": "Example Schema", + "type": "object", + "properties": { + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "age": { + "description": "Age in years", + "type": "integer", + "minimum": 0 + } + }, + "required": ["firstName", "lastName"] +}`), + fmt.Errorf("lastName is required"), + }, { "resource has invalid yaml", []byte(`