From 4ae74305d1eda7dc16ad9676191ddab1bab09160 Mon Sep 17 00:00:00 2001 From: Yann Hamon Date: Sun, 15 Nov 2020 16:19:49 +0100 Subject: [PATCH] Updated names for schema-location vars, added documentation, updated openapi2jsonschema --- Readme.md | 14 ++++++++--- pkg/registry/local.go | 1 - pkg/registry/registry.go | 10 +++++--- pkg/registry/registry_test.go | 8 +++--- pkg/resource/stream.go | 3 +++ scripts/openapi2jsonschema.py | 46 ++++++++++++++++++++++++++--------- 6 files changed, 59 insertions(+), 23 deletions(-) diff --git a/Readme.md b/Readme.md index b194bbc..e5dafb5 100644 --- a/Readme.md +++ b/Readme.md @@ -142,7 +142,7 @@ All 3 following command lines are equivalent: ``` $ ./bin/kubeconform fixtures/valid.yaml $ ./bin/kubeconform -schema-location https://kubernetesjsonschema.dev fixtures/valid.yaml -$ ./bin/kubeconform -schema-location 'https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/valid.yaml +$ ./bin/kubeconform -schema-location 'https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/valid.yaml ``` To support validating CRDs, we need to convert OpenAPI files to JSON schema, storing the JSON schemas @@ -157,10 +157,17 @@ You can validate Openshift manifests using a custom schema location. Set the Ope against using -kubernetes-version. ``` -bin/kubeconform -kubernetes-version 3.8.0 -schema-location 'https://raw.githubusercontent.com/garethr/openshift-json-schema/master/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}.json' -summary fixtures/valid.yaml +bin/kubeconform -kubernetes-version 3.8.0 -schema-location 'https://raw.githubusercontent.com/garethr/openshift-json-schema/master/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}.json' -summary fixtures/valid.yaml Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0 ``` +Here are the variables you can use in -schema-location: + * *NormalizedKubernetesVersion* - Kubernetes Version, prefixed by v + * *StrictSuffix* - "-strict" or "" depending on whether validation is running in strict mode or not + * *ResourceKind* - Kind of the Kubernetes Resource + * *ResourceAPIVersion* - Version of API used for the resource - "v1" in "apiVersion: monitoring.coreos.com/v1" + * *KindSuffix* - suffix computed from apiVersion - for compatibility with Kubeval schema registries + ### Converting an OpenAPI file to a JSON Schema Kubeconform uses JSON schemas to validate Kubernetes resources. For Custom Resource, the CustomResourceDefinition @@ -168,7 +175,8 @@ first needs to be converted to JSON Schema. A script is provided to convert thes to JSON schema. Here is an example how to use it: ``` -$ ./scripts/openapi2jsonschema.py https://raw.githubusercontent.com/aws/amazon-sagemaker-operator-for-k8s/master/config/crd/bases/sagemaker.aws.amazon.com_trainingjobs.yaml > fixtures/registry/trainingjob-sagemaker-v1.json +$ ./scripts/openapi2jsonschema.py https://raw.githubusercontent.com/aws/amazon-sagemaker-operator-for-k8s/master/config/crd/bases/sagemaker.aws.amazon.com_trainingjobs.yaml +JSON schema written to trainingjob_v1.json ``` ### Speed comparison with Kubeval diff --git a/pkg/registry/local.go b/pkg/registry/local.go index 6c41fd8..ed17ce4 100644 --- a/pkg/registry/local.go +++ b/pkg/registry/local.go @@ -36,7 +36,6 @@ func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersi if err != nil { return []byte{}, nil } - f, err := os.Open(schemaFile) if err != nil { if os.IsNotExist(err) { diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index f5e21b4..b7c8e20 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -45,14 +45,16 @@ func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict } tplData := struct { - NormalizedVersion string - StrictSuffix string - ResourceKind string - KindSuffix string + NormalizedKubernetesVersion string + StrictSuffix string + ResourceKind string + ResourceAPIVersion string + KindSuffix string }{ normalisedVersion, strictSuffix, strings.ToLower(resourceKind), + groupParts[len(groupParts)-1], kindSuffix, } diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go index 8594c0f..3f560c8 100644 --- a/pkg/registry/registry_test.go +++ b/pkg/registry/registry_test.go @@ -11,7 +11,7 @@ func TestSchemaPath(t *testing.T) { errExpected error }{ { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", + "https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Deployment", "apps/v1", "1.16.0", @@ -20,7 +20,7 @@ func TestSchemaPath(t *testing.T) { nil, }, { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", + "https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Deployment", "apps/v1", "1.16.0", @@ -29,7 +29,7 @@ func TestSchemaPath(t *testing.T) { nil, }, { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", + "https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Service", "v1", "1.18.0", @@ -38,7 +38,7 @@ func TestSchemaPath(t *testing.T) { nil, }, { - "https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", + "https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json", "Service", "v1", "master", diff --git a/pkg/resource/stream.go b/pkg/resource/stream.go index 6b49b84..2952428 100644 --- a/pkg/resource/stream.go +++ b/pkg/resource/stream.go @@ -45,6 +45,9 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource, go func() { scanner := bufio.NewScanner(r) + const maxResourceSize = 1024 * 1024 + buf := make([]byte, maxResourceSize) + scanner.Buffer(buf, maxResourceSize) scanner.Split(SplitYAMLDocument) SCAN: diff --git a/scripts/openapi2jsonschema.py b/scripts/openapi2jsonschema.py index 77eb3ee..a8d8d13 100755 --- a/scripts/openapi2jsonschema.py +++ b/scripts/openapi2jsonschema.py @@ -4,6 +4,7 @@ import yaml import json import sys +import os import urllib.request def iteritems(d): @@ -91,14 +92,37 @@ if len(sys.argv) == 0: print("missing file") exit(1) -if sys.argv[1].startswith("http"): - f = urllib.request.urlopen(sys.argv[1]) -else: - f = open(sys.argv[1]) -with f: - y = yaml.load(f, Loader=yaml.SafeLoader) - schema = y["spec"]["validation"]["openAPIV3Schema"] - schema = additional_properties(schema) - schema = replace_int_or_string(schema) - print(json.dumps(schema)) - exit(0) +for crdFile in sys.argv[1:]: + if crdFile.startswith("http"): + f = urllib.request.urlopen(crdFile) + else: + f = open(crdFile) + with f: + y = yaml.load(f, Loader=yaml.SafeLoader) + filename = "" + schemaJSON = "" + if "spec" in y and "validation" in y["spec"] and "openAPIV3Schema" in y["spec"]["validation"]: + filename = y["spec"]["names"]["kind"].lower()+"_"+y["spec"]["version"].lower()+".json" + + schema = y["spec"]["validation"]["openAPIV3Schema"] + schema = additional_properties(schema) + schema = replace_int_or_string(schema) + schemaJSON = json.dumps(schema) + elif "spec" in y and "versions" in y["spec"]: + for version in y["spec"]["versions"]: + if "schema" in version and "openAPIV3Schema" in version["schema"]: + filename = y["spec"]["names"]["kind"].lower()+"_"+version["name"].lower()+".json" + + schema = version["schema"]["openAPIV3Schema"] + schema = additional_properties(schema) + schema = replace_int_or_string(schema) + schemaJSON = json.dumps(schema) + + # Dealing with user input here.. + filename = os.path.basename(filename) + f = open(filename, "w") + f.write(schemaJSON) + f.close() + print("JSON schema written to {filename}".format(filename=filename)) + +exit(0)