Updated names for schema-location vars, added documentation, updated openapi2jsonschema

Readme.md

@@ -142,7 +142,7 @@ All 3 following command lines are equivalent:
 ```
 $ ./bin/kubeconform fixtures/valid.yaml
 $ ./bin/kubeconform -schema-location https://kubernetesjsonschema.dev fixtures/valid.yaml
-$ ./bin/kubeconform -schema-location 'https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/valid.yaml
+$ ./bin/kubeconform -schema-location 'https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/valid.yaml
 ```
 
 To support validating CRDs, we need to convert OpenAPI files to JSON schema, storing the JSON schemas
@@ -157,10 +157,17 @@ You can validate Openshift manifests using a custom schema location. Set the Ope
 against using -kubernetes-version.
 
 ```
-bin/kubeconform -kubernetes-version 3.8.0  -schema-location 'https://raw.githubusercontent.com/garethr/openshift-json-schema/master/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}.json'  -summary fixtures/valid.yaml
+bin/kubeconform -kubernetes-version 3.8.0  -schema-location 'https://raw.githubusercontent.com/garethr/openshift-json-schema/master/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}.json'  -summary fixtures/valid.yaml
 Summary: 1 resource found in 1 file - Valid: 1, Invalid: 0, Errors: 0 Skipped: 0
 ```
 
+Here are the variables you can use in -schema-location:
+ * *NormalizedKubernetesVersion* - Kubernetes Version, prefixed by v
+ * *StrictSuffix* - "-strict" or "" depending on whether validation is running in strict mode or not
+ * *ResourceKind* - Kind of the Kubernetes Resource
+ * *ResourceAPIVersion* - Version of API used for the resource - "v1" in "apiVersion: monitoring.coreos.com/v1"
+ * *KindSuffix* - suffix computed from apiVersion - for compatibility with Kubeval schema registries
+
 ### Converting an OpenAPI file to a JSON Schema
 
 Kubeconform uses JSON schemas to validate Kubernetes resources. For Custom Resource, the CustomResourceDefinition
@@ -168,7 +175,8 @@ first needs to be converted to JSON Schema. A script is provided to convert thes
 to JSON schema. Here is an example how to use it:
 
 ```
-$ ./scripts/openapi2jsonschema.py https://raw.githubusercontent.com/aws/amazon-sagemaker-operator-for-k8s/master/config/crd/bases/sagemaker.aws.amazon.com_trainingjobs.yaml > fixtures/registry/trainingjob-sagemaker-v1.json
+$ ./scripts/openapi2jsonschema.py https://raw.githubusercontent.com/aws/amazon-sagemaker-operator-for-k8s/master/config/crd/bases/sagemaker.aws.amazon.com_trainingjobs.yaml
+JSON schema written to trainingjob_v1.json
 ```
 
 ### Speed comparison with Kubeval

pkg/registry/local.go

@@ -36,7 +36,6 @@ func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersi
 	if err != nil {
 		return []byte{}, nil
 	}
-
 	f, err := os.Open(schemaFile)
 	if err != nil {
 		if os.IsNotExist(err) {

pkg/registry/registry.go

@@ -45,14 +45,16 @@ func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict
 	}
 
 	tplData := struct {
-		NormalizedVersion string
-		StrictSuffix      string
-		ResourceKind      string
-		KindSuffix        string
+		NormalizedKubernetesVersion string
+		StrictSuffix                string
+		ResourceKind                string
+		ResourceAPIVersion          string
+		KindSuffix                  string
 	}{
 		normalisedVersion,
 		strictSuffix,
 		strings.ToLower(resourceKind),
+		groupParts[len(groupParts)-1],
 		kindSuffix,
 	}
 

pkg/registry/registry_test.go

@@ -11,7 +11,7 @@ func TestSchemaPath(t *testing.T) {
 		errExpected                                                 error
 	}{
 		{
-			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
+			"https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Deployment",
 			"apps/v1",
 			"1.16.0",
@@ -20,7 +20,7 @@ func TestSchemaPath(t *testing.T) {
 			nil,
 		},
 		{
-			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
+			"https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Deployment",
 			"apps/v1",
 			"1.16.0",
@@ -29,7 +29,7 @@ func TestSchemaPath(t *testing.T) {
 			nil,
 		},
 		{
-			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
+			"https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Service",
 			"v1",
 			"1.18.0",
@@ -38,7 +38,7 @@ func TestSchemaPath(t *testing.T) {
 			nil,
 		},
 		{
-			"https://kubernetesjsonschema.dev/{{ .NormalizedVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
+			"https://kubernetesjsonschema.dev/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json",
 			"Service",
 			"v1",
 			"master",

pkg/resource/stream.go

@@ -45,6 +45,9 @@ func FromStream(ctx context.Context, path string, r io.Reader) (<-chan Resource,
 
 	go func() {
 		scanner := bufio.NewScanner(r)
+		const maxResourceSize = 1024 * 1024
+		buf := make([]byte, maxResourceSize)
+		scanner.Buffer(buf, maxResourceSize)
 		scanner.Split(SplitYAMLDocument)
 
 	SCAN:

scripts/openapi2jsonschema.py

@@ -4,6 +4,7 @@
 import yaml
 import json
 import sys
+import os
 import urllib.request
 
 def iteritems(d):
@@ -91,14 +92,37 @@ if len(sys.argv) == 0:
     print("missing file")
     exit(1)
 
-if sys.argv[1].startswith("http"):
-  f = urllib.request.urlopen(sys.argv[1])
-else:
-  f = open(sys.argv[1])
-with f:
-    y = yaml.load(f, Loader=yaml.SafeLoader)
-    schema = y["spec"]["validation"]["openAPIV3Schema"]
-    schema = additional_properties(schema)
-    schema = replace_int_or_string(schema)
-    print(json.dumps(schema))
-    exit(0)
+for crdFile in sys.argv[1:]:
+  if crdFile.startswith("http"):
+    f = urllib.request.urlopen(crdFile)
+  else:
+    f = open(crdFile)
+  with f:
+      y = yaml.load(f, Loader=yaml.SafeLoader)
+      filename = ""
+      schemaJSON = ""
+      if "spec" in y and "validation" in y["spec"] and "openAPIV3Schema" in y["spec"]["validation"]:
+          filename = y["spec"]["names"]["kind"].lower()+"_"+y["spec"]["version"].lower()+".json"
+
+          schema = y["spec"]["validation"]["openAPIV3Schema"]
+          schema = additional_properties(schema)
+          schema = replace_int_or_string(schema)
+          schemaJSON = json.dumps(schema)
+      elif "spec" in y and "versions" in y["spec"]:
+          for version in y["spec"]["versions"]:
+              if "schema" in version and "openAPIV3Schema" in version["schema"]:
+                  filename = y["spec"]["names"]["kind"].lower()+"_"+version["name"].lower()+".json"
+
+                  schema = version["schema"]["openAPIV3Schema"]
+                  schema = additional_properties(schema)
+                  schema = replace_int_or_string(schema)
+                  schemaJSON = json.dumps(schema)
+
+      # Dealing with user input here..
+      filename = os.path.basename(filename)
+      f = open(filename, "w")
+      f.write(schemaJSON)
+      f.close()
+      print("JSON schema written to {filename}".format(filename=filename))
+
+exit(0)