diff --git a/Readme.md b/Readme.md index f419427..3d24d87 100644 --- a/Readme.md +++ b/Readme.md @@ -9,8 +9,9 @@ configuration using the schemas from the registry maintained by the It is inspired by and similar to [Kubeval](https://github.com/instrumenta/kubeval), but with the following improvements: - * **high performance**: will validate & download manifests over multiple routines - * support for **Kubernetes CRDs** + * **high performance**: will validate & download manifests over multiple routines, caching + downloaded files in memory + * configurable list of schemas registries, enabling validating Kubernetes custom resources (CRDs) ### A small overview of Kubernetes manifest validation @@ -117,6 +118,16 @@ in a local folder - for example schemas. Then we specify this folder as an addit $ ./bin/kubeconform -registry kubernetesjsonschema.dev -registry 'schemas/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/custom-resource.yaml ``` +### Generating a JSON schema from an OpenAPI file + +Kubeconform uses JSON schemas to validate Kubernetes resources. For Custom Resource, the CustomResourceDefinition +first needs to be converted to JSON Schema. A script is provided to convert these CustomResourceDefinitions +to JSON schema. Here is an example how to use it: + +``` +$ ./cmd/openapi2jsonschema/main.py https://raw.githubusercontent.com/aws/amazon-sagemaker-operator-for-k8s/master/config/crd/bases/sagemaker.aws.amazon.com_trainingjobs.yaml > fixtures/registry/trainingjob-sagemaker-v1.json +``` + ### Credits * @garethr for the [Kubeval](https://github.com/instrumenta/kubeval) and diff --git a/acceptance.bats b/acceptance.bats index adec878..99548b5 100755 --- a/acceptance.bats +++ b/acceptance.bats @@ -56,8 +56,12 @@ [ "$status" -eq 1 ] } -@test "Pass when parsing a config with CRD and ignoring missing schemas" { +@test "Pass when parsing a config with Custom Resource and ignoring missing schemas" { run bin/kubeconform -ignore-missing-schemas fixtures/test_crd.yaml [ "$status" -eq 0 ] } +@test "Pass when parsing a Custom Resource and using a local schema registry with appropriate CRD" { + run bin/kubeconform -registry './fixtures/registry/{{ .ResourceKind }}{{ .KindSuffix }}.json' fixtures/test_crd.yaml + [ "$status" -eq 0 ] +} diff --git a/cmd/kubeconform/main.go b/cmd/kubeconform/main.go index a9e2a7a..82f0fb8 100644 --- a/cmd/kubeconform/main.go +++ b/cmd/kubeconform/main.go @@ -263,7 +263,7 @@ func realMain() int { var o output.Output if o, err = getLogger(outputFormat, summary, verbose); err != nil { - fmt.Println(err) + fmt.Fprintln(os.Stderr, err) return 1 } diff --git a/cmd/openapi2jsonschema/main.py b/cmd/openapi2jsonschema/main.py index b0716d8..77eb3ee 100755 --- a/cmd/openapi2jsonschema/main.py +++ b/cmd/openapi2jsonschema/main.py @@ -1,7 +1,10 @@ #!/usr/bin/env python +# Derived from https://github.com/instrumenta/openapi2jsonschema import yaml import json +import sys +import urllib.request def iteritems(d): if hasattr(dict, "iteritems"): @@ -83,9 +86,19 @@ def append_no_duplicates(obj, key, value): if value not in obj[key]: obj[key].append(value) -with open(r'synced_secrets.yaml') as f: + +if len(sys.argv) == 0: + print("missing file") + exit(1) + +if sys.argv[1].startswith("http"): + f = urllib.request.urlopen(sys.argv[1]) +else: + f = open(sys.argv[1]) +with f: y = yaml.load(f, Loader=yaml.SafeLoader) schema = y["spec"]["validation"]["openAPIV3Schema"] schema = additional_properties(schema) schema = replace_int_or_string(schema) - print(json.dumps(schema)) \ No newline at end of file + print(json.dumps(schema)) + exit(0) diff --git a/fixtures/registry/trainingjob-sagemaker-v1.json b/fixtures/registry/trainingjob-sagemaker-v1.json new file mode 100644 index 0000000..dc58bbb --- /dev/null +++ b/fixtures/registry/trainingjob-sagemaker-v1.json @@ -0,0 +1,555 @@ +{ + "description": "TrainingJob is the Schema for the trainingjobs API", + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "type": "object" + }, + "spec": { + "description": "TrainingJobSpec defines the desired state of TrainingJob", + "properties": { + "algorithmSpecification": { + "properties": { + "algorithmName": { + "minLength": 1, + "type": "string" + }, + "metricDefinitions": { + "items": { + "properties": { + "name": { + "minLength": 1, + "type": "string" + }, + "regex": { + "minLength": 1, + "type": "string" + } + }, + "required": [ + "name", + "regex" + ], + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "trainingImage": { + "minLength": 1, + "type": "string" + }, + "trainingInputMode": { + "enum": [ + "File", + "Pipe" + ], + "type": "string" + } + }, + "required": [ + "trainingInputMode" + ], + "type": "object", + "additionalProperties": false + }, + "checkpointConfig": { + "properties": { + "localPath": { + "type": "string" + }, + "s3Uri": { + "pattern": "^(https|s3)://([^/]+)/?(.*)$", + "type": "string" + } + }, + "required": [ + "s3Uri" + ], + "type": "object", + "additionalProperties": false + }, + "debugHookConfig": { + "description": "DebugHookConfig https://docs.aws.amazon.com/sagemaker/latest/dg/API_DebugHookConfig.html", + "properties": { + "collectionConfigurations": { + "items": { + "description": "CollectionConfiguration https://docs.aws.amazon.com/sagemaker/latest/dg/API_CollectionConfiguration.html", + "properties": { + "collectionName": { + "type": "string" + }, + "collectionParameters": { + "items": { + "description": "Used in describing maps in Kubernetes.", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "type": "array" + } + }, + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "localPath": { + "type": "string" + }, + "ruleParameters": { + "items": { + "description": "Used in describing maps in Kubernetes.", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "s3OutputPath": { + "pattern": "^(https|s3)://([^/]+)/?(.*)$", + "type": "string" + } + }, + "required": [ + "s3OutputPath" + ], + "type": "object", + "additionalProperties": false + }, + "debugRuleConfigurations": { + "items": { + "description": "DebugRuleConfiguration https://docs.aws.amazon.com/sagemaker/latest/dg/API_DebugRuleConfiguration.html", + "properties": { + "instanceType": { + "type": "string" + }, + "localPath": { + "type": "string" + }, + "ruleConfigurationName": { + "type": "string" + }, + "ruleEvaluatorImage": { + "type": "string" + }, + "ruleParameters": { + "items": { + "description": "Used in describing maps in Kubernetes.", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "s3OutputPath": { + "pattern": "^(https|s3)://([^/]+)/?(.*)$", + "type": "string" + }, + "volumeSizeInGB": { + "format": "int64", + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "ruleConfigurationName", + "ruleEvaluatorImage" + ], + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "enableInterContainerTrafficEncryption": { + "type": "boolean" + }, + "enableManagedSpotTraining": { + "type": "boolean" + }, + "enableNetworkIsolation": { + "type": "boolean" + }, + "hyperParameters": { + "items": { + "description": "Used in describing maps in Kubernetes.", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "inputDataConfig": { + "items": { + "properties": { + "channelName": { + "minLength": 1, + "pattern": "[A-Za-z0-9\\.\\-_]+", + "type": "string" + }, + "compressionType": { + "enum": [ + "None", + "Gzip" + ], + "type": "string" + }, + "contentType": { + "type": "string" + }, + "dataSource": { + "properties": { + "fileSystemDataSource": { + "properties": { + "directoryPath": { + "type": "string" + }, + "fileSystemAccessMode": { + "type": "string" + }, + "fileSystemId": { + "type": "string" + }, + "fileSystemType": { + "type": "string" + } + }, + "required": [ + "directoryPath", + "fileSystemAccessMode", + "fileSystemId", + "fileSystemType" + ], + "type": "object", + "additionalProperties": false + }, + "s3DataSource": { + "properties": { + "attributeNames": { + "items": { + "type": "string" + }, + "type": "array" + }, + "s3DataDistributionType": { + "enum": [ + "FullyReplicated", + "ShardedByS3Key" + ], + "type": "string" + }, + "s3DataType": { + "enum": [ + "S3Prefix", + "ManifestFile", + "AugmentedManifestFile" + ], + "type": "string" + }, + "s3Uri": { + "pattern": "^(https|s3)://([^/]+)/?(.*)$", + "type": "string" + } + }, + "required": [ + "s3DataType", + "s3Uri" + ], + "type": "object", + "additionalProperties": false + } + }, + "type": "object", + "additionalProperties": false + }, + "inputMode": { + "enum": [ + "Pipe", + "File" + ], + "type": "string" + }, + "recordWrapperType": { + "type": "string" + }, + "shuffleConfig": { + "properties": { + "seed": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "seed" + ], + "type": "object", + "additionalProperties": false + } + }, + "required": [ + "channelName", + "dataSource" + ], + "type": "object", + "additionalProperties": false + }, + "minItems": 1, + "type": "array" + }, + "outputDataConfig": { + "properties": { + "kmsKeyId": { + "type": "string" + }, + "s3OutputPath": { + "pattern": "^(https|s3)://([^/]+)/?(.*)$", + "type": "string" + } + }, + "required": [ + "s3OutputPath" + ], + "type": "object", + "additionalProperties": false + }, + "region": { + "minLength": 1, + "type": "string" + }, + "resourceConfig": { + "properties": { + "instanceCount": { + "format": "int64", + "minimum": 1, + "type": "integer" + }, + "instanceType": { + "minLength": 1, + "type": "string" + }, + "volumeKmsKeyId": { + "type": "string" + }, + "volumeSizeInGB": { + "format": "int64", + "minimum": 1, + "type": "integer" + } + }, + "required": [ + "instanceCount", + "instanceType", + "volumeSizeInGB" + ], + "type": "object", + "additionalProperties": false + }, + "roleArn": { + "minLength": 20, + "type": "string" + }, + "sageMakerEndpoint": { + "description": "A custom SageMaker endpoint to use when communicating with SageMaker.", + "pattern": "^(https|http)://.*$", + "type": "string" + }, + "stoppingCondition": { + "properties": { + "maxRuntimeInSeconds": { + "format": "int64", + "minimum": 1, + "type": "integer" + }, + "maxWaitTimeInSeconds": { + "format": "int64", + "minimum": 1, + "type": "integer" + } + }, + "type": "object", + "additionalProperties": false + }, + "tags": { + "items": { + "properties": { + "key": { + "minLength": 1, + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "key", + "value" + ], + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "tensorBoardOutputConfig": { + "description": "TensorBoardOutputConfig https://docs.aws.amazon.com/sagemaker/latest/dg/API_TensorBoardOutputConfig.html", + "properties": { + "localPath": { + "type": "string" + }, + "s3OutputPath": { + "pattern": "^(https|s3)://([^/]+)/?(.*)$", + "type": "string" + } + }, + "required": [ + "s3OutputPath" + ], + "type": "object", + "additionalProperties": false + }, + "trainingJobName": { + "description": "The SageMaker training job name. This is optional for the SageMaker K8s operator. If it is empty, the operator will populate it with a generated name.", + "maxLength": 63, + "type": "string" + }, + "vpcConfig": { + "properties": { + "securityGroupIds": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 1, + "type": "array" + }, + "subnets": { + "items": { + "type": "string" + }, + "maxItems": 16, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "securityGroupIds", + "subnets" + ], + "type": "object", + "additionalProperties": false + } + }, + "required": [ + "algorithmSpecification", + "outputDataConfig", + "region", + "resourceConfig", + "roleArn", + "stoppingCondition" + ], + "type": "object", + "additionalProperties": false + }, + "status": { + "description": "TrainingJobStatus defines the observed state of TrainingJob", + "properties": { + "additional": { + "description": "Field to store additional information, for example if we are unable to check the status we update this.", + "type": "string" + }, + "cloudWatchLogUrl": { + "description": "Cloud Watch url for training log", + "type": "string" + }, + "debugRuleEvaluationStatuses": { + "description": "Status of rule evaluation jobs, obtained from DebugRuleEvaluationStatuses. https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTrainingJob.html#sagemaker-DescribeTrainingJob-response-DebugRuleEvaluationStatuses", + "items": { + "description": "DebugRuleEvaluationStatus https://docs.aws.amazon.com/sagemaker/latest/dg/API_DebugRuleEvaluationStatus.html", + "properties": { + "lastModifiedTime": { + "format": "date-time", + "type": "string" + }, + "ruleConfigurationName": { + "type": "string" + }, + "ruleEvaluationJobArn": { + "type": "string" + }, + "ruleEvaluationStatus": { + "type": "string" + }, + "statusDetail": { + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + }, + "type": "array" + }, + "lastCheckTime": { + "description": "The last time that we checked the status of the SageMaker job.", + "format": "date-time", + "type": "string" + }, + "modelPath": { + "description": "Full path to the training artifact (model)", + "type": "string" + }, + "sageMakerTrainingJobName": { + "description": "SageMaker training job name", + "type": "string" + }, + "secondaryStatus": { + "description": "The secondary, more granular status of the training job. https://docs.aws.amazon.com/sagemaker/latest/dg/API_DescribeTrainingJob.html#SageMaker-DescribeTrainingJob-response-SecondaryStatus", + "type": "string" + }, + "trainingJobStatus": { + "description": "The status of the training job. https://docs.aws.amazon.com/sagemaker/latest/dg/API_DescribeTrainingJob.html#SageMaker-DescribeTrainingJob-response-TrainingJobStatus", + "type": "string" + } + }, + "type": "object", + "additionalProperties": false + } + }, + "required": [ + "spec" + ], + "type": "object" +} diff --git a/go.mod b/go.mod index a720985..6584fd4 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,7 @@ module github.com/yannh/kubeconform go 1.14 require ( - github.com/instrumenta/kubeval v0.0.0-20200515185822-7721cbec724c github.com/xeipuuv/gojsonschema v1.2.0 - gopkg.in/yaml.v2 v2.3.0 + gopkg.in/yaml.v2 v2.3.0 // indirect sigs.k8s.io/yaml v1.2.0 ) diff --git a/vendor/modules.txt b/vendor/modules.txt index 67f6777..aa316f7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,5 +1,3 @@ -# github.com/instrumenta/kubeval v0.0.0-20200515185822-7721cbec724c -## explicit # github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f github.com/xeipuuv/gojsonpointer # github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415