diff --git a/acceptance.bats b/acceptance.bats index 255b31b..45cf72b 100755 --- a/acceptance.bats +++ b/acceptance.bats @@ -54,3 +54,8 @@ run bin/kubeconform -file fixtures/test_crd.yaml -ignore-missing-schemas [ "$status" -eq 0 ] } + +@test "Succeed parsing a CRD when additional schema passed" { + run bin/kubeconform -file fixtures/test_crd.yaml -schema fixtures/crd_schema.yaml + [ "$status" -eq 0 ] +} diff --git a/fixtures/crd_schema.yaml b/fixtures/crd_schema.yaml new file mode 100644 index 0000000..8c53253 --- /dev/null +++ b/fixtures/crd_schema.yaml @@ -0,0 +1,767 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: trainingjobs.sagemaker.aws.amazon.com +spec: + additionalPrinterColumns: + - JSONPath: .status.trainingJobStatus + name: Status + type: string + - JSONPath: .status.secondaryStatus + name: Secondary-Status + type: string + - JSONPath: .metadata.creationTimestamp + format: date + name: Creation-Time + type: string + - JSONPath: .status.sageMakerTrainingJobName + name: Sagemaker-Job-Name + type: string + group: sagemaker.aws.amazon.com + names: + kind: TrainingJob + plural: trainingjobs + scope: "" + subresources: + status: {} + validation: + openAPIV3Schema: + description: TrainingJob is the Schema for the trainingjobs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + metadata: + properties: + annotations: + additionalProperties: + type: string + description: 'Annotations is an unstructured key value map stored with + a resource that may be set by external tools to store and retrieve + arbitrary metadata. They are not queryable and should be preserved + when modifying objects. More info: http://kubernetes.io/docs/user-guide/annotations' + type: object + clusterName: + description: The name of the cluster which the object belongs to. This + is used to distinguish resources with same name and namespace in different + clusters. This field is not set anywhere right now and apiserver is + going to ignore it if set in create or update request. + type: string + creationTimestamp: + description: "CreationTimestamp is a timestamp representing the server + time when this object was created. It is not guaranteed to be set + in happens-before order across separate operations. Clients may not + set this value. It is represented in RFC3339 form and is in UTC. \n + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" + format: date-time + type: string + deletionGracePeriodSeconds: + description: Number of seconds allowed for this object to gracefully + terminate before it will be removed from the system. Only set when + deletionTimestamp is also set. May only be shortened. Read-only. + format: int64 + type: integer + deletionTimestamp: + description: "DeletionTimestamp is RFC 3339 date and time at which this + resource will be deleted. This field is set by the server when a graceful + deletion is requested by the user, and is not directly settable by + a client. The resource is expected to be deleted (no longer visible + from resource lists, and not reachable by name) after the time in + this field, once the finalizers list is empty. As long as the finalizers + list contains items, deletion is blocked. Once the deletionTimestamp + is set, this value may not be unset or be set further into the future, + although it may be shortened or the resource may be deleted prior + to this time. For example, a user may request that a pod is deleted + in 30 seconds. The Kubelet will react by sending a graceful termination + signal to the containers in the pod. After that 30 seconds, the Kubelet + will send a hard termination signal (SIGKILL) to the container and + after cleanup, remove the pod from the API. In the presence of network + partitions, this object may still exist after this timestamp, until + an administrator or automated process can determine the resource is + fully terminated. If not set, graceful deletion of the object has + not been requested. \n Populated by the system when a graceful deletion + is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata" + format: date-time + type: string + finalizers: + description: Must be empty before the object is deleted from the registry. + Each entry is an identifier for the responsible component that will + remove the entry from the list. If the deletionTimestamp of the object + is non-nil, entries in this list can only be removed. + items: + type: string + type: array + generateName: + description: "GenerateName is an optional prefix, used by the server, + to generate a unique name ONLY IF the Name field has not been provided. + If this field is used, the name returned to the client will be different + than the name passed. This value will also be combined with a unique + suffix. The provided value has the same validation rules as the Name + field, and may be truncated by the length of the suffix required to + make the value unique on the server. \n If this field is specified + and the generated name exists, the server will NOT return a 409 - + instead, it will either return 201 Created or 500 with Reason ServerTimeout + indicating a unique name could not be found in the time allotted, + and the client should retry (optionally after the time indicated in + the Retry-After header). \n Applied only if Name is not specified. + More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#idempotency" + type: string + generation: + description: A sequence number representing a specific generation of + the desired state. Populated by the system. Read-only. + format: int64 + type: integer + initializers: + description: "An initializer is a controller which enforces some system + invariant at object creation time. This field is a list of initializers + that have not yet acted on this object. If nil or empty, this object + has been completely initialized. Otherwise, the object is considered + uninitialized and is hidden (in list/watch and get calls) from clients + that haven't explicitly asked to observe uninitialized objects. \n + When an object is created, the system will populate this list with + the current set of initializers. Only privileged users may set or + modify this list. Once it is empty, it may not be modified further + by any user. \n DEPRECATED - initializers are an alpha field and will + be removed in v1.15." + properties: + pending: + description: Pending is a list of initializers that must execute + in order before this object is visible. When the last pending + initializer is removed, and no failing result is set, the initializers + struct will be set to nil and the object is considered as initialized + and visible to all clients. + items: + properties: + name: + description: name of the process that is responsible for initializing + this object. + type: string + required: + - name + type: object + type: array + result: + description: If result is set with the Failure field, the object + will be persisted to storage and then deleted, ensuring that other + clients can observe the deletion. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this + representation of an object. Servers should convert recognized + schemas to the latest internal value, and may reject unrecognized + values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + type: string + code: + description: Suggested HTTP return code for this status, 0 if + not set. + format: int32 + type: integer + details: + description: Extended data associated with the reason. Each + reason may define its own extended details. This field is + optional and the data returned is not guaranteed to conform + to any schema except that defined by the reason type. + properties: + causes: + description: The Causes array includes more details associated + with the StatusReason failure. Not all StatusReasons may + provide detailed causes. + items: + properties: + field: + description: "The field of the resource that has caused + this error, as named by its JSON serialization. + May include dot and postfix notation for nested + attributes. Arrays are zero-indexed. Fields may + appear more than once in an array of causes due + to fields having multiple errors. Optional. \n Examples: + \ \"name\" - the field \"name\" on the current + resource \"items[0].name\" - the field \"name\" + on the first array entry in \"items\"" + type: string + message: + description: A human-readable description of the cause + of the error. This field may be presented as-is + to a reader. + type: string + reason: + description: A machine-readable description of the + cause of the error. If this value is empty there + is no information available. + type: string + type: object + type: array + group: + description: The group attribute of the resource associated + with the status StatusReason. + type: string + kind: + description: 'The kind attribute of the resource associated + with the status StatusReason. On some operations may differ + from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: The name attribute of the resource associated + with the status StatusReason (when there is a single name + which can be described). + type: string + retryAfterSeconds: + description: If specified, the time in seconds before the + operation should be retried. Some errors may indicate + the client must take an alternate action - for those errors + this field may indicate how long to wait before taking + the alternate action. + format: int32 + type: integer + uid: + description: 'UID of the resource. (when there is a single + resource which can be described). More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + type: string + type: object + kind: + description: 'Kind is a string value representing the REST resource + this object represents. Servers may infer this from the endpoint + the client submits requests to. Cannot be updated. In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + message: + description: A human-readable description of the status of this + operation. + type: string + metadata: + description: 'Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + properties: + continue: + description: continue may be set if the user set a limit + on the number of items returned, and indicates that the + server has more data available. The value is opaque and + may be used to issue another request to the endpoint that + served this list to retrieve the next set of available + objects. Continuing a consistent list may not be possible + if the server configuration has changed or more than a + few minutes have passed. The resourceVersion field returned + when using this continue value will be identical to the + value in the first response, unless you have received + this token from an error message. + type: string + resourceVersion: + description: 'String that identifies the server''s internal + version of this object that can be used by clients to + determine when objects have changed. Value must be treated + as opaque by clients and passed unmodified back to the + server. Populated by the system. Read-only. More info: + https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency' + type: string + selfLink: + description: selfLink is a URL representing this object. + Populated by the system. Read-only. + type: string + type: object + reason: + description: A machine-readable description of why this operation + is in the "Failure" status. If this value is empty there is + no information available. A Reason clarifies an HTTP status + code but does not override it. + type: string + status: + description: 'Status of the operation. One of: "Success" or + "Failure". More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' + type: string + type: object + required: + - pending + type: object + labels: + additionalProperties: + type: string + description: 'Map of string keys and values that can be used to organize + and categorize (scope and select) objects. May match selectors of + replication controllers and services. More info: http://kubernetes.io/docs/user-guide/labels' + type: object + managedFields: + description: "ManagedFields maps workflow-id and version to the set + of fields that are managed by that workflow. This is mostly for internal + housekeeping, and users typically shouldn't need to set or understand + this field. A workflow can be the user's name, a controller's name, + or the name of a specific apply path like \"ci-cd\". The set of fields + is always in the version that the workflow used when modifying the + object. \n This field is alpha and can be changed or removed without + notice." + items: + properties: + apiVersion: + description: APIVersion defines the version of this resource that + this field set applies to. The format is "group/version" just + like the top-level APIVersion field. It is necessary to track + the version of a field set because it cannot be automatically + converted. + type: string + fields: + additionalProperties: true + description: Fields identifies a set of fields. + type: object + manager: + description: Manager is an identifier of the workflow managing + these fields. + type: string + operation: + description: Operation is the type of operation which lead to + this ManagedFieldsEntry being created. The only valid values + for this field are 'Apply' and 'Update'. + type: string + time: + description: Time is timestamp of when these fields were set. + It should always be empty if Operation is 'Apply' + format: date-time + type: string + type: object + type: array + name: + description: 'Name must be unique within a namespace. Is required when + creating resources, although some resources may allow a client to + request the generation of an appropriate name automatically. Name + is primarily intended for creation idempotence and configuration definition. + Cannot be updated. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + namespace: + description: "Namespace defines the space within each name must be unique. + An empty namespace is equivalent to the \"default\" namespace, but + \"default\" is the canonical representation. Not all objects are required + to be scoped to a namespace - the value of this field for those objects + will be empty. \n Must be a DNS_LABEL. Cannot be updated. More info: + http://kubernetes.io/docs/user-guide/namespaces" + type: string + ownerReferences: + description: List of objects depended by this object. If ALL objects + in the list have been deleted, this object will be garbage collected. + If this object is managed by a controller, then an entry in this list + will point to this controller, with the controller field set to true. + There cannot be more than one managing controller. + items: + properties: + apiVersion: + description: API version of the referent. + type: string + blockOwnerDeletion: + description: If true, AND if the owner has the "foregroundDeletion" + finalizer, then the owner cannot be deleted from the key-value + store until this reference is removed. Defaults to false. To + set this field, a user needs "delete" permission of the owner, + otherwise 422 (Unprocessable Entity) will be returned. + type: boolean + controller: + description: If true, this reference points to the managing controller. + type: boolean + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + uid: + description: 'UID of the referent. More info: http://kubernetes.io/docs/user-guide/identifiers#uids' + type: string + required: + - apiVersion + - kind + - name + - uid + type: object + type: array + resourceVersion: + description: "An opaque value that represents the internal version of + this object that can be used by clients to determine when objects + have changed. May be used for optimistic concurrency, change detection, + and the watch operation on a resource or set of resources. Clients + must treat these values as opaque and passed unmodified back to the + server. They may only be valid for a particular resource or set of + resources. \n Populated by the system. Read-only. Value must be treated + as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#concurrency-control-and-consistency" + type: string + selfLink: + description: SelfLink is a URL representing this object. Populated by + the system. Read-only. + type: string + uid: + description: "UID is the unique in time and space value for this object. + It is typically generated by the server on successful creation of + a resource and is not allowed to change on PUT operations. \n Populated + by the system. Read-only. More info: http://kubernetes.io/docs/user-guide/identifiers#uids" + type: string + type: object + spec: + properties: + algorithmSpecification: + properties: + algorithmName: + minLength: 1 + type: string + metricDefinitions: + items: + properties: + name: + minLength: 1 + type: string + regex: + minLength: 1 + type: string + required: + - name + - regex + type: object + type: array + trainingImage: + minLength: 1 + type: string + trainingInputMode: + enum: + - File + - Pipe + type: string + required: + - trainingInputMode + type: object + checkpointConfig: + properties: + localPath: + type: string + s3Uri: + pattern: ^(https|s3)://([^/]+)/?(.*)$ + type: string + required: + - s3Uri + type: object + debugHookConfig: + properties: + collectionConfigurations: + items: + properties: + collectionName: + type: string + collectionParameters: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + type: object + type: array + localPath: + type: string + ruleParameters: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + s3OutputPath: + pattern: ^(https|s3)://([^/]+)/?(.*)$ + type: string + required: + - s3OutputPath + type: object + debugRuleConfigurations: + items: + properties: + instanceType: + type: string + localPath: + type: string + ruleConfigurationName: + type: string + ruleEvaluatorImage: + type: string + ruleParameters: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + s3OutputPath: + pattern: ^(https|s3)://([^/]+)/?(.*)$ + type: string + volumeSizeInGB: + format: int64 + minimum: 1 + type: integer + required: + - ruleConfigurationName + - ruleEvaluatorImage + type: object + type: array + enableInterContainerTrafficEncryption: + type: boolean + enableManagedSpotTraining: + type: boolean + enableNetworkIsolation: + type: boolean + hyperParameters: + items: + properties: + name: + type: string + value: + type: string + type: object + type: array + inputDataConfig: + items: + properties: + channelName: + minLength: 1 + pattern: '[A-Za-z0-9\.\-_]+' + type: string + compressionType: + enum: + - None + - Gzip + type: string + contentType: + type: string + dataSource: + properties: + fileSystemDataSource: + properties: + directoryPath: + type: string + fileSystemAccessMode: + type: string + fileSystemId: + type: string + fileSystemType: + type: string + required: + - directoryPath + - fileSystemAccessMode + - fileSystemId + - fileSystemType + type: object + s3DataSource: + properties: + attributeNames: + items: + type: string + type: array + s3DataDistributionType: + enum: + - FullyReplicated + - ShardedByS3Key + type: string + s3DataType: + enum: + - S3Prefix + - ManifestFile + - AugmentedManifestFile + type: string + s3Uri: + pattern: ^(https|s3)://([^/]+)/?(.*)$ + type: string + required: + - s3DataType + - s3Uri + type: object + type: object + inputMode: + enum: + - Pipe + - File + type: string + recordWrapperType: + type: string + shuffleConfig: + properties: + seed: + format: int64 + type: integer + required: + - seed + type: object + required: + - channelName + - dataSource + type: object + minItems: 1 + type: array + outputDataConfig: + properties: + kmsKeyId: + type: string + s3OutputPath: + pattern: ^(https|s3)://([^/]+)/?(.*)$ + type: string + required: + - s3OutputPath + type: object + region: + minLength: 1 + type: string + resourceConfig: + properties: + instanceCount: + format: int64 + minimum: 1 + type: integer + instanceType: + minLength: 1 + type: string + volumeKmsKeyId: + type: string + volumeSizeInGB: + format: int64 + minimum: 1 + type: integer + required: + - instanceCount + - instanceType + - volumeSizeInGB + type: object + roleArn: + minLength: 20 + type: string + sageMakerEndpoint: + description: A custom SageMaker endpoint to use when communicating with + SageMaker. + pattern: ^(https|http)://.*$ + type: string + stoppingCondition: + properties: + maxRuntimeInSeconds: + format: int64 + minimum: 1 + type: integer + maxWaitTimeInSeconds: + format: int64 + minimum: 1 + type: integer + type: object + tags: + items: + properties: + key: + minLength: 1 + type: string + value: + type: string + required: + - key + - value + type: object + type: array + tensorBoardOutputConfig: + properties: + localPath: + type: string + s3OutputPath: + pattern: ^(https|s3)://([^/]+)/?(.*)$ + type: string + required: + - s3OutputPath + type: object + trainingJobName: + description: The SageMaker training job name. This is optional for the + SageMaker K8s operator. If it is empty, the operator will populate + it with a generated name. + maxLength: 63 + type: string + vpcConfig: + properties: + securityGroupIds: + items: + type: string + minItems: 1 + type: array + subnets: + items: + type: string + minItems: 1 + type: array + required: + - securityGroupIds + - subnets + type: object + required: + - algorithmSpecification + - outputDataConfig + - resourceConfig + - roleArn + - region + - stoppingCondition + type: object + status: + properties: + additional: + description: Field to store additional information, for example if we + are unable to check the status we update this. + type: string + cloudWatchLogUrl: + description: Cloud Watch url for training log + type: string + debugRuleEvaluationStatuses: + description: Status of rule evaluation jobs, obtained from DebugRuleEvaluationStatuses. + https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTrainingJob.html#sagemaker-DescribeTrainingJob-response-DebugRuleEvaluationStatuses + items: + properties: + lastModifiedTime: + format: date-time + type: string + ruleConfigurationName: + type: string + ruleEvaluationJobArn: + type: string + ruleEvaluationStatus: + type: string + statusDetail: + type: string + type: object + type: array + lastCheckTime: + description: The last time that we checked the status of the SageMaker + job. + format: date-time + type: string + modelPath: + description: Full path to the training artifact (model) + type: string + sageMakerTrainingJobName: + description: SageMaker training job name + type: string + secondaryStatus: + description: The secondary, more granular status of the training job. + https://docs.aws.amazon.com/sagemaker/latest/dg/API_DescribeTrainingJob.html#SageMaker-DescribeTrainingJob-response-SecondaryStatus + type: string + trainingJobStatus: + description: The status of the training job. https://docs.aws.amazon.com/sagemaker/latest/dg/API_DescribeTrainingJob.html#SageMaker-DescribeTrainingJob-response-TrainingJobStatus + type: string + type: object + required: + - spec + type: object + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/fixtures/test_crd.yaml b/fixtures/test_crd.yaml index 66652c9..84f00e4 100755 --- a/fixtures/test_crd.yaml +++ b/fixtures/test_crd.yaml @@ -1,18 +1,81 @@ ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret +apiVersion: sagemaker.aws.amazon.com/v1 +kind: TrainingJob metadata: - name: test-secret - namespace: test-namespace + name: xgboost-mnist-debugger spec: - encryptedData: - SOME_ENCRYPTED_DATA: c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2 ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - name: test-secret-clone - namespace: test-namespace -spec: - encryptedData: - SOME_ENCRYPTED_DATA: c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2 + hyperParameters: + - name: max_depth + value: "5" + - name: eta + value: "0.2" + - name: gamma + value: "4" + - name: min_child_weight + value: "6" + - name: silent + value: "0" + - name: objective + value: reg:squarederror + - name: subsample + value: "0.7" + - name: num_round + value: "51" + algorithmSpecification: + trainingImage: 246618743249.dkr.ecr.us-west-2.amazonaws.com/sagemaker-xgboost:0.90-2-cpu-py3 + trainingInputMode: File + roleArn: arn:aws:iam::123456789012:role/service-role/AmazonSageMaker-ExecutionRole + region: us-west-2 + outputDataConfig: + s3OutputPath: s3://my-bucket/xgboost-debugger/output + resourceConfig: + instanceCount: 1 + instanceType: ml.m4.xlarge + volumeSizeInGB: 5 + stoppingCondition: + maxRuntimeInSeconds: 86400 + inputDataConfig: + - channelName: train + dataSource: + s3DataSource: + s3DataType: S3Prefix + s3Uri: s3://my-bucket/xgboost-debugger/train + s3DataDistributionType: FullyReplicated + contentType: libsvm + compressionType: None + - channelName: validation + dataSource: + s3DataSource: + s3DataType: S3Prefix + s3Uri: s3://my-bucket/xgboost-debugger/validation + s3DataDistributionType: FullyReplicated + contentType: libsvm + compressionType: None + debugHookConfig: + s3OutputPath: s3://my-bucket/xgboost-debugger/hookconfig + collectionConfigurations: + - collectionName: feature_importance + collectionParameters: + - name: save_interval + value: "5" + - collectionName: losses + collectionParameters: + - name: save_interval" + value: "500" + - collectionName: average_shap + collectionParameters: + - name: save_interval + value: "5" + - collectionName: metrics + collectionParameters: + - name: save_interval + value: "5" + debugRuleConfigurations: + - ruleConfigurationName: LossNotDecreasing + ruleEvaluatorImage: 895741380848.dkr.ecr.us-west-2.amazonaws.com/sagemaker-debugger-rules:latest + ruleParameters: + - name: collection_names + value: metrics + - name: num_steps + value: "10" + - name: rule_to_invoke + value: LossNotDecreasing \ No newline at end of file diff --git a/pkg/registry/local.go b/pkg/registry/local.go index e600ed6..4de8040 100644 --- a/pkg/registry/local.go +++ b/pkg/registry/local.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "sigs.k8s.io/yaml" + "strings" ) type LocalSchemas struct { @@ -56,5 +57,17 @@ func (r LocalSchemas) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersio return nil, fmt.Errorf("failed to open schema %s", schemaFile) } defer f.Close() - return ioutil.ReadAll(f) + content, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + asJSON := content + if strings.HasSuffix(schemaFile, ".yml") || strings.HasSuffix(schemaFile, ".yaml") { + asJSON, err = yaml.YAMLToJSON(content) + if err != nil { + return nil, fmt.Errorf("error converting manifest %s to JSON: %s", schemaFile, err) + } + } + return asJSON, nil }