From 59c23325d3be4e4aad24f3b35435989d9ab7993b Mon Sep 17 00:00:00 2001 From: Yann Hamon Date: Sun, 7 Jun 2020 18:59:03 +0200 Subject: [PATCH] replace -schema with -local-registry --- Readme.md | 16 ++++++ acceptance.bats | 4 -- main.go | 16 +++--- pkg/registry/kubernetesjsonschema.go | 25 +--------- pkg/registry/kubernetesjsonschema_test.go | 46 ----------------- pkg/registry/local.go | 61 +++++------------------ pkg/registry/registry.go | 27 ++++++++++ pkg/registry/registry_test.go | 45 +++++++++++++++++ 8 files changed, 110 insertions(+), 130 deletions(-) delete mode 100644 pkg/registry/kubernetesjsonschema_test.go create mode 100644 pkg/registry/registry_test.go diff --git a/Readme.md b/Readme.md index 5f4ee14..4b9a050 100644 --- a/Readme.md +++ b/Readme.md @@ -12,6 +12,22 @@ following improvements: * **high performance**: will validate & download manifests over multiple routines * support for **Kubernetes CRDs** (in progress) +### A small overview of Kubernetes manifest validation + +Kubernetes's API is described using the [OpenAPI (formerly swagger) specification](https://www.openapis.org), +in a [file](https://github.com/kubernetes/kubernetes/blob/master/api/openapi-spec/swagger.json) checked into +the main Kubernetes repository. + +Because of the state of the tooling to perform validation against OpenAPI schemas, projects usually convert +the OpenAPI schemas to [JSON schemas](https://json-schema.org/) first. Kubeval relies on +[instrumenta/OpenApi2JsonSchema](https://github.com/instrumenta/openapi2jsonschema) to convert Kubernetes' Swagger file +and break it down into multiple JSON schemas, stored in github at +[instrumenta/kubernetes-json-schema](https://github.com/instrumenta/kubernetes-json-schema) and published on +[kubernetesjsonschema.dev](https://kubernetesjsonschema.dev/). + +Kubeconform relies on the same JSON schemas from kubernetesjsonschema.dev, and will download required +schemas at runtime as required. + ### Usage ``` diff --git a/acceptance.bats b/acceptance.bats index 6c71957..adec878 100755 --- a/acceptance.bats +++ b/acceptance.bats @@ -61,7 +61,3 @@ [ "$status" -eq 0 ] } -@test "Succeed parsing a CRD when additional schema passed" { - run bin/kubeconform -schema fixtures/crd_schema.yaml fixtures/test_crd.yaml - [ "$status" -eq 0 ] -} diff --git a/main.go b/main.go index 7559457..31d10e6 100644 --- a/main.go +++ b/main.go @@ -210,7 +210,7 @@ func getFiles(files []string, fileBatches chan []string, validationResults chan } func realMain() int { - var schemas arrayParam + var localRegistryFolders arrayParam var skipKindsCSV, k8sVersion, outputFormat string var summary, strict, verbose, ignoreMissingSchemas bool var nWorkers int @@ -218,7 +218,7 @@ func realMain() int { var files []string flag.StringVar(&k8sVersion, "k8sversion", "1.18.0", "version of Kubernetes to test against") - flag.Var(&schemas, "schema", "file containing an additional Schema (can be specified multiple times)") + flag.Var(&localRegistryFolders, "local-registry", "folder containing additional schemas (can be specified multiple times)") flag.BoolVar(&ignoreMissingSchemas, "ignore-missing-schemas", false, "skip files with missing schemas instead of failing") flag.BoolVar(&summary, "summary", false, "print a summary at the end") flag.IntVar(&nWorkers, "n", 4, "number of routines to run in parallel") @@ -241,12 +241,14 @@ func realMain() int { registries := []registry.Registry{} registries = append(registries, registry.NewKubernetesRegistry(strict)) - if len(schemas) > 0 { - localRegistry, err := registry.NewLocalSchemas(schemas) - if err != nil { - log.Fatalf("%s", err) + if len(localRegistryFolders) > 0 { + for _, localRegistryFolder := range localRegistryFolders { + localRegistry, err := registry.NewLocalRegistry(localRegistryFolder, strict) + if err != nil { + log.Fatalf("%s", err) + } + registries = append(registries, localRegistry) } - registries = append(registries, localRegistry) } validationResults := make(chan []validationResult) diff --git a/pkg/registry/kubernetesjsonschema.go b/pkg/registry/kubernetesjsonschema.go index 17748ca..9ac8ed1 100644 --- a/pkg/registry/kubernetesjsonschema.go +++ b/pkg/registry/kubernetesjsonschema.go @@ -4,7 +4,6 @@ import ( "fmt" "io/ioutil" "net/http" - "strings" ) type KubernetesRegistry struct { @@ -30,30 +29,8 @@ func NewKubernetesRegistry(strict bool) *KubernetesRegistry { } } -func (r KubernetesRegistry) schemaURL(resourceKind, resourceAPIVersion, k8sVersion string) string { - normalisedVersion := k8sVersion - if normalisedVersion != "master" { - normalisedVersion = "v" + normalisedVersion - } - - strictSuffix := "" - if r.strict { - strictSuffix = "-strict" - } - - groupParts := strings.Split(resourceAPIVersion, "/") - versionParts := strings.Split(groupParts[0], ".") - - kindSuffix := "-" + strings.ToLower(versionParts[0]) - if len(groupParts) > 1 { - kindSuffix += "-" + strings.ToLower(groupParts[1]) - } - - return fmt.Sprintf("%s/%s-standalone%s/%s%s.json", r.baseURL, normalisedVersion, strictSuffix, strings.ToLower(resourceKind), kindSuffix) -} - func (r KubernetesRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - url := r.schemaURL(resourceKind, resourceAPIVersion, k8sVersion) + url := r.baseURL + "/" + schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict) resp, err := http.Get(url) if err != nil { diff --git a/pkg/registry/kubernetesjsonschema_test.go b/pkg/registry/kubernetesjsonschema_test.go deleted file mode 100644 index bb5b3b2..0000000 --- a/pkg/registry/kubernetesjsonschema_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package registry - -import ( - "testing" -) - -func TestSchemaURL(t *testing.T) { - for i, testCase := range []struct { - resourceKind, resourceAPIVersion, k8sVersion, expected string - strict bool - }{ - { - "Deployment", - "apps/v1", - "1.16.0", - "https://kubernetesjsonschema.dev/v1.16.0-standalone-strict/deployment-apps-v1.json", - true, - }, - { - "Deployment", - "apps/v1", - "1.16.0", - "https://kubernetesjsonschema.dev/v1.16.0-standalone/deployment-apps-v1.json", - false, - }, - { - "Service", - "v1", - "1.18.0", - "https://kubernetesjsonschema.dev/v1.18.0-standalone/service-v1.json", - false, - }, - { - "Service", - "v1", - "master", - "https://kubernetesjsonschema.dev/master-standalone/service-v1.json", - false, - }, - } { - reg := NewKubernetesRegistry(testCase.strict) - if got := reg.schemaURL(testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion); got != testCase.expected { - t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected) - } - } -} diff --git a/pkg/registry/local.go b/pkg/registry/local.go index 60fc733..5c4561d 100644 --- a/pkg/registry/local.go +++ b/pkg/registry/local.go @@ -4,55 +4,25 @@ import ( "fmt" "io/ioutil" "os" - "sigs.k8s.io/yaml" - "strings" + "path" ) -type LocalSchemas struct { - schemas map[string]string +type LocalRegistry struct { + folder string + strict bool } // NewLocalSchemas creates a new "registry", that will serve schemas from files, given a list of schema filenames -func NewLocalSchemas(schemaFiles []string) (*LocalSchemas, error) { - schemas := &LocalSchemas{ - schemas: map[string]string{}, - } - - for _, schemaFile := range schemaFiles { - f, err := os.Open(schemaFile) - if err != nil { - return nil, fmt.Errorf("failed to open schema %s", schemaFile) - } - defer f.Close() - content, err := ioutil.ReadAll(f) - if err != nil { - return nil, fmt.Errorf("failed to read schema %s", schemaFile) - } - - var parsedSchema struct { - Spec struct { - Names struct { - Kind string `json:"Kind"` - } `json:"Names"` - } `json:"Spec"` - } - err = yaml.Unmarshal(content, &parsedSchema) // Index Schemas by kind - if err != nil { - return nil, fmt.Errorf("failed parsing schema %s", schemaFile) - } - - schemas.schemas[parsedSchema.Spec.Names.Kind] = schemaFile - } - - return schemas, nil +func NewLocalRegistry(folder string, strict bool) (*LocalRegistry, error) { + return &LocalRegistry{ + folder, + strict, + }, nil } // DownloadSchema retrieves the schema from a file for the resource -func (r LocalSchemas) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { - schemaFile, ok := r.schemas[resourceKind] - if !ok { - return nil, fmt.Errorf("no local schema for Kind %s", resourceKind) - } +func (r LocalRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersion string) ([]byte, error) { + schemaFile := path.Join(r.folder, schemaPath(resourceKind, resourceAPIVersion, k8sVersion, r.strict)) f, err := os.Open(schemaFile) if err != nil { @@ -64,12 +34,5 @@ func (r LocalSchemas) DownloadSchema(resourceKind, resourceAPIVersion, k8sVersio return nil, err } - asJSON := content - if strings.HasSuffix(schemaFile, ".yml") || strings.HasSuffix(schemaFile, ".yaml") { - asJSON, err = yaml.YAMLToJSON(content) - if err != nil { - return nil, fmt.Errorf("error converting manifest %s to JSON: %s", schemaFile, err) - } - } - return asJSON, nil + return content, nil } diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 6dd932e..58ecde6 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -1,5 +1,10 @@ package registry +import ( + "fmt" + "strings" +) + type Manifest struct { Kind, Version string } @@ -13,3 +18,25 @@ type Registry interface { type Retryable interface { IsRetryable() bool } + +func schemaPath(resourceKind, resourceAPIVersion, k8sVersion string, strict bool) string { + normalisedVersion := k8sVersion + if normalisedVersion != "master" { + normalisedVersion = "v" + normalisedVersion + } + + strictSuffix := "" + if strict { + strictSuffix = "-strict" + } + + groupParts := strings.Split(resourceAPIVersion, "/") + versionParts := strings.Split(groupParts[0], ".") + + kindSuffix := "-" + strings.ToLower(versionParts[0]) + if len(groupParts) > 1 { + kindSuffix += "-" + strings.ToLower(groupParts[1]) + } + + return fmt.Sprintf("%s-standalone%s/%s%s.json", normalisedVersion, strictSuffix, strings.ToLower(resourceKind), kindSuffix) +} diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go new file mode 100644 index 0000000..9140f7d --- /dev/null +++ b/pkg/registry/registry_test.go @@ -0,0 +1,45 @@ +package registry + +import ( + "testing" +) + +func TestSchemaPath(t *testing.T) { + for i, testCase := range []struct { + resourceKind, resourceAPIVersion, k8sVersion, expected string + strict bool + }{ + { + "Deployment", + "apps/v1", + "1.16.0", + "v1.16.0-standalone-strict/deployment-apps-v1.json", + true, + }, + { + "Deployment", + "apps/v1", + "1.16.0", + "v1.16.0-standalone/deployment-apps-v1.json", + false, + }, + { + "Service", + "v1", + "1.18.0", + "v1.18.0-standalone/service-v1.json", + false, + }, + { + "Service", + "v1", + "master", + "master-standalone/service-v1.json", + false, + }, + } { + if got := schemaPath(testCase.resourceKind, testCase.resourceAPIVersion, testCase.k8sVersion, testCase.strict); got != testCase.expected { + t.Errorf("%d - got %s, expected %s", i+1, got, testCase.expected) + } + } +}