diff --git a/Readme.md b/Readme.md index a5c7078..f4c3bfe 100644 --- a/Readme.md +++ b/Readme.md @@ -49,6 +49,10 @@ configuration errors. ``` $ ./bin/kubeconform -h Usage: ./bin/kubeconform [OPTION]... [FILE OR FOLDER]... + -cache string + cache schemas downloaded via HTTP to this folder + -cpu-prof string + debug - log CPU profiling to file -exit-on-error immediately stop execution when the first error is encountered -h show help information diff --git a/cmd/kubeconform/main.go b/cmd/kubeconform/main.go index 894cdc5..fdc1b75 100644 --- a/cmd/kubeconform/main.go +++ b/cmd/kubeconform/main.go @@ -82,6 +82,7 @@ func realMain() int { } v, err := validator.New(cfg.SchemaLocations, validator.Opts{ + Cache: cfg.Cache, SkipTLS: cfg.SkipTLS, SkipKinds: cfg.SkipKinds, RejectKinds: cfg.RejectKinds, diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 0000000..734075b --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,6 @@ +package cache + +type Cache interface { + Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) + Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error +} diff --git a/pkg/cache/inmemory.go b/pkg/cache/inmemory.go new file mode 100644 index 0000000..594bace --- /dev/null +++ b/pkg/cache/inmemory.go @@ -0,0 +1,49 @@ +package cache + +import ( + "fmt" + "sync" +) + +// SchemaCache is a cache for downloaded schemas, so each file is only retrieved once +// It is different from pkg/registry/http_cache.go in that: +// - This cache caches the parsed Schemas +type inMemory struct { + sync.RWMutex + schemas map[string]interface{} +} + +// New creates a new cache for downloaded schemas +func NewInMemoryCache() Cache { + return &inMemory{ + schemas: map[string]interface{}{}, + } +} + +func key(resourceKind, resourceAPIVersion, k8sVersion string) string { + return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion) +} + +// Get retrieves the JSON schema given a resource signature +func (c *inMemory) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) { + k := key(resourceKind, resourceAPIVersion, k8sVersion) + c.RLock() + defer c.RUnlock() + schema, ok := c.schemas[k] + + if ok == false { + return nil, fmt.Errorf("schema not found in in-memory cache") + } + + return schema, nil +} + +// Set adds a JSON schema to the schema cache +func (c *inMemory) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error { + k := key(resourceKind, resourceAPIVersion, k8sVersion) + c.Lock() + defer c.Unlock() + c.schemas[k] = schema + + return nil +} diff --git a/pkg/cache/ondisk.go b/pkg/cache/ondisk.go new file mode 100644 index 0000000..123a541 --- /dev/null +++ b/pkg/cache/ondisk.go @@ -0,0 +1,48 @@ +package cache + +import ( + "crypto/md5" + "encoding/hex" + "fmt" + "io/ioutil" + "os" + "path" + "sync" +) + +type onDisk struct { + sync.RWMutex + folder string +} + +// New creates a new cache for downloaded schemas +func NewOnDiskCache(cache string) Cache { + return &onDisk{ + folder: cache, + } +} + +func cachePath(folder, resourceKind, resourceAPIVersion, k8sVersion string) string { + hash := md5.Sum([]byte(fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion))) + return path.Join(folder, hex.EncodeToString(hash[:])) +} + +// Get retrieves the JSON schema given a resource signature +func (c *onDisk) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) { + c.RLock() + defer c.RUnlock() + + f, err := os.Open(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion)) + if err != nil { + return nil, err + } + + return ioutil.ReadAll(f) +} + +// Set adds a JSON schema to the schema cache +func (c *onDisk) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error { + c.Lock() + defer c.Unlock() + return ioutil.WriteFile(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion), schema.([]byte), 0644) +} diff --git a/pkg/cache/schemacache.go b/pkg/cache/schemacache.go deleted file mode 100644 index 20ace1b..0000000 --- a/pkg/cache/schemacache.go +++ /dev/null @@ -1,42 +0,0 @@ -package cache - -import ( - "fmt" - "sync" - - "github.com/xeipuuv/gojsonschema" -) - -// SchemaCache is a cache for downloaded schemas, so each file is only retrieved once -type SchemaCache struct { - sync.RWMutex - schemas map[string]*gojsonschema.Schema -} - -// New creates a new cache for downloaded schemas -func New() *SchemaCache { - return &SchemaCache{ - schemas: map[string]*gojsonschema.Schema{}, - } -} - -// Key computes a key for a specific JSON schema from its Kind, the resource API Version, and the -// Kubernetes version -func Key(resourceKind, resourceAPIVersion, k8sVersion string) string { - return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion) -} - -// Get retrieves the JSON schema given a resource signature -func (c *SchemaCache) Get(key string) (*gojsonschema.Schema, bool) { - c.RLock() - defer c.RUnlock() - schema, ok := c.schemas[key] - return schema, ok -} - -// Set adds a JSON schema to the schema cache -func (c *SchemaCache) Set(key string, schema *gojsonschema.Schema) { - c.Lock() - defer c.Unlock() - c.schemas[key] = schema -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 88146a4..a321728 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -9,6 +9,7 @@ import ( ) type Config struct { + Cache string CPUProfileFile string ExitOnError bool Files []string @@ -75,6 +76,7 @@ func FromFlags(progName string, args []string) (Config, string, error) { flags.StringVar(&c.OutputFormat, "output", "text", "output format - json, tap, text") flags.BoolVar(&c.Verbose, "verbose", false, "print results for all resources (ignored for tap output)") flags.BoolVar(&c.SkipTLS, "insecure-skip-tls-verify", false, "disable verification of the server's SSL certificate. This will make your HTTPS connections insecure") + flags.StringVar(&c.Cache, "cache", "", "cache schemas downloaded via HTTP to this folder") flags.StringVar(&c.CPUProfileFile, "cpu-prof", "", "debug - log CPU profiling to file") flags.BoolVar(&c.Help, "h", false, "show help information") flags.Usage = func() { diff --git a/pkg/registry/http.go b/pkg/registry/http.go index 765632a..5e1f878 100644 --- a/pkg/registry/http.go +++ b/pkg/registry/http.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "net/http" "time" + + "github.com/yannh/kubeconform/pkg/cache" ) type httpGetter interface { @@ -16,10 +18,11 @@ type httpGetter interface { type SchemaRegistry struct { c httpGetter schemaPathTemplate string + cache cache.Cache strict bool } -func newHTTPRegistry(schemaPathTemplate string, strict bool, skipTLS bool) *SchemaRegistry { +func newHTTPRegistry(schemaPathTemplate string, cacheFolder string, strict bool, skipTLS bool) *SchemaRegistry { reghttp := &http.Transport{ MaxIdleConns: 100, IdleConnTimeout: 3 * time.Second, @@ -30,9 +33,15 @@ func newHTTPRegistry(schemaPathTemplate string, strict bool, skipTLS bool) *Sche reghttp.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } + var filecache cache.Cache = nil + if cacheFolder != "" { + filecache = cache.NewOnDiskCache(cacheFolder) + } + return &SchemaRegistry{ c: &http.Client{Transport: reghttp}, schemaPathTemplate: schemaPathTemplate, + cache: filecache, strict: strict, } } @@ -44,6 +53,12 @@ func (r SchemaRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVers return nil, err } + if r.cache != nil { + if b, err := r.cache.Get(resourceKind, resourceAPIVersion, k8sVersion); err == nil { + return b.([]byte), nil + } + } + resp, err := r.c.Get(url) if err != nil { return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err) @@ -63,5 +78,11 @@ func (r SchemaRegistry) DownloadSchema(resourceKind, resourceAPIVersion, k8sVers return nil, fmt.Errorf("failed downloading schema at %s: %s", url, err) } + if r.cache != nil { + if err := r.cache.Set(resourceKind, resourceAPIVersion, k8sVersion, body); err != nil { + return nil, fmt.Errorf("failed writing schema to cache: %s", err) + } + } + return body, nil } diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 32ecbb7..3b1cb51 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -79,7 +79,7 @@ func schemaPath(tpl, resourceKind, resourceAPIVersion, k8sVersion string, strict return buf.String(), nil } -func New(schemaLocation string, strict bool, skipTLS bool) (Registry, error) { +func New(schemaLocation string, cache string, strict bool, skipTLS bool) (Registry, error) { if !strings.HasSuffix(schemaLocation, "json") { // If we dont specify a full templated path, we assume the paths of kubernetesjsonschema.dev schemaLocation += "/{{ .NormalizedKubernetesVersion }}-standalone{{ .StrictSuffix }}/{{ .ResourceKind }}{{ .KindSuffix }}.json" } @@ -90,7 +90,7 @@ func New(schemaLocation string, strict bool, skipTLS bool) (Registry, error) { } if strings.HasPrefix(schemaLocation, "http") { - return newHTTPRegistry(schemaLocation, strict, skipTLS), nil + return newHTTPRegistry(schemaLocation, cache, strict, skipTLS), nil } return newLocalRegistry(schemaLocation, strict), nil diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index 2bbdcb7..d8d96c4 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -42,6 +42,7 @@ type Validator interface { // Opts contains a set of options for the validator. type Opts struct { + Cache string // Cache schemas downloaded via HTTP to this folder SkipTLS bool // skip TLS validation when downloading from an HTTP Schema Registry SkipKinds map[string]struct{} // List of resource Kinds to ignore RejectKinds map[string]struct{} // List of resource Kinds to reject @@ -59,7 +60,7 @@ func New(schemaLocations []string, opts Opts) (Validator, error) { registries := []registry.Registry{} for _, schemaLocation := range schemaLocations { - reg, err := registry.New(schemaLocation, opts.Strict, opts.SkipTLS) + reg, err := registry.New(schemaLocation, opts.Cache, opts.Strict, opts.SkipTLS) if err != nil { return nil, err } @@ -80,14 +81,14 @@ func New(schemaLocations []string, opts Opts) (Validator, error) { return &v{ opts: opts, schemaDownload: downloadSchema, - schemaCache: cache.New(), + schemaCache: cache.NewInMemoryCache(), regs: registries, }, nil } type v struct { opts Opts - schemaCache *cache.SchemaCache + schemaCache cache.Cache schemaDownload func(registries []registry.Registry, kind, version, k8sVersion string) (*gojsonschema.Schema, error) regs []registry.Registry } @@ -133,11 +134,13 @@ func (val *v) ValidateResource(res resource.Resource) Result { cached := false var schema *gojsonschema.Schema - cacheKey := "" if val.schemaCache != nil { - cacheKey = cache.Key(sig.Kind, sig.Version, val.opts.KubernetesVersion) - schema, cached = val.schemaCache.Get(cacheKey) + s, err := val.schemaCache.Get(sig.Kind, sig.Version, val.opts.KubernetesVersion) + if err == nil { + cached = true + schema = s.(*gojsonschema.Schema) + } } if !cached { @@ -146,7 +149,7 @@ func (val *v) ValidateResource(res resource.Resource) Result { } if val.schemaCache != nil { - val.schemaCache.Set(cacheKey, schema) + val.schemaCache.Set(sig.Kind, sig.Version, val.opts.KubernetesVersion, schema) } }