From ecc042d7f975b3158385eba1bd3a4d057a87f009 Mon Sep 17 00:00:00 2001 From: Yann Hamon Date: Sat, 10 May 2025 23:12:14 +0200 Subject: [PATCH] WIP --- pkg/cache/cache.go | 4 +-- pkg/cache/inmemory.go | 14 +++----- pkg/cache/ondisk.go | 13 ++++--- pkg/loader/file.go | 71 ++++++++++++++++++++++++++++++++++++++ pkg/loader/http.go | 20 ++++++++--- pkg/registry/registry.go | 11 +++--- pkg/validator/validator.go | 20 ++--------- 7 files changed, 107 insertions(+), 46 deletions(-) create mode 100644 pkg/loader/file.go diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 734075b..63f59b0 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1,6 +1,6 @@ package cache type Cache interface { - Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) - Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error + Get(key string) (interface{}, error) + Set(key string, schema interface{}) error } diff --git a/pkg/cache/inmemory.go b/pkg/cache/inmemory.go index ff8a829..170d709 100644 --- a/pkg/cache/inmemory.go +++ b/pkg/cache/inmemory.go @@ -20,16 +20,11 @@ func NewInMemoryCache() Cache { } } -func key(resourceKind, resourceAPIVersion, k8sVersion string) string { - return fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion) -} - // Get retrieves the JSON schema given a resource signature -func (c *inMemory) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) { - k := key(resourceKind, resourceAPIVersion, k8sVersion) +func (c *inMemory) Get(key string) (interface{}, error) { c.RLock() defer c.RUnlock() - schema, ok := c.schemas[k] + schema, ok := c.schemas[key] if !ok { return nil, fmt.Errorf("schema not found in in-memory cache") @@ -39,11 +34,10 @@ func (c *inMemory) Get(resourceKind, resourceAPIVersion, k8sVersion string) (int } // Set adds a JSON schema to the schema cache -func (c *inMemory) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error { - k := key(resourceKind, resourceAPIVersion, k8sVersion) +func (c *inMemory) Set(key string, schema interface{}) error { c.Lock() defer c.Unlock() - c.schemas[k] = schema + c.schemas[key] = schema return nil } diff --git a/pkg/cache/ondisk.go b/pkg/cache/ondisk.go index 31b69ec..b45c8fc 100644 --- a/pkg/cache/ondisk.go +++ b/pkg/cache/ondisk.go @@ -3,7 +3,6 @@ package cache import ( "crypto/sha256" "encoding/hex" - "fmt" "io" "os" "path" @@ -22,17 +21,17 @@ func NewOnDiskCache(cache string) Cache { } } -func cachePath(folder, resourceKind, resourceAPIVersion, k8sVersion string) string { - hash := sha256.Sum256([]byte(fmt.Sprintf("%s-%s-%s", resourceKind, resourceAPIVersion, k8sVersion))) +func cachePath(folder, key string) string { + hash := sha256.Sum256([]byte(key)) return path.Join(folder, hex.EncodeToString(hash[:])) } // Get retrieves the JSON schema given a resource signature -func (c *onDisk) Get(resourceKind, resourceAPIVersion, k8sVersion string) (interface{}, error) { +func (c *onDisk) Get(key string) (interface{}, error) { c.RLock() defer c.RUnlock() - f, err := os.Open(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion)) + f, err := os.Open(cachePath(c.folder, key)) if err != nil { return nil, err } @@ -42,8 +41,8 @@ func (c *onDisk) Get(resourceKind, resourceAPIVersion, k8sVersion string) (inter } // Set adds a JSON schema to the schema cache -func (c *onDisk) Set(resourceKind, resourceAPIVersion, k8sVersion string, schema interface{}) error { +func (c *onDisk) Set(key string, schema interface{}) error { c.Lock() defer c.Unlock() - return os.WriteFile(cachePath(c.folder, resourceKind, resourceAPIVersion, k8sVersion), schema.([]byte), 0644) + return os.WriteFile(cachePath(c.folder, key), schema.([]byte), 0644) } diff --git a/pkg/loader/file.go b/pkg/loader/file.go new file mode 100644 index 0000000..1374858 --- /dev/null +++ b/pkg/loader/file.go @@ -0,0 +1,71 @@ +package loader + +import ( + "fmt" + "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/yannh/kubeconform/pkg/cache" + gourl "net/url" + "os" + "path/filepath" + "runtime" + "strings" +) + +// FileLoader loads json file url. +type FileLoader struct { + cache cache.Cache +} + +func (l FileLoader) Load(url string) (any, error) { + path, err := l.ToFile(url) + if err != nil { + return nil, err + } + if l.cache != nil { + if cached, err := l.cache.Get(path); err == nil { + return cached, nil + } + } + + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + s, err := jsonschema.UnmarshalJSON(f) + if err != nil { + return nil, err + } + + if l.cache != nil { + if err = l.cache.Set(path, s); err != nil { + return nil, fmt.Errorf("failed to write cache to disk: %s", err) + } + } + + return s, nil +} + +// ToFile is helper method to convert file url to file path. +func (l FileLoader) ToFile(url string) (string, error) { + u, err := gourl.Parse(url) + if err != nil { + return "", err + } + if u.Scheme != "file" { + return "", fmt.Errorf("invalid file url: %s", u) + } + path := u.Path + if runtime.GOOS == "windows" { + path = strings.TrimPrefix(path, "/") + path = filepath.FromSlash(path) + } + return path, nil +} + +func NewFileLoader(cache cache.Cache) *FileLoader { + return &FileLoader{ + cache: cache, + } +} diff --git a/pkg/loader/http.go b/pkg/loader/http.go index 21fe2f3..d0e3165 100644 --- a/pkg/loader/http.go +++ b/pkg/loader/http.go @@ -19,6 +19,12 @@ type HTTPURLLoader struct { } func (l *HTTPURLLoader) Load(url string) (any, error) { + if l.cache != nil { + if cached, err := l.cache.Get(url); err == nil { + return cached, nil + } + } + resp, err := l.client.Get(url) if err != nil { msg := fmt.Sprintf("failed downloading schema at %s: %s", url, err) @@ -41,12 +47,18 @@ func (l *HTTPURLLoader) Load(url string) (any, error) { msg := fmt.Sprintf("failed parsing schema from %s: %s", url, err) return nil, errors.New(msg) } - - if l.cache != nil { - // To implement + s, err := jsonschema.UnmarshalJSON(bytes.NewReader(body)) + if err != nil { + return nil, err } - return jsonschema.UnmarshalJSON(bytes.NewReader(body)) + if l.cache != nil { + if err = l.cache.Set(url, s); err != nil { + return nil, fmt.Errorf("failed to write cache to disk: %s", err) + } + } + + return s, nil } func NewHTTPURLLoader(skipTLS bool, cache cache.Cache) (*HTTPURLLoader, error) { diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 32078ad..9584c17 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -3,7 +3,6 @@ package registry import ( "bytes" "fmt" - "github.com/santhosh-tekuri/jsonschema/v6" "github.com/yannh/kubeconform/pkg/cache" "github.com/yannh/kubeconform/pkg/loader" "os" @@ -81,7 +80,7 @@ func New(schemaLocation string, cacheFolder string, strict bool, skipTLS bool, d return nil, fmt.Errorf("failed initialising schema location registry: %s", err) } - var filecache cache.Cache = nil + var c cache.Cache = nil if cacheFolder != "" { fi, err := os.Stat(cacheFolder) if err != nil { @@ -91,17 +90,19 @@ func New(schemaLocation string, cacheFolder string, strict bool, skipTLS bool, d return nil, fmt.Errorf("cache folder %s is not a directory", err) } - filecache = cache.NewOnDiskCache(cacheFolder) + c = cache.NewOnDiskCache(cacheFolder) + } else { + c = cache.NewInMemoryCache() } if strings.HasPrefix(schemaLocation, "http") { - httpLoader, err := loader.NewHTTPURLLoader(skipTLS, filecache) + httpLoader, err := loader.NewHTTPURLLoader(skipTLS, c) if err != nil { return nil, fmt.Errorf("failed creating HTTP loader: %s", err) } return newHTTPRegistry(schemaLocation, httpLoader, strict, debug) } - fileLoader := jsonschema.FileLoader{} + fileLoader := loader.NewFileLoader(c) return newLocalRegistry(schemaLocation, fileLoader, strict, debug) } diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go index fa1e7a1..2e8e362 100644 --- a/pkg/validator/validator.go +++ b/pkg/validator/validator.go @@ -188,25 +188,9 @@ func (val *v) ValidateResource(res resource.Resource) Result { return Result{Resource: res, Err: fmt.Errorf("prohibited resource kind %s", sig.Kind), Status: Error} } - cached := false var schema *jsonschema.Schema - - if val.schemaCache != nil { - s, err := val.schemaCache.Get(sig.Kind, sig.Version, val.opts.KubernetesVersion) - if err == nil { - cached = true - schema = s.(*jsonschema.Schema) - } - } - - if !cached { - if schema, err = val.schemaDownload(val.regs, val.loader, sig.Kind, sig.Version, val.opts.KubernetesVersion); err != nil { - return Result{Resource: res, Err: err, Status: Error} - } - - if val.schemaCache != nil { - val.schemaCache.Set(sig.Kind, sig.Version, val.opts.KubernetesVersion, schema) - } + if schema, err = val.schemaDownload(val.regs, val.loader, sig.Kind, sig.Version, val.opts.KubernetesVersion); err != nil { + return Result{Resource: res, Err: err, Status: Error} } if schema == nil {