Merge pull request #19 from yannh/perf

Performance improvement

cmd/kubeconform/main.go

@@ -3,7 +3,10 @@ package main
 import (
 	"context"
 	"fmt"
+	"log"
 	"os"
+	"runtime"
+	"runtime/pprof"
 	"sync"
 
 	"github.com/yannh/kubeconform/pkg/config"
@@ -51,6 +54,20 @@ func realMain() int {
 		return 1
 	}
 
+	if cfg.CPUProfileFile != "" {
+		f, err := os.Create(cfg.CPUProfileFile)
+		if err != nil {
+			log.Fatal("could not create CPU profile: ", err)
+		}
+		defer f.Close()
+		if err := pprof.StartCPUProfile(f); err != nil {
+			log.Fatal("could not start CPU profile: ", err)
+		}
+		runtime.SetBlockProfileRate(1)
+
+		defer pprof.StopCPUProfile()
+	}
+
 	// Detect whether we have data being piped through stdin
 	stat, _ := os.Stdin.Stat()
 	isStdin := (stat.Mode() & os.ModeCharDevice) == 0

pkg/config/config.go

@@ -9,6 +9,7 @@ import (
 )
 
 type Config struct {
+	CPUProfileFile         string
 	ExitOnError            bool
 	Files                  []string
 	SchemaLocations        []string
@@ -74,6 +75,7 @@ func FromFlags(progName string, args []string) (Config, string, error) {
 	flags.StringVar(&c.OutputFormat, "output", "text", "output format - json, tap, text")
 	flags.BoolVar(&c.Verbose, "verbose", false, "print results for all resources")
 	flags.BoolVar(&c.SkipTLS, "insecure-skip-tls-verify", false, "disable verification of the server's SSL certificate. This will make your HTTPS connections insecure")
+	flags.StringVar(&c.CPUProfileFile, "cpu-prof", "", "debug - log CPU profiling to file")
 	flags.BoolVar(&c.Help, "h", false, "show help information")
 	flags.Usage = func() {
 		fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... [FILE OR FOLDER]...\n", progName)

pkg/resource/files.go

@@ -42,6 +42,7 @@ func isIgnored(path string, ignoreFilePatterns []string) (bool, error) {
 
 func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string) (<-chan Resource, <-chan error) {
 	resources := make(chan Resource)
+	files := make(chan string)
 	errors := make(chan error)
 
 	go func() {
@@ -71,27 +72,7 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string
 					return nil
 				}
 
-				f, err := os.Open(p)
+				files <- p
-				if err != nil {
-					return err
-				}
-
-				scanner := bufio.NewScanner(f)
-				const maxResourceSize = 4 * 1024 * 1024 // 4MB ought to be enough for everybody
-				buf := make([]byte, maxResourceSize)
-				scanner.Buffer(buf, maxResourceSize)
-				scanner.Split(SplitYAMLDocument)
-				nRes := 0
-				for res := scanner.Scan(); res != false; res = scanner.Scan() {
-					resources <- Resource{Path: p, Bytes: scanner.Bytes()}
-					nRes++
-				}
-				if err := scanner.Err(); err != nil {
-					errors <- DiscoveryError{p, err}
-				}
-				if nRes == 0 {
-					resources <- Resource{Path: p, Bytes: []byte{}}
-				}
 
 				return nil
 			})
@@ -101,8 +82,40 @@ func FromFiles(ctx context.Context, ignoreFilePatterns []string, paths ...string
 			}
 		}
 
-		close(resources)
+		close(files)
+	}()
+
+	go func() {
+		maxResourceSize := 4 * 1024 * 1024
+		buf := make([]byte, maxResourceSize)
+
+		for p := range files {
+			f, err := os.Open(p)
+			if err != nil {
+				errors <- DiscoveryError{p, err}
+				continue
+			}
+
+			scanner := bufio.NewScanner(f)
+			scanner.Buffer(buf, maxResourceSize)
+			scanner.Split(SplitYAMLDocument)
+			nRes := 0
+			for res := scanner.Scan(); res != false; res = scanner.Scan() {
+				resources <- Resource{Path: p, Bytes: []byte(scanner.Text())}
+				nRes++
+			}
+			if err := scanner.Err(); err != nil {
+				errors <- DiscoveryError{p, err}
+			}
+			if nRes == 0 {
+				resources <- Resource{Path: p, Bytes: []byte{}}
+			}
+
+			f.Close()
+		}
+
 		close(errors)
+		close(resources)
 	}()
 
 	return resources, errors

pkg/resource/resource.go

@@ -42,3 +42,26 @@ func (res *Resource) Signature() (*Signature, error) {
 	res.sig = &Signature{Kind: resource.Kind, Version: resource.APIVersion, Namespace: resource.Metadata.Namespace, Name: name}
 	return res.sig, err
 }
+
+func (res *Resource) SignatureFromMap(m map[string]interface{}) (*Signature, error) {
+	if res.sig != nil {
+		return res.sig, nil
+	}
+
+	APIVersion, _ := m["apiVersion"].(string)
+	Kind, _ := m["kind"].(string)
+
+	var name, ns string
+	Metadata, ok := m["metadata"].(map[string]interface{})
+	if ok {
+		name, _ = Metadata["name"].(string)
+		ns, _ = Metadata["namespace"].(string)
+		if _, ok := Metadata["generateName"].(string); ok {
+			name = Metadata["generateName"].(string) + "{{ generateName }}"
+		}
+	}
+
+	// We cache the result to not unmarshall every time we want to access the signature
+	res.sig = &Signature{Kind: Kind, Version: APIVersion, Namespace: ns, Name: name}
+	return res.sig, nil
+}

pkg/resource/resource_test.go

@@ -1,9 +1,12 @@
 package resource_test
 
 import (
+	"fmt"
+	"log"
 	"testing"
 
 	"github.com/yannh/kubeconform/pkg/resource"
+	"sigs.k8s.io/yaml"
 )
 
 func TestSignatureFromBytes(t *testing.T) {
@@ -47,3 +50,29 @@ spec:
 		}
 	}
 }
+
+func TestSignatureFromMap(t *testing.T) {
+	testCases := []struct {
+		b string
+	}{
+		{
+			"apiVersion: v1\nkind: ReplicationController\nmetadata:\n  name: \"bob\"\nspec:\n  replicas: 2\n",
+		},
+	}
+
+	for _, testCase := range testCases {
+		res := resource.Resource{
+			Path:  "foo",
+			Bytes: []byte(testCase.b),
+		}
+
+		var r map[string]interface{}
+		if err := yaml.Unmarshal(res.Bytes, &r); err != nil {
+			log.Fatal(err)
+		}
+
+		res.SignatureFromMap(r)
+		sig, _ := res.Signature()
+		fmt.Printf("%+v", sig)
+	}
+}

pkg/validator/validator.go

@@ -109,7 +109,12 @@ func (val *v) ValidateResource(res resource.Resource) Result {
 		return Result{Resource: res, Err: nil, Status: Empty}
 	}
 
-	sig, err := res.Signature()
+	var r map[string]interface{}
+	if err := yaml.Unmarshal(res.Bytes, &r); err != nil {
+		return Result{Resource: res, Status: Error, Err: fmt.Errorf("error unmarshalling resource: %s", err)}
+	}
+
+	sig, err := res.SignatureFromMap(r)
 	if err != nil {
 		return Result{Resource: res, Err: fmt.Errorf("error while parsing: %s", err), Status: Error}
 	}
@@ -122,11 +127,6 @@ func (val *v) ValidateResource(res resource.Resource) Result {
 		return Result{Resource: res, Err: fmt.Errorf("prohibited resource kind %s", sig.Kind), Status: Error}
 	}
 
-	var r map[string]interface{}
-	if err := yaml.Unmarshal(res.Bytes, &r); err != nil {
-		return Result{Resource: res, Status: Error, Err: fmt.Errorf("error unmarshalling resource: %s", err)}
-	}
-
 	if r == nil { // Resource is empty
 		return Result{Resource: res, Err: nil, Status: Empty}
 	}

pkg/validator/validator_test.go

@@ -18,7 +18,7 @@ func TestValidate(t *testing.T) {
 		{
 			"valid resource",
 			[]byte(`
-Kind: name
+kind: name
 firstName: foo
 lastName: bar
 `),
@@ -26,7 +26,7 @@ lastName: bar
   "title": "Example Schema",
   "type": "object",
   "properties": {
-    "Kind": {
+    "kind": {
       "type": "string"
     },
     "firstName": {
@@ -48,7 +48,7 @@ lastName: bar
 		{
 			"invalid resource",
 			[]byte(`
-Kind: name
+kind: name
 firstName: foo
 lastName: bar
 `),
@@ -56,7 +56,7 @@ lastName: bar
   "title": "Example Schema",
   "type": "object",
   "properties": {
-    "Kind": {
+    "kind": {
       "type": "string"
     },
     "firstName": {
@@ -78,14 +78,14 @@ lastName: bar
 		{
 			"missing required field",
 			[]byte(`
-Kind: name
+kind: name
 firstName: foo
 `),
 			[]byte(`{
   "title": "Example Schema",
   "type": "object",
   "properties": {
-    "Kind": {
+    "kind": {
       "type": "string"
     },
     "firstName": {
@@ -107,7 +107,7 @@ firstName: foo
 		{
 			"resource has invalid yaml",
 			[]byte(`
-Kind: name
+kind: name
 firstName foo
 lastName: bar
 `),
@@ -115,7 +115,7 @@ lastName: bar
   "title": "Example Schema",
   "type": "object",
   "properties": {
-    "Kind": {
+    "kind": {
       "type": "string"
     },
     "firstName": {
@@ -151,7 +151,7 @@ lastName: bar
 			regs: nil,
 		}
 		if got := val.ValidateResource(resource.Resource{Bytes: testCase.rawResource}); got.Status != testCase.expect {
-			t.Errorf("%d - expected %d, got %d", i, testCase.expect, got.Status)
+			t.Errorf("%d - expected %d, got %d: %s", i, testCase.expect, got.Status, got.Err.Error())
 		}
 	}
 }