From df465f89f626ca4936d36772a2f0894d97edcad3 Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Wed, 25 Jun 2025 17:29:04 +0200 Subject: [PATCH 1/5] WIP: Add WeightedJPath backend code This will allow injecting additional jpath elements into the Resolve function. Sorting happens now every time that function is called which is not really ideal but also - given the size of the list - shouldn't matter all that much. --- cmd/tk/tool.go | 2 +- go.work.sum | 11 +++++++ pkg/jsonnet/eval.go | 9 ++++-- pkg/jsonnet/imports.go | 2 +- pkg/jsonnet/jpath/dirs_test.go | 2 +- pkg/jsonnet/jpath/jpath.go | 54 ++++++++++++++++++++++++++++----- pkg/jsonnet/jpath/jpath_test.go | 35 +++++++++++++++++++++ pkg/jsonnet/lint.go | 2 +- pkg/tanka/export.go | 4 +++ pkg/tanka/parallel.go | 7 +++++ 10 files changed, 114 insertions(+), 14 deletions(-) diff --git a/cmd/tk/tool.go b/cmd/tk/tool.go index 7f25fcabb..1ca9c0b07 100644 --- a/cmd/tk/tool.go +++ b/cmd/tk/tool.go @@ -51,7 +51,7 @@ func jpathCmd() *cli.Command { return fmt.Errorf("resolving JPATH: %s", err) } - jsonnetpath, base, root, err := jpath.Resolve(entrypoint, false) + jsonnetpath, base, root, err := jpath.Resolve(entrypoint, false, nil) if err != nil { return fmt.Errorf("resolving JPATH: %s", err) } diff --git a/go.work.sum b/go.work.sum index 3a9a18d5c..6648a30ff 100644 --- a/go.work.sum +++ b/go.work.sum @@ -21,7 +21,9 @@ github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7 github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/agnivade/levenshtein v1.2.1/go.mod h1:QVVI16kDrtSuwcpd0p1+xMC6Z/VfhtCyDIjcwga4/DU= github.com/alexflint/go-arg v1.4.2/go.mod h1:9iRbDxne7LcR/GSvEr7ma++GLpdIU1zrghf2y2768kM= +github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8= github.com/alexflint/go-scalar v1.0.0/go.mod h1:GpHzbCOZXEKMEcygYQ5n/aa4Aq84zbxjy3MxYW0gjYw= +github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= @@ -75,6 +77,7 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -155,6 +158,8 @@ golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= @@ -169,15 +174,18 @@ golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -188,6 +196,7 @@ golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= @@ -199,6 +208,7 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= @@ -232,5 +242,6 @@ k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/jsonnet/eval.go b/pkg/jsonnet/eval.go index 16901982f..c72327316 100644 --- a/pkg/jsonnet/eval.go +++ b/pkg/jsonnet/eval.go @@ -36,8 +36,11 @@ type Opts struct { ExtCode InjectedCode TLACode InjectedCode ImportPaths []string - EvalScript string - CachePath string + // AdditionalImportPaths represent custom import paths that are used as + // input to the operation + AdditionalImportPaths []jpath.WeightedJPath + EvalScript string + CachePath string CachePathRegexes []*regexp.Regexp } @@ -108,7 +111,7 @@ func evaluateSnippet(jsonnetImpl types.JsonnetImplementation, evalFunc evalFunc, cache = NewFileEvalCache(opts.CachePath) } - jpath, _, _, err := jpath.Resolve(path, false) + jpath, _, _, err := jpath.Resolve(path, false, opts.AdditionalImportPaths) if err != nil { return "", errors.Wrap(err, "resolving import paths") } diff --git a/pkg/jsonnet/imports.go b/pkg/jsonnet/imports.go index 59c27171d..bd892d32c 100644 --- a/pkg/jsonnet/imports.go +++ b/pkg/jsonnet/imports.go @@ -43,7 +43,7 @@ func TransitiveImports(dir string) ([]string, error) { return nil, errors.Wrap(err, "opening file") } - jpath, _, rootDir, err := jpath.Resolve(dir, false) + jpath, _, rootDir, err := jpath.Resolve(dir, false, nil) if err != nil { return nil, errors.Wrap(err, "resolving JPATH") } diff --git a/pkg/jsonnet/jpath/dirs_test.go b/pkg/jsonnet/jpath/dirs_test.go index 5ca6dc2ff..d854ba248 100644 --- a/pkg/jsonnet/jpath/dirs_test.go +++ b/pkg/jsonnet/jpath/dirs_test.go @@ -188,7 +188,7 @@ func TestFindRoot(t *testing.T) { dir := makeTestdata(t, s.testdata) defer os.RemoveAll(dir) - _, base, root, err := Resolve(filepath.Join(dir, s.environment), false) + _, base, root, err := Resolve(filepath.Join(dir, s.environment), false, nil) assert.Equal(t, s.err, err) if err == nil { diff --git a/pkg/jsonnet/jpath/jpath.go b/pkg/jsonnet/jpath/jpath.go index e5fd7590a..2a917dcb1 100644 --- a/pkg/jsonnet/jpath/jpath.go +++ b/pkg/jsonnet/jpath/jpath.go @@ -3,10 +3,36 @@ package jpath import ( "os" "path/filepath" + "slices" ) const DefaultEntrypoint = "main.jsonnet" +type WeightedJPath interface { + Path() string + Weight() int +} + +type StaticallyWeightedJPath struct { + weight int + path string +} + +func NewStaticallyWeightedJPath(path string, weight int) *StaticallyWeightedJPath { + return &StaticallyWeightedJPath{ + weight: weight, + path: path, + } +} + +func (jp *StaticallyWeightedJPath) Weight() int { + return jp.weight +} + +func (jp *StaticallyWeightedJPath) Path() string { + return jp.path +} + // Resolve the given path and resolves the jPath around it. This means it: // - figures out the project root (the one with .jsonnetfile, vendor/ and lib/) // - figures out the environments base directory (usually the main.jsonnet) @@ -14,7 +40,7 @@ const DefaultEntrypoint = "main.jsonnet" // It then constructs a jPath with the base directory, vendor/ and lib/. // This results in predictable imports, as it doesn't matter whether the user called // called the command further down tree or not. A little bit like git. -func Resolve(path string, allowMissingBase bool) (jpath []string, base, root string, err error) { +func Resolve(path string, allowMissingBase bool, additionalJPaths []WeightedJPath) (jpath []string, base, root string, err error) { root, err = FindRoot(path) if err != nil { return nil, "", "", err @@ -30,13 +56,27 @@ func Resolve(path string, allowMissingBase bool) (jpath []string, base, root str return nil, "", "", err } + paths := make([]WeightedJPath, 0, 4+len(additionalJPaths)) + paths = append(paths, NewStaticallyWeightedJPath(filepath.Join(root, "vendor"), 300)) + paths = append(paths, NewStaticallyWeightedJPath(filepath.Join(base, "vendor"), 200)) + paths = append(paths, NewStaticallyWeightedJPath(filepath.Join(root, "lib"), 100)) + paths = append(paths, NewStaticallyWeightedJPath(base, 0)) + if additionalJPaths != nil { + paths = append(paths, additionalJPaths...) + } + + slices.SortStableFunc(paths, func(a, b WeightedJPath) int { + return b.Weight() - a.Weight() + }) + + // TODO: Sort these paths with highest weight first + result := make([]string, 0, len(paths)) + for _, path := range paths { + result = append(result, path.Path()) + } + // The importer iterates through this list in reverse order - return []string{ - filepath.Join(root, "vendor"), - filepath.Join(base, "vendor"), // Look for a vendor folder in the base dir before using the root vendor - filepath.Join(root, "lib"), - base, - }, base, root, nil + return result, base, root, nil } // Filename returns the name of the entrypoint file. diff --git a/pkg/jsonnet/jpath/jpath_test.go b/pkg/jsonnet/jpath/jpath_test.go index 31e31c33e..fccfdcd5f 100644 --- a/pkg/jsonnet/jpath/jpath_test.go +++ b/pkg/jsonnet/jpath/jpath_test.go @@ -2,6 +2,7 @@ package jpath_test import ( "encoding/json" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -9,6 +10,7 @@ import ( "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" + "github.com/grafana/tanka/pkg/jsonnet/jpath" ) var jsonnetImpl = &goimpl.JsonnetGoImplementation{} @@ -29,3 +31,36 @@ func TestResolvePrecedence(t *testing.T) { assert.JSONEq(t, string(w), s) } + +func TestJPathWeights(t *testing.T) { + path := "./testdata/valid/environments/default/main.jsonnet" + + t.Run("default-weights", func(t *testing.T) { + paths, _, root, err := jpath.Resolve(path, false, nil) + require.Equal(t, []string{ + filepath.Join(root, "vendor"), + filepath.Join(root, "environments", "default", "vendor"), + filepath.Join(root, "lib"), + filepath.Join(root, "environments", "default"), + }, paths) + require.NoError(t, err) + }) + + t.Run("custom-paths", func(t *testing.T) { + _, _, root, err := jpath.Resolve(path, false, nil) + require.NoError(t, err) + paths, _, root, err := jpath.Resolve(path, false, []jpath.WeightedJPath{ + jpath.NewStaticallyWeightedJPath(filepath.Join(root, "vendor-dev"), 250), + jpath.NewStaticallyWeightedJPath(filepath.Join(root, "prio-lib"), 2), + }) + require.Equal(t, []string{ + filepath.Join(root, "vendor"), + filepath.Join(root, "vendor-dev"), + filepath.Join(root, "environments", "default", "vendor"), + filepath.Join(root, "lib"), + filepath.Join(root, "prio-lib"), + filepath.Join(root, "environments", "default"), + }, paths) + require.NoError(t, err) + }) +} diff --git a/pkg/jsonnet/lint.go b/pkg/jsonnet/lint.go index 98989ad90..467d0fa43 100644 --- a/pkg/jsonnet/lint.go +++ b/pkg/jsonnet/lint.go @@ -107,7 +107,7 @@ func lintWithRecover(file string) (buf bytes.Buffer, success bool) { return } - jpaths, _, _, err := jpath.Resolve(file, true) + jpaths, _, _, err := jpath.Resolve(file, true, nil) if err != nil { fmt.Fprintf(&buf, "got an error getting jpath for %s: %v\n\n", file, err) return diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 8346f4fca..c88f5092d 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -86,6 +86,10 @@ func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnv return fmt.Errorf("deleting previously exported manifests from deleted environments: %w", err) } + for _, env := range envs { + fmt.Println(env.Metadata.Labels) + } + // get all environments for paths loadedEnvs, err := parallelLoadEnvironments(envs, parallelOpts{ Opts: opts.Opts, diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index 483e07391..64c07bc9b 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -3,6 +3,7 @@ package tanka import ( "fmt" "path/filepath" + "strings" "time" "k8s.io/apimachinery/pkg/labels" @@ -57,6 +58,12 @@ func parallelLoadEnvironments(envs []*v1alpha1.Environment, opts parallelOpts) ( // to Tanka workflow thus being able to handle such cases o.JsonnetOpts = o.JsonnetOpts.Clone() + if strings.Contains(env.Metadata.Name, "dev") { + // Also inject the dev environment into the possible import paths + // with higher priority than root/vendor: + o.JsonnetOpts.ImportPaths = append(o.JsonnetOpts.ImportPaths, "shared-vendors/dev/vendor") + } + o.Name = env.Metadata.Name path := env.Metadata.Namespace rootDir, err := jpath.FindRoot(path) From 2e12e8938d3a779b3df84b2775e188e47b3311da Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Thu, 26 Jun 2025 12:12:06 +0200 Subject: [PATCH 2/5] Load tkrc configuration including additional JPath entries --- internal/tkrc/tkrc.go | 62 ++++++++++++++++++++++++++++++++++++++ internal/tkrc/tkrc_test.go | 44 +++++++++++++++++++++++++++ pkg/tanka/load.go | 28 ++++++++++++++--- pkg/tanka/parallel.go | 10 ++++++ pkg/tanka/tanka.go | 5 +++ 5 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 internal/tkrc/tkrc.go create mode 100644 internal/tkrc/tkrc_test.go diff --git a/internal/tkrc/tkrc.go b/internal/tkrc/tkrc.go new file mode 100644 index 000000000..dbb0c150f --- /dev/null +++ b/internal/tkrc/tkrc.go @@ -0,0 +1,62 @@ +package tkrc + +import ( + "os" + + "github.com/rs/zerolog/log" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "sigs.k8s.io/yaml" +) + +type Config struct { + AdditionalJPaths []AdditionalJPath `json:"additionalJPaths"` +} + +type AdditionalJPath struct { + RawName string `json:"name"` + RawPath string `json:"path"` + RawWeight int `json:"weight"` + MatchExpressions []metav1.LabelSelectorRequirement `json:"matchExpressions"` +} + +func (jp *AdditionalJPath) Weight() int { + return jp.RawWeight +} + +func (jp *AdditionalJPath) Name() string { + return jp.RawName +} + +func (jp *AdditionalJPath) Path() string { + return jp.RawPath +} + +func (jp *AdditionalJPath) Matches(set labels.Labels) bool { + if len(jp.MatchExpressions) == 0 { + return true + } + selector := labels.NewSelector() + for _, req := range jp.MatchExpressions { + r, err := labels.NewRequirement(req.Key, selection.Operator(req.Operator), req.Values) + if err != nil { + log.Warn().Err(err).Str("rule", jp.Name()).Msg("invalid requirement, skipping whole rule") + return false + } + selector = selector.Add(*r) + } + return selector.Matches(set) +} + +func Load(path string) (*Config, error) { + config := Config{} + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if err := yaml.UnmarshalStrict(raw, &config); err != nil { + return nil, err + } + return &config, nil +} diff --git a/internal/tkrc/tkrc_test.go b/internal/tkrc/tkrc_test.go new file mode 100644 index 000000000..6d284c0f8 --- /dev/null +++ b/internal/tkrc/tkrc_test.go @@ -0,0 +1,44 @@ +package tkrc + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" +) + +func TestMatching(t *testing.T) { + tests := map[string]struct { + Labels map[string]string + Matches bool + MatchExpressions []metav1.LabelSelectorRequirement + }{ + "match-due-to-no-requirements": { + MatchExpressions: []metav1.LabelSelectorRequirement{}, + Labels: map[string]string{"cluster_name": "test"}, + Matches: true, + }, + "single-req-match": { + MatchExpressions: []metav1.LabelSelectorRequirement{ + { + Key: "cluster_name", + Operator: metav1.LabelSelectorOperator(selection.In), + Values: []string{"test"}, + }, + }, + Labels: map[string]string{"cluster_name": "test"}, + Matches: true, + }, + } + + for testname, test := range tests { + t.Run(testname, func(t *testing.T) { + rule := AdditionalJPath{ + MatchExpressions: test.MatchExpressions, + } + require.Equal(t, test.Matches, rule.Matches(labels.Set(test.Labels))) + }) + } +} diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index 0721c6f7b..509d7e0e3 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" + "github.com/grafana/tanka/internal/tkrc" "github.com/grafana/tanka/pkg/jsonnet/implementations/binary" "github.com/grafana/tanka/pkg/jsonnet/implementations/goimpl" "github.com/grafana/tanka/pkg/jsonnet/implementations/types" @@ -17,6 +18,7 @@ import ( "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/pkg/errors" "github.com/rs/zerolog/log" + "k8s.io/apimachinery/pkg/labels" ) // environmentExtCode is the extCode ID `tk.env` uses underneath @@ -60,7 +62,22 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { return nil, err } - env, err := loader.Load(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) + // First, we need to take a peek at the environment so that we can extract + // the necessary labels for doing matching additional jpaths: + peeked, err := loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name, opts.AdditionalJPathRules}) + if err != nil { + return nil, fmt.Errorf("failed to peek at environment: %w", err) + } + additionalPaths := make([]jpath.WeightedJPath, 0, 10) + for _, rule := range opts.AdditionalJPathRules { + if rule.Matches(labels.Set(peeked.Metadata.Labels)) { + additionalPaths = append(additionalPaths, &rule) + } + } + + opts.JsonnetOpts.AdditionalImportPaths = additionalPaths + + env, err := loader.Load(path, LoaderOpts{JsonnetOpts: opts.JsonnetOpts, Name: opts.Name}) if err != nil { return nil, err } @@ -89,7 +106,7 @@ func Peek(path string, opts Opts) (*v1alpha1.Environment, error) { return nil, err } - return loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) + return loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name, opts.AdditionalJPathRules}) } // List finds metadata of all environments at path that could possibly be @@ -101,7 +118,7 @@ func List(path string, opts Opts) ([]*v1alpha1.Environment, error) { return nil, err } - return loader.List(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) + return loader.List(path, LoaderOpts{opts.JsonnetOpts, opts.Name, opts.AdditionalJPathRules}) } func getJsonnetImplementation(path string, opts Opts) (types.JsonnetImplementation, error) { @@ -139,7 +156,7 @@ func Eval(path string, opts Opts) (interface{}, error) { return nil, err } - return loader.Eval(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) + return loader.Eval(path, LoaderOpts{opts.JsonnetOpts, opts.Name, opts.AdditionalJPathRules}) } // DetectLoader detects whether the environment is inline or static and picks @@ -188,7 +205,8 @@ type Loader interface { type LoaderOpts struct { JsonnetOpts - Name string + Name string + AdditionalJPathRules []tkrc.AdditionalJPath } type LoadResult struct { diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index 64c07bc9b..fdc409c30 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -2,12 +2,14 @@ package tanka import ( "fmt" + "os" "path/filepath" "strings" "time" "k8s.io/apimachinery/pkg/labels" + "github.com/grafana/tanka/internal/tkrc" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/pkg/errors" @@ -70,6 +72,14 @@ func parallelLoadEnvironments(envs []*v1alpha1.Environment, opts parallelOpts) ( if err != nil { return nil, errors.Wrap(err, "finding root") } + // Try to load the tkrc file if it exists + tkrcConfig, err := tkrc.Load(filepath.Join(rootDir, "tkrc.yaml")) + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to load tkrc.yaml: %w", err) + } + if tkrcConfig != nil { + o.AdditionalJPathRules = tkrcConfig.AdditionalJPaths + } jobsCh <- parallelJob{ path: filepath.Join(rootDir, path), opts: o, diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go index eee730ce7..256f60041 100644 --- a/pkg/tanka/tanka.go +++ b/pkg/tanka/tanka.go @@ -9,6 +9,7 @@ import ( "github.com/Masterminds/semver" + "github.com/grafana/tanka/internal/tkrc" "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/process" ) @@ -25,6 +26,10 @@ type Opts struct { // Name is used to extract a single environment from multiple environments Name string + + // AdditionalJPathRules are injected through the tkrc file and allow for + // dynamic lib/vendor folders + AdditionalJPathRules []tkrc.AdditionalJPath } // defaultDevVersion is the placeholder version used when no actual semver is From 91bfb4a37fe285133bbb3f57c0c342d2ad38c5dc Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Thu, 26 Jun 2025 13:13:54 +0200 Subject: [PATCH 3/5] Undo custom error message for failed peeking This adds zero value to the end-user and also breaks various tests. --- pkg/tanka/load.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index 509d7e0e3..edf741490 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -66,7 +66,7 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { // the necessary labels for doing matching additional jpaths: peeked, err := loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name, opts.AdditionalJPathRules}) if err != nil { - return nil, fmt.Errorf("failed to peek at environment: %w", err) + return nil, err } additionalPaths := make([]jpath.WeightedJPath, 0, 10) for _, rule := range opts.AdditionalJPathRules { From d08f95ab961361170cdf141f537d3ff79faa65c3 Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Mon, 7 Jul 2025 10:17:29 +0200 Subject: [PATCH 4/5] Remove unnecessary output --- pkg/tanka/export.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index c88f5092d..8346f4fca 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -86,10 +86,6 @@ func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnv return fmt.Errorf("deleting previously exported manifests from deleted environments: %w", err) } - for _, env := range envs { - fmt.Println(env.Metadata.Labels) - } - // get all environments for paths loadedEnvs, err := parallelLoadEnvironments(envs, parallelOpts{ Opts: opts.Opts, From e77686cad4706b54b90e6abee1c6935b3f528759 Mon Sep 17 00:00:00 2001 From: Horst Gutmann Date: Mon, 7 Jul 2025 10:58:04 +0200 Subject: [PATCH 5/5] Integrate with all commands that use LoadEnvironment This is especially intended for `show`. --- pkg/tanka/load.go | 18 ++++++++++++++++-- pkg/tanka/parallel.go | 10 ---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index edf741490..4ad7bcbc7 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -57,6 +57,20 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { return nil, err } + rootDir, err := jpath.FindRoot(path) + if err != nil { + return nil, errors.Wrap(err, "finding root") + } + // Try to load the tkrc file if it exists + var additionalJPathsRules []tkrc.AdditionalJPath + tkrcConfig, err := tkrc.Load(filepath.Join(rootDir, "tkrc.yaml")) + if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to load tkrc.yaml: %w", err) + } + if tkrcConfig != nil { + additionalJPathsRules = tkrcConfig.AdditionalJPaths + } + loader, err := DetectLoader(path, opts) if err != nil { return nil, err @@ -64,12 +78,12 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { // First, we need to take a peek at the environment so that we can extract // the necessary labels for doing matching additional jpaths: - peeked, err := loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name, opts.AdditionalJPathRules}) + peeked, err := loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name, additionalJPathsRules}) if err != nil { return nil, err } additionalPaths := make([]jpath.WeightedJPath, 0, 10) - for _, rule := range opts.AdditionalJPathRules { + for _, rule := range additionalJPathsRules { if rule.Matches(labels.Set(peeked.Metadata.Labels)) { additionalPaths = append(additionalPaths, &rule) } diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index fdc409c30..64c07bc9b 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -2,14 +2,12 @@ package tanka import ( "fmt" - "os" "path/filepath" "strings" "time" "k8s.io/apimachinery/pkg/labels" - "github.com/grafana/tanka/internal/tkrc" "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/pkg/errors" @@ -72,14 +70,6 @@ func parallelLoadEnvironments(envs []*v1alpha1.Environment, opts parallelOpts) ( if err != nil { return nil, errors.Wrap(err, "finding root") } - // Try to load the tkrc file if it exists - tkrcConfig, err := tkrc.Load(filepath.Join(rootDir, "tkrc.yaml")) - if err != nil && !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to load tkrc.yaml: %w", err) - } - if tkrcConfig != nil { - o.AdditionalJPathRules = tkrcConfig.AdditionalJPaths - } jobsCh <- parallelJob{ path: filepath.Join(rootDir, path), opts: o,