feat: state values

This adds `values` to state files as proposed in #640.

```yaml
values:
- key1: val1
- defaults.yaml

environments:
  default:
  - values:
    - environments/default.yaml
  production:
  - values:
    - environments/production.yaml

```

Resolves #640

Author: mumoshu

pkg/app/app_test.go

@@ -900,6 +900,130 @@ bar: "bar1"
 	}
 }
 
+func TestVisitDesiredStatesWithReleasesFiltered_StateValueOverrides(t *testing.T) {
+	envTmplExpr := "{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \"b\" | first | first | pluck \"c\" | first }}"
+	relTmplExpr := "\"{{`{{ .Values.foo }}-{{ .Values.bar }}-{{ .Values.baz }}-{{ .Values.hoge }}-{{ .Values.fuga }}-{{ .Values.a | first | pluck \\\"b\\\" | first | first | pluck \\\"c\\\" | first }}`}}\""
+
+	testcases := []struct {
+		expr, env, expected string
+	}{
+		{
+			expr:     envTmplExpr,
+			env:      "default",
+			expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C",
+		},
+		{
+			expr:     envTmplExpr,
+			env:      "production",
+			expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C",
+		},
+		{
+			expr:     relTmplExpr,
+			env:      "default",
+			expected: "foo-bar_default-baz_override-hoge_set-fuga_set-C",
+		},
+		{
+			expr:     relTmplExpr,
+			env:      "production",
+			expected: "foo-bar_production-baz_override-hoge_set-fuga_set-C",
+		},
+	}
+	for i := range testcases {
+		testcase := testcases[i]
+		t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
+			files := map[string]string{
+				"/path/to/helmfile.yaml": fmt.Sprintf(`
+# The top-level "values" are "base" values has inherited to state values with the lowest priority.
+# The lowest priority results in environment-specific values to override values defined in the base.
+values:
+- values.yaml
+
+environments:
+  default:
+    values:
+    - default.yaml
+  production:
+    values:
+    - production.yaml
+---
+releases:
+- name: %s
+  chart: %s
+  namespace: %s
+`, testcase.expr, testcase.expr, testcase.expr),
+				"/path/to/values.yaml": `
+foo: foo
+bar: bar
+baz: baz
+hoge: hoge
+fuga: fuga
+
+a: []
+`,
+				"/path/to/default.yaml": `
+bar: "bar_default"
+baz: "baz_default"
+
+a:
+- b: []
+`,
+				"/path/to/production.yaml": `
+bar: "bar_production"
+baz: "baz_production"
+
+a:
+- b: []
+`,
+				"/path/to/overrides.yaml": `
+baz: baz_override
+hoge: hoge_override
+
+a:
+- b:
+  - c: C
+`,
+			}
+
+			actual := []state.ReleaseSpec{}
+
+			collectReleases := func(st *state.HelmState, helm helmexec.Interface) []error {
+				for _, r := range st.Releases {
+					actual = append(actual, r)
+				}
+				return []error{}
+			}
+			app := appWithFs(&App{
+				KubeContext: "default",
+				Logger:      helmexec.NewLogger(os.Stderr, "debug"),
+				Reverse:     false,
+				Namespace:   "",
+				Selectors:   []string{},
+				Env:         testcase.env,
+				ValuesFiles: []string{"overrides.yaml"},
+				Set:         map[string]interface{}{"hoge": "hoge_set", "fuga": "fuga_set"},
+			}, files)
+			err := app.VisitDesiredStatesWithReleasesFiltered(
+				"helmfile.yaml", collectReleases,
+			)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if len(actual) != 1 {
+				t.Errorf("unexpected number of processed releases: expected=1, got=%d", len(actual))
+			}
+			if actual[0].Name != testcase.expected {
+				t.Errorf("unexpected name: expected=%s, got=%s", testcase.expected, actual[0].Name)
+			}
+			if actual[0].Chart != testcase.expected {
+				t.Errorf("unexpected chart: expected=%s, got=%s", testcase.expected, actual[0].Chart)
+			}
+			if actual[0].Namespace != testcase.expected {
+				t.Errorf("unexpected namespace: expected=%s, got=%s", testcase.expected, actual[0].Namespace)
+			}
+		})
+	}
+}
+
 func TestLoadDesiredStateFromYaml_DuplicateReleaseName(t *testing.T) {
 	yamlFile := "example/path/to/yaml/file"
 	yamlContent := []byte(`releases:

pkg/app/desired_state_file_loader.go

@@ -83,6 +83,7 @@ func (ld *desiredStateLoader) Load(f string, opts LoadOpts) (*state.HelmState, e
 func (ld *desiredStateLoader) loadFile(inheritedEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	return ld.loadFileWithOverrides(inheritedEnv, nil, baseDir, file, evaluateBases)
 }
+
 func (ld *desiredStateLoader) loadFileWithOverrides(inheritedEnv, overrodeEnv *environment.Environment, baseDir, file string, evaluateBases bool) (*state.HelmState, error) {
 	var f string
 	if filepath.IsAbs(file) {

pkg/app/two_pass_renderer.go

@@ -19,8 +19,12 @@ func prependLineNumbers(text string) string {
 	return buf.String()
 }
 
-func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) *environment.Environment {
-	tmplData := state.EnvironmentTemplateData{Environment: *firstPassEnv, Namespace: r.namespace}
+func (r *desiredStateLoader) renderPrestate(firstPassEnv *environment.Environment, baseDir, filename string, content []byte) (*environment.Environment, *state.HelmState) {
+	tmplData := state.EnvironmentTemplateData{
+		Environment: *firstPassEnv,
+		Namespace:   r.namespace,
+		Values:      map[string]interface{}{},
+	}
 	firstPassRenderer := tmpl.NewFirstPassRenderer(baseDir, tmplData)
 
 	// parse as much as we can, tolerate errors, this is a preparse
@@ -29,7 +33,7 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ
 		r.logger.Debugf("first-pass rendering input of \"%s\":\n%s", filename, prependLineNumbers(string(content)))
 		if yamlBuf == nil { // we have a template syntax error, let the second parse report
 			r.logger.Debugf("template syntax error: %v", err)
-			return firstPassEnv
+			return firstPassEnv, nil
 		}
 	}
 	yamlData := yamlBuf.String()
@@ -57,7 +61,8 @@ func (r *desiredStateLoader) renderEnvironment(firstPassEnv *environment.Environ
 	if prestate != nil {
 		firstPassEnv = &prestate.Env
 	}
-	return firstPassEnv
+
+	return firstPassEnv, prestate
 }
 
 type RenderOpts struct {
@@ -88,13 +93,18 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
 		r.logger.Debugf("first-pass uses: %v", initEnv)
 	}
 
-	renderedEnv := r.renderEnvironment(initEnv, baseDir, filename, content)
+	renderedEnv, prestate := r.renderPrestate(initEnv, baseDir, filename, content)
 
 	if r.logger != nil {
 		r.logger.Debugf("first-pass produced: %v", renderedEnv)
 	}
 
-	finalEnv, err := renderedEnv.Merge(overrode)
+	finalEnv, err := inherited.Merge(renderedEnv)
+	if err != nil {
+		return nil, err
+	}
+
+	finalEnv, err = finalEnv.Merge(overrode)
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +113,23 @@ func (r *desiredStateLoader) twoPassRenderTemplateToYaml(inherited, overrode *en
 		r.logger.Debugf("first-pass rendering result of \"%s\": %v", filename, *finalEnv)
 	}
 
-	tmplData := state.EnvironmentTemplateData{Environment: *finalEnv, Namespace: r.namespace}
+	vals := map[string]interface{}{}
+	if prestate != nil {
+		prestate.Env = *finalEnv
+		vals, err = prestate.Values()
+		if err != nil {
+			return nil, err
+		}
+	}
+	if prestate != nil {
+		r.logger.Debugf("vals:\n%v\ndefaultVals:%v", vals, prestate.DefaultValues)
+	}
+
+	tmplData := state.EnvironmentTemplateData{
+		Environment: *finalEnv,
+		Namespace:   r.namespace,
+		Values:      vals,
+	}
 	secondPassRenderer := tmpl.NewFileRenderer(r.readFile, baseDir, tmplData)
 	yamlBuf, err := secondPassRenderer.RenderTemplateContentToBuffer(content)
 	if err != nil {

pkg/environment/environment.go

@@ -7,28 +7,44 @@ import (
 )
 
 type Environment struct {
-	Name   string
-	Values map[string]interface{}
+	Name     string
+	Values   map[string]interface{}
+	Defaults map[string]interface{}
 }
 
 var EmptyEnvironment Environment
 
 func (e Environment) DeepCopy() Environment {
-	bytes, err := yaml.Marshal(e.Values)
+	valuesBytes, err := yaml.Marshal(e.Values)
 	if err != nil {
 		panic(err)
 	}
 	var values map[string]interface{}
-	if err := yaml.Unmarshal(bytes, &values); err != nil {
+	if err := yaml.Unmarshal(valuesBytes, &values); err != nil {
 		panic(err)
 	}
 	values, err = maputil.CastKeysToStrings(values)
 	if err != nil {
 		panic(err)
 	}
+
+	defaultsBytes, err := yaml.Marshal(e.Defaults)
+	if err != nil {
+		panic(err)
+	}
+	var defaults map[string]interface{}
+	if err := yaml.Unmarshal(defaultsBytes, &defaults); err != nil {
+		panic(err)
+	}
+	defaults, err = maputil.CastKeysToStrings(defaults)
+	if err != nil {
+		panic(err)
+	}
+
 	return Environment{
-		Name:   e.Name,
-		Values: values,
+		Name:     e.Name,
+		Values:   values,
+		Defaults: defaults,
 	}
 }
 

pkg/maputil/maputil_test.go

@@ -0,0 +1,61 @@
+package maputil
+
+import "testing"
+
+func TestMapUtil_StrKeys(t *testing.T) {
+	m := map[string]interface{}{
+		"a": []interface{}{
+			map[string]interface{}{
+				"b": []interface{}{
+					map[string]interface{}{
+						"c": "C",
+					},
+				},
+			},
+		},
+	}
+
+	r, err := CastKeysToStrings(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	a := r["a"].([]interface{})
+	a0 := a[0].(map[string]interface{})
+	b := a0["b"].([]interface{})
+	b0 := b[0].(map[string]interface{})
+	c := b0["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}
+
+func TestMapUtil_IFKeys(t *testing.T) {
+	m := map[interface{}]interface{}{
+		"a": []interface{}{
+			map[interface{}]interface{}{
+				"b": []interface{}{
+					map[interface{}]interface{}{
+						"c": "C",
+					},
+				},
+			},
+		},
+	}
+
+	r, err := CastKeysToStrings(m)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	a := r["a"].([]interface{})
+	a0 := a[0].(map[string]interface{})
+	b := a0["b"].([]interface{})
+	b0 := b[0].(map[string]interface{})
+	c := b0["c"]
+
+	if c != "C" {
+		t.Errorf("unexpected c: expected=C, got=%s", c)
+	}
+}

pkg/state/create.go

@@ -114,6 +114,12 @@ func (c *StateCreator) LoadEnvValues(target *HelmState, env string, ctxEnv *envi
 	if err != nil {
 		return nil, &StateLoadError{fmt.Sprintf("failed to read %s", state.FilePath), err}
 	}
+
+	e.Defaults, err = state.loadValuesEntries(nil, state.DefaultValues)
+	if err != nil {
+		return nil, err
+	}
+
 	state.Env = *e
 
 	return &state, nil
@@ -137,7 +143,12 @@ func (c *StateCreator) ParseAndLoad(content []byte, baseDir, file string, envNam
 		return nil, err
 	}
 
-	return c.LoadEnvValues(state, envName, envValues)
+	state, err = c.LoadEnvValues(state, envName, envValues)
+	if err != nil {
+		return nil, err
+	}
+
+	return state, nil
 }
 
 func (c *StateCreator) loadBases(envValues *environment.Environment, st *HelmState, baseDir string) (*HelmState, error) {
@@ -164,13 +175,8 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 	envVals := map[string]interface{}{}
 	envSpec, ok := st.Environments[name]
 	if ok {
-		envValues := append([]interface{}{}, envSpec.Values...)
-		ld := &EnvironmentValuesLoader{
-			storage:  st.storage(),
-			readFile: st.readFile,
-		}
 		var err error
-		envVals, err = ld.LoadEnvironmentValues(envSpec.MissingFileHandler, envValues)
+		envVals, err = st.loadValuesEntries(envSpec.MissingFileHandler, envSpec.Values)
 		if err != nil {
 			return nil, err
 		}
@@ -237,3 +243,20 @@ func (st *HelmState) loadEnvValues(name string, ctxEnv *environment.Environment,
 
 	return newEnv, nil
 }
+
+func (st *HelmState) loadValuesEntries(missingFileHandler *string, entries []interface{}) (map[string]interface{}, error) {
+	envVals := map[string]interface{}{}
+
+	valuesEntries := append([]interface{}{}, entries...)
+	ld := &EnvironmentValuesLoader{
+		storage:  st.storage(),
+		readFile: st.readFile,
+	}
+	var err error
+	envVals, err = ld.LoadEnvironmentValues(missingFileHandler, valuesEntries)
+	if err != nil {
+		return nil, err
+	}
+
+	return envVals, nil
+}