From 8abb4aa63950af3235009d66bcc58b4d91bf836c Mon Sep 17 00:00:00 2001 From: Jayme Bird Date: Fri, 14 Oct 2022 10:19:59 +0100 Subject: [PATCH] feat: implementation of manifestTomlEx in Go Co-authored-by: Wojciech Kocjan --- builtin-benchmarks/manifestTomlEx.jsonnet | 47 +++ builtins.go | 343 ++++++++++++++++++ builtins_benchmark_test.go | 4 + testdata/builtin_manifestTomlEx.golden | 4 + testdata/builtin_manifestTomlEx.jsonnet | 58 +++ testdata/builtin_manifestTomlEx.linter.golden | 0 testdata/builtin_manifestTomlEx_array.golden | 13 + testdata/builtin_manifestTomlEx_array.jsonnet | 12 + ...builtin_manifestTomlEx_array.linter.golden | 0 testdata/builtin_manifestTomlEx_null.golden | 13 + testdata/builtin_manifestTomlEx_null.jsonnet | 3 + .../builtin_manifestTomlEx_null.linter.golden | 0 12 files changed, 497 insertions(+) create mode 100644 builtin-benchmarks/manifestTomlEx.jsonnet create mode 100644 testdata/builtin_manifestTomlEx.golden create mode 100644 testdata/builtin_manifestTomlEx.jsonnet create mode 100644 testdata/builtin_manifestTomlEx.linter.golden create mode 100644 testdata/builtin_manifestTomlEx_array.golden create mode 100644 testdata/builtin_manifestTomlEx_array.jsonnet create mode 100644 testdata/builtin_manifestTomlEx_array.linter.golden create mode 100644 testdata/builtin_manifestTomlEx_null.golden create mode 100644 testdata/builtin_manifestTomlEx_null.jsonnet create mode 100644 testdata/builtin_manifestTomlEx_null.linter.golden diff --git a/builtin-benchmarks/manifestTomlEx.jsonnet b/builtin-benchmarks/manifestTomlEx.jsonnet new file mode 100644 index 0000000..6ecbe99 --- /dev/null +++ b/builtin-benchmarks/manifestTomlEx.jsonnet @@ -0,0 +1,47 @@ +{ + bar: { + prometheusOperator+: { + service+: { + spec+: { + ports: [ + { + name: 'https', + port: 8443, + targetPort: 'https', + }, + ], + }, + }, + serviceMonitor+: { + spec+: { + endpoints: [ + { + port: 'https', + scheme: 'https', + honorLabels: true, + bearerTokenFile: '/var/run/secrets/kubernetes.io/serviceaccount/token', + tlsConfig: { + insecureSkipVerify: true, + }, + }, + ], + }, + }, + clusterRole+: { + rules+: [ + { + apiGroups: ['authentication.k8s.io'], + resources: ['tokenreviews'], + verbs: ['create'], + }, + { + apiGroups: ['authorization.k8s.io'], + resources: ['subjectaccessreviews'], + verbs: ['create'], + }, + ], + }, + }, + }, + nothing: std.manifestTomlEx(self.bar, ' '), +} diff --git a/builtins.go b/builtins.go index d6b226d..3bae03c 100644 --- a/builtins.go +++ b/builtins.go @@ -1302,6 +1302,348 @@ func jsonEncode(v interface{}) (string, error) { return strings.TrimRight(buf.String(), "\n"), nil } +// tomlIsSection checks whether an object or array is a section - a TOML section is an +// object or an an array has all of its children being objects +func tomlIsSection(i *interpreter, val value) (bool, error) { + switch v := val.(type) { + case *valueObject: + return true, nil + case *valueArray: + if v.length() == 0 { + return false, nil + } + + for _, thunk := range v.elements { + thunkValue, err := thunk.getValue(i) + if err != nil { + return false, err + } + + switch thunkValue.(type) { + case *valueObject: + // this is expected, return true if all children are objects + default: + // return false if at least one child is not an object + return false, nil + } + } + + return true, nil + default: + return false, nil + } +} + +// tomlEncodeString encodes a string as quoted TOML string +func tomlEncodeString(s string) string { + res := "\"" + + for _, c := range s { + // escape specific characters, rendering non-ASCII ones as \uXXXX, + // appending remaining characters as is + if c == '"' { + res = res + "\\\"" + } else if c == '\\' { + res = res + "\\\\" + } else if c == '\b' { + res = res + "\\b" + } else if c == '\f' { + res = res + "\\f" + } else if c == '\n' { + res = res + "\\n" + } else if c == '\r' { + res = res + "\\r" + } else if c == '\t' { + res = res + "\\t" + } else if c < 32 || (c >= 127 && c <= 159) { + res = res + fmt.Sprintf("\\u%04x", c) + } else { + res = res + string(c) + } + } + + res = res + "\"" + + return res +} + +// tomlEncodeKey encodes a key - returning same string if it does not need quoting, +// otherwise return it quoted; returns empty key as '' +func tomlEncodeKey(s string) string { + bareAllowed := true + + // for empty string, return '' + if len(s) == 0 { + return "''" + } + + for _, c := range s { + if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' { + continue + } + + bareAllowed = false + break + } + + if bareAllowed { + return s + } + return tomlEncodeString(s) +} + +func tomlAddToPath(path []string, tail string) []string { + result := make([]string, 0, len(path)+1) + result = append(result, path...) + result = append(result, tail) + return result +} + +// tomlRenderValue returns a rendered value as string, with proper indenting +func tomlRenderValue(i *interpreter, val value, sindent string, indexedPath []string, inline bool, cindent string) (string, error) { + switch v := val.(type) { + case *valueNull: + return "", i.Error(fmt.Sprintf("Tried to manifest \"null\" at %v", indexedPath)) + case *valueBoolean: + return fmt.Sprintf("%t", v.value), nil + case *valueNumber: + return unparseNumber(v.value), nil + case valueString: + return tomlEncodeString(v.getGoString()), nil + case *valueFunction: + return "", i.Error(fmt.Sprintf("Tried to manifest function at %v", indexedPath)) + case *valueArray: + if len(v.elements) == 0 { + return "[]", nil + } + + // initialize indenting and separators based on whether this is added inline or not + newIndent := cindent + sindent + separator := "\n" + if inline { + newIndent = "" + separator = " " + } + + // open the square bracket to start array values + res := "[" + separator + + // iterate over elents and add their values to result + for j, thunk := range v.elements { + thunkValue, err := thunk.getValue(i) + if err != nil { + return "", err + } + + childIndexedPath := tomlAddToPath(indexedPath, strconv.FormatInt(int64(j), 10)) + + if j > 0 { + res = res + "," + separator + } + + res = res + newIndent + value, err := tomlRenderValue(i, thunkValue, sindent, childIndexedPath, true, "") + if err != nil { + return "", err + } + res = res + value + } + + res = res + separator + if inline { + res = res + cindent + } + + // close the array and return it + res = res + "]" + + return res, nil + case *valueObject: + res := "" + + fields := objectFields(v, withoutHidden) + sort.Strings(fields) + + // iterate over sorted field keys and render their values + for j, fieldName := range fields { + fieldValue, err := v.index(i, fieldName) + if err != nil { + return "", err + } + + childIndexedPath := tomlAddToPath(indexedPath, fieldName) + + value, err := tomlRenderValue(i, fieldValue, sindent, childIndexedPath, true, "") + if err != nil { + return "", err + } + + if j > 0 { + res = res + ", " + } + res = res + tomlEncodeKey(fieldName) + " = " + value + } + + // wrap fields in an array + return "{ " + res + " }", nil + default: + return "", i.Error(fmt.Sprintf("Unknown object type %v at %v", reflect.TypeOf(v), indexedPath)) + } +} + +func tomlRenderTableArray(i *interpreter, v *valueArray, sindent string, path []string, indexedPath []string, cindent string) (string, error) { + + sections := make([]string, 0, len(v.elements)) + + // render all elements of an array + for j, thunk := range v.elements { + thunkValue, err := thunk.getValue(i) + if err != nil { + return "", err + } + + switch tv := thunkValue.(type) { + case *valueObject: + // render the entire path as section name + section := cindent + "[[" + + for i, element := range path { + if i > 0 { + section = section + "." + } + section = section + tomlEncodeKey(element) + } + + section = section + "]]" + + // add newline if the table has elements + if len(objectFields(tv, withoutHidden)) > 0 { + section = section + "\n" + } + + childIndexedPath := tomlAddToPath(indexedPath, strconv.FormatInt(int64(j), 10)) + + // render the table and add it to result + table, err := tomlTableInternal(i, tv, sindent, path, childIndexedPath, cindent+sindent) + if err != nil { + return "", err + } + section = section + table + + sections = append(sections, section) + default: + return "", i.Error(fmt.Sprintf("invalid type for section: %v", reflect.TypeOf(thunkValue))) + } + } + + // combine all sections + return strings.Join(sections, "\n\n"), nil +} + +func tomlRenderTable(i *interpreter, v *valueObject, sindent string, path []string, indexedPath []string, cindent string) (string, error) { + res := cindent + "[" + for i, element := range path { + if i > 0 { + res = res + "." + } + res = res + tomlEncodeKey(element) + } + res = res + "]" + if len(objectFields(v, withoutHidden)) > 0 { + res = res + "\n" + } + + table, err := tomlTableInternal(i, v, sindent, path, indexedPath, cindent+sindent) + if err != nil { + return "", err + } + res = res + table + + return res, nil +} + +func tomlTableInternal(i *interpreter, v *valueObject, sindent string, path []string, indexedPath []string, cindent string) (string, error) { + resFields := []string{} + resSections := []string{""} + fields := objectFields(v, withoutHidden) + sort.Strings(fields) + + // iterate over non-section items + for _, fieldName := range fields { + fieldValue, err := v.index(i, fieldName) + if err != nil { + return "", err + } + + isSection, err := tomlIsSection(i, fieldValue) + if err != nil { + return "", err + } + + childIndexedPath := tomlAddToPath(indexedPath, fieldName) + + if isSection { + // render as section and add to array of sections + + childPath := tomlAddToPath(path, fieldName) + + switch fv := fieldValue.(type) { + case *valueObject: + section, err := tomlRenderTable(i, fv, sindent, childPath, childIndexedPath, cindent) + if err != nil { + return "", err + } + resSections = append(resSections, section) + case *valueArray: + section, err := tomlRenderTableArray(i, fv, sindent, childPath, childIndexedPath, cindent) + if err != nil { + return "", err + } + resSections = append(resSections, section) + default: + return "", i.Error(fmt.Sprintf("invalid type for section: %v", reflect.TypeOf(fieldValue))) + } + } else { + // render as value and append to result fields + + renderedValue, err := tomlRenderValue(i, fieldValue, sindent, childIndexedPath, false, "") + if err != nil { + return "", err + } + resFields = append(resFields, strings.Split(tomlEncodeKey(fieldName)+" = "+renderedValue, "\n")...) + } + } + + // create the result string + res := "" + + if len(resFields) > 0 { + res = "" + cindent + } + res = res + strings.Join(resFields, "\n"+cindent) + strings.Join(resSections, "\n\n") + return res, nil +} + +func builtinManifestTomlEx(i *interpreter, arguments []value) (value, error) { + val := arguments[0] + vindent, err := i.getString(arguments[1]) + if err != nil { + return nil, err + } + sindent := vindent.getGoString() + + switch v := val.(type) { + case *valueObject: + res, err := tomlTableInternal(i, v, sindent, []string{}, []string{}, "") + if err != nil { + return nil, err + } + return makeValueString(res), nil + default: + return nil, i.Error(fmt.Sprintf("TOML body must be an object. Got %s", v.getType().name)) + } +} + // We have a very similar logic here /interpreter.go@v0.16.0#L695 and here: /interpreter.go@v0.16.0#L627 // These should ideally be unified // For backwards compatibility reasons, we are manually marshalling to json so we can control formatting @@ -1735,6 +2077,7 @@ var funcBuiltins = buildBuiltinMap([]builtin{ &generalBuiltin{name: "manifestJsonEx", function: builtinManifestJSONEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"}, {name: "newline", defaultValue: &valueFlatString{value: []rune("\n")}}, {name: "key_val_sep", defaultValue: &valueFlatString{value: []rune(": ")}}}}, + &generalBuiltin{name: "manifestTomlEx", function: builtinManifestTomlEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"}}}, &unaryBuiltin{name: "base64", function: builtinBase64, params: ast.Identifiers{"input"}}, &unaryBuiltin{name: "encodeUTF8", function: builtinEncodeUTF8, params: ast.Identifiers{"str"}}, &unaryBuiltin{name: "decodeUTF8", function: builtinDecodeUTF8, params: ast.Identifiers{"arr"}}, diff --git a/builtins_benchmark_test.go b/builtins_benchmark_test.go index dcaaafb..3bdbe6a 100644 --- a/builtins_benchmark_test.go +++ b/builtins_benchmark_test.go @@ -60,6 +60,10 @@ func Benchmark_Builtin_manifestJsonEx(b *testing.B) { RunBenchmark(b, "manifestJsonEx") } +func Benchmark_Builtin_manifestTomlEx(b *testing.B) { + RunBenchmark(b, "manifestTomlEx") +} + func Benchmark_Builtin_comparison(b *testing.B) { RunBenchmark(b, "comparison") } diff --git a/testdata/builtin_manifestTomlEx.golden b/testdata/builtin_manifestTomlEx.golden new file mode 100644 index 0000000..8a56cd6 --- /dev/null +++ b/testdata/builtin_manifestTomlEx.golden @@ -0,0 +1,4 @@ +{ + "object": "abc = \"def\"\nbam = true\nbar = \"baz\"\nbaz = 1\nbazel = 1.4199999999999999\nbim = false\nboom = -1\nfoo = \"baz\"\n\n[blamo]\n cereal = [\n \"<>& fizbuzz\",\n [ \"a\", [ \"b\" ] ]\n ]\n\n [[blamo.treats]]\n name = \"chocolate\"", + "object2": "\"\\\"\" = 4\narray = [\n \"s\",\n 1,\n [ 2, 3 ],\n { a = [ \"0\", \"z\" ], r = 6 }\n]\nbool = true\nemptyArray = []\nkey = \"value\"\nnotBool = false\nnumber = 7\n\n[[arraySection]]\n q = 1\n\n[[arraySection]]\n w = 2\n\n[[emptyArraySection]]\n\n[emptySection]\n\n[\"escaped\\\"Section\"]\n z = \"q\"\n\n[section]\n a = 1\n\n [[section.array]]\n c = 3\n\n [[section.array]]\n d = 4\n\n [section.\"e$caped\"]\n q = \"t\"\n\n [section.nested]\n b = 2\n\n [[section.nestedArray]]\n k = \"v\"\n\n [section.nestedArray.nested]\n e = 5\n\n[simple]\n t = 5" +} diff --git a/testdata/builtin_manifestTomlEx.jsonnet b/testdata/builtin_manifestTomlEx.jsonnet new file mode 100644 index 0000000..96d643f --- /dev/null +++ b/testdata/builtin_manifestTomlEx.jsonnet @@ -0,0 +1,58 @@ +local object = { + foo: 'baz', + abc: 'def', + bar: self.foo, + baz: 1, + bazel: 1.42, + boom: -1, + bim: false, + bam: true, + blamo: { + cereal: [ + '<>& fizbuzz', + ['a', ['b']], + ], + + treats: [ + { + name: 'chocolate', + }, + ], + }, +}; + +local object2 = { + key: 'value', + simple: { t: 5 }, + section: { + a: 1, + nested: { b: 2 }, + 'e$caped': { q: 't' }, + array: [ + { c: 3 }, + { d: 4 }, + ], + nestedArray: [{ + k: 'v', + nested: { e: 5 }, + }], + }, + arraySection: [ + { q: 1 }, + { w: 2 }, + ], + 'escaped"Section': { z: 'q' }, + emptySection: {}, + emptyArraySection: [{}], + bool: true, + notBool: false, + number: 7, + array: ['s', 1, [2, 3], { r: 6, a: ['0', 'z'] }], + emptyArray: [], + '"': 4, +}; + +{ + object: std.manifestTomlEx(object, ' '), + object2: std.manifestTomlEx(object2, ' '), +} \ No newline at end of file diff --git a/testdata/builtin_manifestTomlEx.linter.golden b/testdata/builtin_manifestTomlEx.linter.golden new file mode 100644 index 0000000..e69de29 diff --git a/testdata/builtin_manifestTomlEx_array.golden b/testdata/builtin_manifestTomlEx_array.golden new file mode 100644 index 0000000..66d1cbe --- /dev/null +++ b/testdata/builtin_manifestTomlEx_array.golden @@ -0,0 +1,13 @@ +RUNTIME ERROR: TOML body must be an object. Got array +------------------------------------------------- + testdata/builtin_manifestTomlEx_array:11:10-41 object + + array: std.manifestTomlEx(array, ' '), + +------------------------------------------------- + Field "array" + +------------------------------------------------- + During manifestation + + diff --git a/testdata/builtin_manifestTomlEx_array.jsonnet b/testdata/builtin_manifestTomlEx_array.jsonnet new file mode 100644 index 0000000..8aea732 --- /dev/null +++ b/testdata/builtin_manifestTomlEx_array.jsonnet @@ -0,0 +1,12 @@ +local array = [ + 'bar', + 1, + 1.42, + -1, + false, + true, +]; + +{ + array: std.manifestTomlEx(array, ' '), +} diff --git a/testdata/builtin_manifestTomlEx_array.linter.golden b/testdata/builtin_manifestTomlEx_array.linter.golden new file mode 100644 index 0000000..e69de29 diff --git a/testdata/builtin_manifestTomlEx_null.golden b/testdata/builtin_manifestTomlEx_null.golden new file mode 100644 index 0000000..095ac3a --- /dev/null +++ b/testdata/builtin_manifestTomlEx_null.golden @@ -0,0 +1,13 @@ +RUNTIME ERROR: TOML body must be an object. Got null +------------------------------------------------- + testdata/builtin_manifestTomlEx_null:2:11-42 object + + 'null': std.manifestTomlEx(null, ' '), + +------------------------------------------------- + Field "null" + +------------------------------------------------- + During manifestation + + diff --git a/testdata/builtin_manifestTomlEx_null.jsonnet b/testdata/builtin_manifestTomlEx_null.jsonnet new file mode 100644 index 0000000..950532b --- /dev/null +++ b/testdata/builtin_manifestTomlEx_null.jsonnet @@ -0,0 +1,3 @@ +{ + 'null': std.manifestTomlEx(null, ' '), +} diff --git a/testdata/builtin_manifestTomlEx_null.linter.golden b/testdata/builtin_manifestTomlEx_null.linter.golden new file mode 100644 index 0000000..e69de29