From ed281bc5634b3adfd9ed48a82fb3f7206ab4dbd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Barzowski?= Date: Mon, 9 Oct 2017 12:53:34 -0400 Subject: [PATCH] Support for native callbacks --- builtins.go | 17 +++++++- desugarer.go | 6 +++ interpreter.go | 63 ++++++++++++++++++++++++----- main_test.go | 13 ++++++ testdata/native1.golden | 1 + testdata/native1.jsonnet | 1 + testdata/native2.golden | 1 + testdata/native2.jsonnet | 1 + testdata/native3.golden | 1 + testdata/native3.jsonnet | 1 + testdata/native4.golden | 15 +++++++ testdata/native4.jsonnet | 1 + testdata/native5.golden | 10 +++++ testdata/native5.jsonnet | 1 + testdata/native_nonexistent.golden | 8 ++++ testdata/native_nonexistent.jsonnet | 1 + thunks.go | 31 ++++++++++++++ vm.go | 31 ++++++++------ 18 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 testdata/native1.golden create mode 100644 testdata/native1.jsonnet create mode 100644 testdata/native2.golden create mode 100644 testdata/native2.jsonnet create mode 100644 testdata/native3.golden create mode 100644 testdata/native3.jsonnet create mode 100644 testdata/native4.golden create mode 100644 testdata/native4.jsonnet create mode 100644 testdata/native5.golden create mode 100644 testdata/native5.jsonnet create mode 100644 testdata/native_nonexistent.golden create mode 100644 testdata/native_nonexistent.jsonnet diff --git a/builtins.go b/builtins.go index 89faa98..44dc304 100644 --- a/builtins.go +++ b/builtins.go @@ -589,13 +589,27 @@ func builtinExtVar(e *evaluator, namep potentialValue) (value, error) { if err != nil { return nil, err } - index := ast.Identifier(name.getString()) + index := name.getString() if pv, ok := e.i.extVars[index]; ok { return e.evaluate(pv) } return nil, e.Error("Undefined external variable: " + string(index)) } +func builtinNative(e *evaluator, namep potentialValue) (value, error) { + name, err := e.evaluateString(namep) + if err != nil { + return nil, err + } + index := name.getString() + if f, exists := e.i.nativeFuncs[index]; exists { + return &valueFunction{ec: f}, nil + + } + return nil, e.Error(fmt.Sprintf("Unrecognized native function name: %v", index)) + +} + type unaryBuiltin func(*evaluator, potentialValue) (value, error) type binaryBuiltin func(*evaluator, potentialValue, potentialValue) (value, error) type ternaryBuiltin func(*evaluator, potentialValue, potentialValue, potentialValue) (value, error) @@ -772,6 +786,7 @@ var funcBuiltins = buildBuiltinMap([]builtin{ &BinaryBuiltin{name: "pow", function: builtinPow, parameters: ast.Identifiers{"base", "exp"}}, &BinaryBuiltin{name: "modulo", function: builtinModulo, parameters: ast.Identifiers{"x", "y"}}, &UnaryBuiltin{name: "md5", function: builtinMd5, parameters: ast.Identifiers{"x"}}, + &UnaryBuiltin{name: "native", function: builtinNative, parameters: ast.Identifiers{"x"}}, // internal &UnaryBuiltin{name: "$objectFlatMerge", function: builtinUglyObjectFlatMerge, parameters: ast.Identifiers{"x"}}, diff --git a/desugarer.go b/desugarer.go index 0eeb45e..d5b1630 100644 --- a/desugarer.go +++ b/desugarer.go @@ -298,6 +298,12 @@ func desugar(astPtr *ast.Node, objLevel int) (err error) { return } } + for i := range node.Arguments.Named { + err = desugar(&node.Arguments.Named[i].Arg, objLevel) + if err != nil { + return + } + } case *ast.ApplyBrace: err = desugar(&node.Left, objLevel) diff --git a/interpreter.go b/interpreter.go index 036171e..165d8ab 100644 --- a/interpreter.go +++ b/interpreter.go @@ -214,7 +214,10 @@ type interpreter struct { stack callStack // External variables - extVars map[ast.Identifier]potentialValue + extVars map[string]potentialValue + + // Native functions + nativeFuncs map[string]*nativeFunction // A part of std object common to all files baseStd valueObject @@ -730,6 +733,46 @@ func (i *interpreter) manifestAndSerializeJSON(trace *TraceElement, v value, mul return buf.String(), nil } +func jsonToValue(e *evaluator, v interface{}) (value, error) { + switch v := v.(type) { + case nil: + return &nullValue, nil + + case []interface{}: + elems := make([]potentialValue, len(v)) + for i, elem := range v { + val, err := jsonToValue(e, elem) + if err != nil { + return nil, err + } + elems[i] = &readyValue{val} + } + return makeValueArray(elems), nil + + case bool: + return makeValueBoolean(v), nil + case float64: + return makeValueNumber(v), nil + + case map[string]interface{}: + fieldMap := map[string]value{} + for name, f := range v { + val, err := jsonToValue(e, f) + if err != nil { + return nil, err + } + fieldMap[name] = val + } + return buildObject(ast.ObjectFieldInherit, fieldMap), nil + + case string: + return makeValueString(v), nil + + default: + return nil, e.Error(fmt.Sprintf("Not a json type: %#+v", v)) + } +} + func (i *interpreter) EvalInCleanEnv(fromWhere *TraceElement, env *environment, ast ast.Node, trimmable bool) (value, error) { err := i.newCall(fromWhere, *env, trimmable) if err != nil { @@ -776,8 +819,8 @@ func evaluateStd(i *interpreter) (value, error) { return i.EvalInCleanEnv(evalTrace, &beforeStdEnv, node, false) } -func prepareExtVars(i *interpreter, ext vmExtMap, kind string) map[ast.Identifier]potentialValue { - result := make(map[ast.Identifier]potentialValue) +func prepareExtVars(i *interpreter, ext vmExtMap, kind string) map[string]potentialValue { + result := make(map[string]potentialValue) for name, content := range ext { if content.isCode { varLoc := ast.MakeLocationRangeMessage("During evaluation") @@ -788,9 +831,9 @@ func prepareExtVars(i *interpreter, ext vmExtMap, kind string) map[ast.Identifie i: i, trace: varTrace, } - result[ast.Identifier(name)] = codeToPV(e, "<"+kind+":"+name+">", content.value) + result[name] = codeToPV(e, "<"+kind+":"+name+">", content.value) } else { - result[ast.Identifier(name)] = &readyValue{makeValueString(content.value)} + result[name] = &readyValue{makeValueString(content.value)} } } return result @@ -804,10 +847,11 @@ func buildObject(hide ast.ObjectFieldHide, fields map[string]value) valueObject return makeValueSimpleObject(bindingFrame{}, fieldMap, nil) } -func buildInterpreter(ext vmExtMap, maxStack int, importer Importer) (*interpreter, error) { +func buildInterpreter(ext vmExtMap, nativeFuncs map[string]*nativeFunction, maxStack int, importer Importer) (*interpreter, error) { i := interpreter{ stack: makeCallStack(maxStack), importCache: MakeImportCache(importer), + nativeFuncs: nativeFuncs, } stdObj, err := buildStdObject(&i) @@ -834,8 +878,9 @@ func makeInitialEnv(filename string, baseStd valueObject) environment { ) } -func evaluate(node ast.Node, ext vmExtMap, tla vmExtMap, maxStack int, importer Importer) (string, error) { - i, err := buildInterpreter(ext, maxStack, importer) +// TODO(sbarzowski) this function takes far too many arguments - build interpreter in vm instead +func evaluate(node ast.Node, ext vmExtMap, tla vmExtMap, nativeFuncs map[string]*nativeFunction, maxStack int, importer Importer) (string, error) { + i, err := buildInterpreter(ext, nativeFuncs, maxStack, importer) if err != nil { return "", err } @@ -854,7 +899,7 @@ func evaluate(node ast.Node, ext vmExtMap, tla vmExtMap, maxStack int, importer toplevelArgMap := prepareExtVars(i, tla, "top-level-arg") args := callArguments{} for argName, pv := range toplevelArgMap { - args.named = append(args.named, namedCallArgument{name: argName, pv: pv}) + args.named = append(args.named, namedCallArgument{name: ast.Identifier(argName), pv: pv}) } funcLoc := ast.MakeLocationRangeMessage("Top-level-function") funcTrace := &TraceElement{ diff --git a/main_test.go b/main_test.go index 64141f6..bfa2e9c 100644 --- a/main_test.go +++ b/main_test.go @@ -18,6 +18,7 @@ package jsonnet import ( "bytes" + "encoding/json" "flag" "io/ioutil" "path/filepath" @@ -25,6 +26,7 @@ import ( "testing" "unicode/utf8" + "github.com/google/go-jsonnet/ast" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -100,6 +102,17 @@ func TestMain(t *testing.T) { vm.ExtCode(name, value) } + vm.NativeFunction(&nativeFunction{ + name: "jsonToString", + params: ast.Identifiers{"x"}, + f: func(x []interface{}) (interface{}, error) { + bytes, err := json.Marshal(x[0]) + if err != nil { + return nil, err + } + return string(bytes), nil + }, + }) read := func(file string) []byte { bytz, err := ioutil.ReadFile(file) if err != nil { diff --git a/testdata/native1.golden b/testdata/native1.golden new file mode 100644 index 0000000..fbb132a --- /dev/null +++ b/testdata/native1.golden @@ -0,0 +1 @@ +"\"test\"" diff --git a/testdata/native1.jsonnet b/testdata/native1.jsonnet new file mode 100644 index 0000000..772273f --- /dev/null +++ b/testdata/native1.jsonnet @@ -0,0 +1 @@ +std.native("jsonToString")("test") diff --git a/testdata/native2.golden b/testdata/native2.golden new file mode 100644 index 0000000..71829ad --- /dev/null +++ b/testdata/native2.golden @@ -0,0 +1 @@ +"{}" diff --git a/testdata/native2.jsonnet b/testdata/native2.jsonnet new file mode 100644 index 0000000..1a3b604 --- /dev/null +++ b/testdata/native2.jsonnet @@ -0,0 +1 @@ +std.native("jsonToString")({}) diff --git a/testdata/native3.golden b/testdata/native3.golden new file mode 100644 index 0000000..fb7ccc1 --- /dev/null +++ b/testdata/native3.golden @@ -0,0 +1 @@ +"{\"x\":{\"y\":\"z\"},\"xx\":42}" diff --git a/testdata/native3.jsonnet b/testdata/native3.jsonnet new file mode 100644 index 0000000..f444c7f --- /dev/null +++ b/testdata/native3.jsonnet @@ -0,0 +1 @@ +std.native("jsonToString")({"x": {"y": "z"}, "xx": 42}) diff --git a/testdata/native4.golden b/testdata/native4.golden new file mode 100644 index 0000000..f164203 --- /dev/null +++ b/testdata/native4.golden @@ -0,0 +1,15 @@ +RUNTIME ERROR: xxx +------------------------------------------------- + testdata/native4:1:28-38 thunk from <$> + +std.native("jsonToString")(error "xxx") + +------------------------------------------------- + testdata/native4:1:1-40 $ + +std.native("jsonToString")(error "xxx") + +------------------------------------------------- + During evaluation + + diff --git a/testdata/native4.jsonnet b/testdata/native4.jsonnet new file mode 100644 index 0000000..48c65cf --- /dev/null +++ b/testdata/native4.jsonnet @@ -0,0 +1 @@ +std.native("jsonToString")(error "xxx") diff --git a/testdata/native5.golden b/testdata/native5.golden new file mode 100644 index 0000000..65785cf --- /dev/null +++ b/testdata/native5.golden @@ -0,0 +1,10 @@ +RUNTIME ERROR: Couldn't manifest function in JSON output. +------------------------------------------------- + testdata/native5:1:1-42 $ + +std.native("jsonToString")(function() 42) + +------------------------------------------------- + During evaluation + + diff --git a/testdata/native5.jsonnet b/testdata/native5.jsonnet new file mode 100644 index 0000000..afca904 --- /dev/null +++ b/testdata/native5.jsonnet @@ -0,0 +1 @@ +std.native("jsonToString")(function() 42) diff --git a/testdata/native_nonexistent.golden b/testdata/native_nonexistent.golden new file mode 100644 index 0000000..b10018d --- /dev/null +++ b/testdata/native_nonexistent.golden @@ -0,0 +1,8 @@ +RUNTIME ERROR: Unrecognized native function name: blah +------------------------------------------------- + builtin function + +------------------------------------------------- + During evaluation + + diff --git a/testdata/native_nonexistent.jsonnet b/testdata/native_nonexistent.jsonnet new file mode 100644 index 0000000..e4e807f --- /dev/null +++ b/testdata/native_nonexistent.jsonnet @@ -0,0 +1 @@ +std.native("blah") diff --git a/thunks.go b/thunks.go index 8ab748a..ff68d67 100644 --- a/thunks.go +++ b/thunks.go @@ -287,6 +287,37 @@ func makeClosure(env environment, function *ast.Function) *closure { } } +type nativeFunction struct { + f func([]interface{}) (interface{}, error) + params ast.Identifiers + name string +} + +func (native *nativeFunction) EvalCall(arguments callArguments, e *evaluator) (value, error) { + flatArgs := flattenArgs(arguments, native.Parameters()) + nativeArgs := make([]interface{}, 0, len(flatArgs)) + for _, arg := range flatArgs { + v, err := e.evaluate(arg) + if err != nil { + return nil, err + } + json, err := e.i.manifestJSON(e.trace, v) + if err != nil { + return nil, err + } + nativeArgs = append(nativeArgs, json) + } + resultJSON, err := native.f(nativeArgs) + if err != nil { + return nil, err + } + return jsonToValue(e, resultJSON) +} + +func (native *nativeFunction) Parameters() Parameters { + return Parameters{required: native.params} +} + // partialPotentialValue // ------------------------------------- diff --git a/vm.go b/vm.go index 3043b5a..7da10fd 100644 --- a/vm.go +++ b/vm.go @@ -31,12 +31,13 @@ import ( // VM is the core interpreter and is the touchpoint used to parse and execute // Jsonnet. type VM struct { - MaxStack int - MaxTrace int // The number of lines of stack trace to display (0 for all of them). - ext vmExtMap - tla vmExtMap - importer Importer - ef ErrorFormatter + MaxStack int + MaxTrace int // The number of lines of stack trace to display (0 for all of them). + ext vmExtMap + tla vmExtMap + nativeFuncs map[string]*nativeFunction + importer Importer + ef ErrorFormatter } // External variable or top level argument provided before execution @@ -53,11 +54,12 @@ type vmExtMap map[string]vmExt // MakeVM creates a new VM with default parameters. func MakeVM() *VM { return &VM{ - MaxStack: 500, - ext: make(vmExtMap), - tla: make(vmExtMap), - ef: ErrorFormatter{pretty: true, colorful: true, MaxStackTraceSize: 20}, - importer: &FileImporter{}, + MaxStack: 500, + ext: make(vmExtMap), + tla: make(vmExtMap), + nativeFuncs: make(map[string]*nativeFunction), + ef: ErrorFormatter{pretty: true, colorful: true, MaxStackTraceSize: 20}, + importer: &FileImporter{}, } } @@ -96,13 +98,18 @@ func (vm *VM) evaluateSnippet(filename string, snippet string) (output string, e if err != nil { return "", err } - output, err = evaluate(node, vm.ext, vm.tla, vm.MaxStack, vm.importer) + output, err = evaluate(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer) if err != nil { return "", err } return output, nil } +// NativeFunction registers a native function +func (vm *VM) NativeFunction(f *nativeFunction) { + vm.nativeFuncs[f.name] = f +} + // EvaluateSnippet evaluates a string containing Jsonnet code, return a JSON // string. //