Extract AST processing to separate packages

Making it independent from the jsonnet package breaks the circular
dependency during stdast generation.

BUILD.bazel

@@ -13,13 +13,11 @@ go_library(
     name = "go_default_library",
     srcs = [
         "builtins.go",
-        "desugarer.go",
         "doc.go",
         "error_formatter.go",
         "imports.go",
         "interpreter.go",
         "runtime_error.go",
-        "static_analyzer.go",
         "thunks.go",
         "value.go",
         "vm.go",
@@ -28,6 +26,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//ast:go_default_library",
+        "//internal/transformations:go_default_library",
         "//parser:go_default_library",
     ],
 )
@@ -35,11 +34,9 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = [
-        "desugarer_test.go",
         "interpreter_test.go",
         "jsonnet_test.go",
         "main_test.go",
-        "static_analyzer_test.go",
     ],
     data = glob(["testdata/**"]),
     embed = [":go_default_library"],

README.md

@@ -47,6 +47,11 @@ Windows         | _bazel build --platforms=@io_bazel_rules_go//go/toolchain:wind
 
 For additional target platform names, see the per-Go release definitions [here](https://github.com/bazelbuild/rules_go/blob/master/go/private/sdk_list.bzl#L21-L31) in the _rules_go_ Bazel package.
 
+Additionally if files were moved around, you may need to run the following command to update Bazel files:
+```
+bazel run //:gazelle
+```
+
 ## Build instructions (go 1.8 - 1.10)
 
 ```bash

ast/clone.go

@@ -298,6 +298,7 @@ func clone(astPtr *Node) {
 	cloneNodeBase(*astPtr)
 }
 
+// Clone creates an independent copy of an AST
 func Clone(astPtr Node) Node {
 	clone(&astPtr)
 	return astPtr

builtins.go

@@ -1027,15 +1027,10 @@ func (b *ternaryBuiltin) Name() ast.Identifier {
 	return b.name
 }
 
-var desugaredBop = map[ast.BinaryOp]ast.Identifier{
-	ast.BopPercent: "mod",
-	ast.BopIn:      "objectHasAll",
-}
-
 var bopBuiltins = []*binaryBuiltin{
+	// Note that % and `in` are desugared instead of being handled here
 	ast.BopMult: &binaryBuiltin{name: "operator*", function: builtinMult, parameters: ast.Identifiers{"x", "y"}},
 	ast.BopDiv:  &binaryBuiltin{name: "operator/", function: builtinDiv, parameters: ast.Identifiers{"x", "y"}},
-	// ast.BopPercent:  <desugared>,
 
 	ast.BopPlus:  &binaryBuiltin{name: "operator+", function: builtinPlus, parameters: ast.Identifiers{"x", "y"}},
 	ast.BopMinus: &binaryBuiltin{name: "operator-", function: builtinMinus, parameters: ast.Identifiers{"x", "y"}},

c-bindings/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
         "internal.h",
         "json.cpp",
         "json.h",
+        "libgojsonnet.h",
         "libjsonnet.cpp",
     ],
     cdeps = [

cmd/dumpstdlibast/BUILD.bazel

@@ -6,8 +6,8 @@ go_library(
     importpath = "github.com/google/go-jsonnet/cmd/dumpstdlibast",
     visibility = ["//visibility:private"],
     deps = [
-        "//:go_default_library",
+        "//internal/dump:go_default_library",
-        "//dump:go_default_library",
+        "//internal/transformations:go_default_library",
     ],
 )
 

cmd/dumpstdlibast/dumpstdlibast.go

@@ -19,8 +19,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/google/go-jsonnet"
+	"github.com/google/go-jsonnet/internal/dump"
-	"github.com/google/go-jsonnet/dump"
+	"github.com/google/go-jsonnet/internal/transformations"
 )
 
 func main() {
@@ -33,7 +33,7 @@ func main() {
 		panic(err)
 	}
 
-	node, err := jsonnet.SnippetToAST("<std>", string(buf))
+	node, err := transformations.SnippetToAST("<std>", string(buf))
 	if err != nil {
 		panic(err)
 	}

imports.go

@@ -107,7 +107,7 @@ func (cache *importCache) importString(importedFrom, importedPath string, i *int
 }
 
 func codeToPV(i *interpreter, filename string, code string) *cachedThunk {
-	node, err := snippetToAST(filename, code)
+	node, err := SnippetToAST(filename, code)
 	if err != nil {
 		// TODO(sbarzowski) we should wrap (static) error here
 		// within a RuntimeError. Because whether we get this error or not

internal/dump/BUILD.bazel

@@ -7,7 +7,7 @@ go_library(
         "pointermap.go",
         "utils.go",
     ],
-    importpath = "github.com/google/go-jsonnet/dump",
+    importpath = "github.com/google/go-jsonnet/internal/dump",
     visibility = ["//visibility:public"],
 )
 

internal/transformations/BUILD.bazel

@@ -0,0 +1,26 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "desugarer.go",
+        "static_analyzer.go",
+        "transformations.go",
+    ],
+    importpath = "github.com/google/go-jsonnet/internal/transformations",
+    visibility = ["//:__subpackages__"],
+    deps = [
+        "//ast:go_default_library",
+        "//parser:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "desugarer_test.go",
+        "static_analyzer_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = ["//ast:go_default_library"],
+)

internal/transformations/desugarer.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package jsonnet
+package transformations
 
 import (
 	"bytes"
@@ -27,6 +27,11 @@ import (
 	"github.com/google/go-jsonnet/parser"
 )
 
+var desugaredBop = map[ast.BinaryOp]ast.Identifier{
+	ast.BopPercent: "mod",
+	ast.BopIn:      "objectHasAll",
+}
+
 func makeStr(s string) *ast.LiteralString {
 	return &ast.LiteralString{
 		NodeBase:    ast.NodeBase{},
@@ -232,8 +237,6 @@ func wrapInArray(inside ast.Node) ast.Node {
 	return &ast.Array{Elements: ast.Nodes{inside}}
 }
 
-
-
 func desugarArrayComp(comp *ast.ArrayComp, objLevel int) (ast.Node, error) {
 	err := desugar(&comp.Body, objLevel)
 	if err != nil {
@@ -301,17 +304,6 @@ func desugarLocalBinds(binds ast.LocalBinds, objLevel int) (err error) {
 	return nil
 }
 
-// Desugar Jsonnet expressions to reduce the number of constructs the rest of the implementation
-// needs to understand.
-//
-// Note that despite the name, desugar() is not idempotent.  String literals have their escape
-// codes translated to low-level characters during desugaring.
-//
-// Desugaring should happen immediately after parsing, i.e. before static analysis and execution.
-// Temporary variables introduced here should be prefixed with $ to ensure they do not clash with
-// variables used in user code.
-// TODO(sbarzowski) Actually we may want to do some static analysis before desugaring, e.g.
-// warning user about dangerous use of constructs that we desugar.
 func desugar(astPtr *ast.Node, objLevel int) (err error) {
 	node := *astPtr
 
@@ -579,7 +571,18 @@ func desugar(astPtr *ast.Node, objLevel int) (err error) {
 	return nil
 }
 
-func desugarFile(ast *ast.Node) error {
+// Desugar Jsonnet expressions to reduce the number of constructs the rest of the implementation
+// needs to understand.
+//
+// Note that despite the name, desugar() is not idempotent.  String literals have their escape
+// codes translated to low-level characters during desugaring.
+//
+// Desugaring should happen immediately after parsing, i.e. before static analysis and execution.
+// Temporary variables introduced here should be prefixed with $ to ensure they do not clash with
+// variables used in user code.
+// TODO(sbarzowski) Actually we may want to do some static analysis before desugaring, e.g.
+// warning user about dangerous use of constructs that we desugar.
+func Desugar(ast *ast.Node) error {
 	err := desugar(ast, 0)
 	if err != nil {
 		return err

internal/transformations/desugarer_test.go

@@ -14,4 +14,4 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package jsonnet
+package transformations

internal/transformations/static_analyzer.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package jsonnet
+package transformations
 
 import (
 	"fmt"
@@ -164,6 +164,9 @@ func analyzeVisit(a ast.Node, inObject bool, vars ast.IdentifierSet) error {
 	return s.err
 }
 
-func analyze(node ast.Node) error {
+// Analyze checks variable references (these could be checked statically in Jsonnet).
+// It enriches ast with additional information about free variables in every node,
+// so it is necessary to always run it before executing AST.
+func Analyze(node ast.Node) error {
 	return analyzeVisit(node, false, ast.NewIdentifierSet("std"))
 }

internal/transformations/static_analyzer_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package jsonnet
+package transformations
 
 import (
 	"testing"
@@ -28,7 +28,7 @@ import (
 
 func TestSimpleNull(t *testing.T) {
 	ast := &ast.LiteralNull{}
-	err := analyze(ast)
+	err := Analyze(ast)
 	if err != nil {
 		t.Errorf("Unexpected error: %+v", err)
 	}
@@ -60,7 +60,7 @@ func TestSimpleLocal(t *testing.T) {
 		Body: &ast.Var{Id: "x"},
 	}
 
-	err := analyze(node)
+	err := Analyze(node)
 	if err != nil {
 		t.Errorf("Unexpected error: %+v", err)
 	}

internal/transformations/transformations.go

@@ -0,0 +1,31 @@
+package transformations
+
+import (
+	"github.com/google/go-jsonnet/ast"
+	"github.com/google/go-jsonnet/parser"
+)
+
+func snippetToRawAST(filename string, snippet string) (ast.Node, error) {
+	tokens, err := parser.Lex(filename, snippet)
+	if err != nil {
+		return nil, err
+	}
+	return parser.Parse(tokens)
+}
+
+// SnippetToAST converts Jsonnet code snippet to desugared and analyzed AST
+func SnippetToAST(filename string, snippet string) (ast.Node, error) {
+	node, err := snippetToRawAST(filename, snippet)
+	if err != nil {
+		return nil, err
+	}
+	err = Desugar(&node)
+	if err != nil {
+		return nil, err
+	}
+	err = Analyze(node)
+	if err != nil {
+		return nil, err
+	}
+	return node, nil
+}

main_test.go

@@ -137,7 +137,7 @@ func runInternalJsonnet(i jsonnetInput) jsonnetResult {
 	vm.NativeFunction(jsonToString)
 	vm.NativeFunction(nativeError)
 
-	rawAST, err := snippetToRawAST(i.name, string(i.input))
+	rawAST, err := parser.SnippetToRawAST(i.name, string(i.input))
 	if err != nil {
 		return jsonnetResult{
 			output:  errFormatter.Format(err) + "\n",
@@ -146,7 +146,7 @@ func runInternalJsonnet(i jsonnetInput) jsonnetResult {
 	}
 	testChildren(rawAST)
 
-	desugaredAST, err := snippetToAST(i.name, string(i.input))
+	desugaredAST, err := SnippetToAST(i.name, string(i.input))
 	if err != nil {
 		return jsonnetResult{
 			output:  errFormatter.Format(err) + "\n",

parser/parser.go

@@ -1191,3 +1191,12 @@ func Parse(t Tokens) (ast.Node, error) {
 
 	return expr, nil
 }
+
+// SnippetToRawAST converts Jsonnet code snippet to AST (without any transformations)
+func SnippetToRawAST(filename string, snippet string) (ast.Node, error) {
+	tokens, err := Lex(filename, snippet)
+	if err != nil {
+		return nil, err
+	}
+	return Parse(tokens)
+}

value.go

@@ -428,8 +428,8 @@ func (*valueObject) getType() *valueType {
 	return objectType
 }
 
-func (o *valueObject) index(i *interpreter, trace TraceElement, field string) (value, error) {
+func (obj *valueObject) index(i *interpreter, trace TraceElement, field string) (value, error) {
-	return objectIndex(i, trace, objectBinding(o), field)
+	return objectIndex(i, trace, objectBinding(obj), field)
 }
 
 func (obj *valueObject) assertionsChecked() bool {

vm.go

@@ -22,7 +22,7 @@ import (
 	"runtime/debug"
 
 	"github.com/google/go-jsonnet/ast"
-	"github.com/google/go-jsonnet/parser"
+	"github.com/google/go-jsonnet/internal/transformations"
 )
 
 // Note: There are no garbage collection params because we're using the native
@@ -137,7 +137,7 @@ func (vm *VM) evaluateSnippet(filename string, snippet string, kind evalKind) (o
 			err = fmt.Errorf("(CRASH) %v\n%s", r, debug.Stack())
 		}
 	}()
-	node, err := snippetToAST(filename, snippet)
+	node, err := SnippetToAST(filename, snippet)
 	if err != nil {
 		return "", err
 	}
@@ -199,33 +199,9 @@ func (vm *VM) EvaluateSnippetMulti(filename string, snippet string) (files map[s
 	return
 }
 
-func snippetToRawAST(filename string, snippet string) (ast.Node, error) {
-	tokens, err := parser.Lex(filename, snippet)
-	if err != nil {
-		return nil, err
-	}
-	return parser.Parse(tokens)
-}
-
-func snippetToAST(filename string, snippet string) (ast.Node, error) {
-	node, err := snippetToRawAST(filename, snippet)
-	if err != nil {
-		return nil, err
-	}
-	err = desugarFile(&node)
-	if err != nil {
-		return nil, err
-	}
-	err = analyze(node)
-	if err != nil {
-		return nil, err
-	}
-	return node, nil
-}
-
 // SnippetToAST parses a snippet and returns the resulting AST.
 func SnippetToAST(filename string, snippet string) (ast.Node, error) {
-	return snippetToAST(filename, snippet)
+	return transformations.SnippetToAST(filename, snippet)
 }
 
 // Version returns the Jsonnet version number.