jsonnet-deps: Jsonnet static dependency parser

Fixes #833

BUILD.bazel

@@ -28,6 +28,7 @@ go_library(
         "//ast:go_default_library",
         "//astgen:go_default_library",
         "//internal/errors:go_default_library",
+        "//internal/parser:go_default_library",
         "//internal/program:go_default_library",
     ],
 )

README.md

@@ -32,6 +32,7 @@ git clone git@github.com:google/go-jsonnet.git
 cd go-jsonnet
 go build ./cmd/jsonnet
 go build ./cmd/jsonnetfmt
+go build ./cmd/jsonnet-deps
 ```
 To build with [Bazel](https://bazel.build/) instead:
 ```bash
@@ -41,6 +42,7 @@ git submodule init
 git submodule update
 bazel build //cmd/jsonnet
 bazel build //cmd/jsonnetfmt
+bazel build //cmd/jsonnet-deps
 ```
 The resulting _jsonnet_ program will then be available at a platform-specific path, such as _bazel-bin/cmd/jsonnet/darwin_amd64_stripped/jsonnet_ for macOS.
 

cmd/jsonnet-deps/BUILD.bazel

@@ -0,0 +1,19 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["cmd.go"],
+    importpath = "github.com/google/go-jsonnet/cmd/jsonnet-deps",
+    visibility = ["//visibility:private"],
+    deps = [
+        "//:go_default_library",
+        "//cmd/internal/cmd:go_default_library",
+        "@com_github_fatih_color//:go_default_library",
+    ],
+)
+
+go_binary(
+    name = "jsonnet-deps",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

cmd/jsonnet-deps/cmd.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2020 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/fatih/color"
+
+	"github.com/google/go-jsonnet"
+	"github.com/google/go-jsonnet/cmd/internal/cmd"
+)
+
+func version(o io.Writer) {
+	fmt.Fprintf(o, "Jsonnet static dependency parser %s\n", jsonnet.Version())
+}
+
+func usage(o io.Writer) {
+	version(o)
+	fmt.Fprintln(o)
+	fmt.Fprintln(o, "jsonnet-deps {<option>} <filename>...")
+	fmt.Fprintln(o)
+	fmt.Fprintln(o, "Available options:")
+	fmt.Fprintln(o, "  -h / --help                This message")
+	fmt.Fprintln(o, "  -J / --jpath <dir>         Specify an additional library search dir")
+	fmt.Fprintln(o, "                             (right-most wins)")
+	fmt.Fprintln(o, "  -o / --output-file <file>  Write to the output file rather than stdout")
+	fmt.Fprintln(o, "  --version                  Print version")
+	fmt.Fprintln(o)
+	fmt.Fprintln(o, "Environment variables:")
+	fmt.Fprintln(o, "  JSONNET_PATH is a colon (semicolon on Windows) separated list of directories")
+	fmt.Fprintln(o, "  added in reverse order before the paths specified by --jpath (i.e. left-most")
+	fmt.Fprintln(o, "  wins). E.g. these are equivalent:")
+	fmt.Fprintln(o, "    JSONNET_PATH=a:b jsonnet -J c -J d")
+	fmt.Fprintln(o, "    JSONNET_PATH=d:c:a:b jsonnet")
+	fmt.Fprintln(o, "    jsonnet -J b -J a -J c -J d")
+	fmt.Fprintln(o)
+	fmt.Fprintln(o, "In all cases:")
+	fmt.Fprintln(o, "  Multichar options are expanded e.g. -abc becomes -a -b -c.")
+	fmt.Fprintln(o, "  The -- option suppresses option processing for subsequent arguments.")
+	fmt.Fprintln(o, "  Note that since filenames and jsonnet programs can begin with -, it is")
+	fmt.Fprintln(o, "  advised to use -- if the argument is unknown, e.g. jsonnet-deps -- \"$FILENAME\".")
+}
+
+type config struct {
+	inputFiles []string
+	outputFile string
+	jPaths     []string
+}
+
+type processArgsStatus int
+
+const (
+	processArgsStatusContinue     = iota
+	processArgsStatusSuccessUsage = iota
+	processArgsStatusFailureUsage = iota
+	processArgsStatusSuccess      = iota
+	processArgsStatusFailure      = iota
+)
+
+func processArgs(givenArgs []string, conf *config, vm *jsonnet.VM) (processArgsStatus, error) {
+	args := cmd.SimplifyArgs(givenArgs)
+	remainingArgs := make([]string, 0, len(args))
+
+	for i := 0; i < len(args); i++ {
+		arg := args[i]
+		if arg == "-h" || arg == "--help" {
+			return processArgsStatusSuccessUsage, nil
+		} else if arg == "-v" || arg == "--version" {
+			version(os.Stdout)
+			return processArgsStatusSuccess, nil
+		} else if arg == "-o" || arg == "--output-file" {
+			outputFile := cmd.NextArg(&i, args)
+			if len(outputFile) == 0 {
+				return processArgsStatusFailure, fmt.Errorf("-o argument was empty string")
+			}
+			conf.outputFile = outputFile
+		} else if arg == "-J" || arg == "--jpath" {
+			dir := cmd.NextArg(&i, args)
+			if len(dir) == 0 {
+				return processArgsStatusFailure, fmt.Errorf("-J argument was empty string")
+			}
+			conf.jPaths = append(conf.jPaths, dir)
+		} else if arg == "--" {
+			// All subsequent args are not options.
+			i++
+			for ; i < len(args); i++ {
+				remainingArgs = append(remainingArgs, args[i])
+			}
+			break
+		} else if len(arg) > 1 && arg[0] == '-' {
+			return processArgsStatusFailure, fmt.Errorf("unrecognized argument: %s", arg)
+		} else {
+			remainingArgs = append(remainingArgs, arg)
+		}
+	}
+
+	if len(remainingArgs) == 0 {
+		return processArgsStatusFailureUsage, fmt.Errorf("must give filename")
+	}
+	conf.inputFiles = remainingArgs
+
+	return processArgsStatusContinue, nil
+}
+
+func writeDependencies(dependencies []string, outputFile string) (err error) {
+	var f *os.File
+
+	if outputFile == "" {
+		f = os.Stdout
+	} else {
+		f, err = os.Create(outputFile)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if ferr := f.Close(); ferr != nil {
+				err = ferr
+			}
+		}()
+	}
+
+	if len(dependencies) != 0 {
+		output := strings.Join(dependencies, "\n") + "\n"
+		_, err = f.WriteString(output)
+		if err != nil {
+			return err
+		}
+	}
+
+	return
+}
+
+func main() {
+	cmd.StartCPUProfile()
+	defer cmd.StopCPUProfile()
+
+	vm := jsonnet.MakeVM()
+	vm.ErrorFormatter.SetColorFormatter(color.New(color.FgRed).Fprintf)
+
+	conf := config{}
+	jsonnetPath := filepath.SplitList(os.Getenv("JSONNET_PATH"))
+	for i := len(jsonnetPath) - 1; i >= 0; i-- {
+		conf.jPaths = append(conf.jPaths, jsonnetPath[i])
+	}
+
+	status, err := processArgs(os.Args[1:], &conf, vm)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "ERROR: "+err.Error())
+	}
+	switch status {
+	case processArgsStatusContinue:
+		break
+	case processArgsStatusSuccessUsage:
+		usage(os.Stdout)
+		os.Exit(0)
+	case processArgsStatusFailureUsage:
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "")
+		}
+		usage(os.Stderr)
+		os.Exit(1)
+	case processArgsStatusSuccess:
+		os.Exit(0)
+	case processArgsStatusFailure:
+		os.Exit(1)
+	}
+
+	vm.Importer(&jsonnet.FileImporter{JPaths: conf.jPaths})
+
+	for _, file := range conf.inputFiles {
+		if _, err := os.Stat(file); err != nil {
+			fmt.Fprintln(os.Stderr, err.Error())
+			os.Exit(1)
+		}
+	}
+
+	dependencies, err := vm.FindDependencies("", conf.inputFiles)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+	cmd.MemProfile()
+
+	err = writeDependencies(dependencies, conf.outputFile)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}

cmd/jsonnet/cmd.go

@@ -23,7 +23,6 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"strconv"
 	"strings"
 
 	"github.com/fatih/color"
@@ -93,15 +92,6 @@ func usage(o io.Writer) {
 	fmt.Fprintln(o, "  advised to use -- if the argument is unknown, e.g. jsonnet -- \"$FILENAME\".")
 }
 
-func safeStrToInt(str string) (i int) {
-	i, err := strconv.Atoi(str)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Invalid integer \"%s\"\n", str)
-		os.Exit(1)
-	}
-	return
-}
-
 type config struct {
 	inputFiles     []string
 	outputFile     string
@@ -203,7 +193,7 @@ func processArgs(givenArgs []string, config *config, vm *jsonnet.VM) (processArg
 			}
 			break
 		} else if arg == "-s" || arg == "--max-stack" {
-			l := safeStrToInt(cmd.NextArg(&i, args))
+			l := cmd.SafeStrToInt(cmd.NextArg(&i, args))
 			if l < 1 {
 				return processArgsStatusFailure, fmt.Errorf("invalid --max-stack value: %d", l)
 			}
@@ -213,9 +203,6 @@ func processArgs(givenArgs []string, config *config, vm *jsonnet.VM) (processArg
 			if len(dir) == 0 {
 				return processArgsStatusFailure, fmt.Errorf("-J argument was empty string")
 			}
-			if dir[len(dir)-1] != '/' {
-				dir += "/"
-			}
 			config.evalJpath = append(config.evalJpath, dir)
 		} else if arg == "-V" || arg == "--ext-str" {
 			if err := handleVarVal(vm.ExtVar); err != nil {
@@ -250,7 +237,7 @@ func processArgs(givenArgs []string, config *config, vm *jsonnet.VM) (processArg
 				return processArgsStatusFailure, err
 			}
 		} else if arg == "-t" || arg == "--max-trace" {
-			l := safeStrToInt(cmd.NextArg(&i, args))
+			l := cmd.SafeStrToInt(cmd.NextArg(&i, args))
 			if l < 0 {
 				return processArgsStatusFailure, fmt.Errorf("invalid --max-trace value: %d", l)
 			}
@@ -314,7 +301,7 @@ func writeMultiOutputFiles(output map[string]string, outputDir, outputFile strin
 			return err
 		}
 		defer func() {
-			if ferr := manifest.Close(); err != nil {
+			if ferr := manifest.Close(); ferr != nil {
 				err = ferr
 			}
 		}()
@@ -363,7 +350,7 @@ func writeMultiOutputFiles(output map[string]string, outputDir, outputFile strin
 			return err
 		}
 		defer func() {
-			if ferr := f.Close(); err != nil {
+			if ferr := f.Close(); ferr != nil {
 				err = ferr
 			}
 		}()
@@ -389,7 +376,7 @@ func writeOutputStream(output []string, outputFile string) (err error) {
 			return err
 		}
 		defer func() {
-			if ferr := f.Close(); err != nil {
+			if ferr := f.Close(); ferr != nil {
 				err = ferr
 			}
 		}()

vm.go

@@ -19,9 +19,14 @@ package jsonnet
 import (
 	"errors"
 	"fmt"
+	"os"
+	"path/filepath"
 	"runtime/debug"
+	"sort"
+	"strings"
 
 	"github.com/google/go-jsonnet/ast"
+	"github.com/google/go-jsonnet/internal/parser"
 	"github.com/google/go-jsonnet/internal/program"
 )
 
@@ -186,6 +191,77 @@ func (vm *VM) evaluateSnippet(filename string, snippet string, kind evalKind) (o
 	return output, nil
 }
 
+func getAbsPath(path string) (string, error) {
+	var absPath string
+	if filepath.IsAbs(path) {
+		absPath = path
+	} else {
+		wd, err := os.Getwd()
+		if err != nil {
+			return "", nil
+		}
+		absPath = strings.Join([]string{wd, path}, string(filepath.Separator))
+	}
+	cleanedAbsPath, err := filepath.EvalSymlinks(absPath)
+	if err != nil {
+		return "", err
+	}
+	return cleanedAbsPath, nil
+}
+
+func (vm *VM) findDependencies(filePath string, node *ast.Node, dependencies map[string]struct{}, stackTrace *[]traceFrame) (err error) {
+	var cleanedAbsPath string
+	switch i := (*node).(type) {
+	case *ast.Import:
+		node, foundAt, err := vm.ImportAST(filePath, i.File.Value)
+		if err != nil {
+			*stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...)
+			return err
+		}
+		cleanedAbsPath = foundAt
+		if _, isFileImporter := vm.importer.(*FileImporter); isFileImporter {
+			cleanedAbsPath, err = getAbsPath(foundAt)
+			if err != nil {
+				*stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...)
+				return err
+			}
+		}
+		// Check that we haven't already parsed the imported file.
+		if _, alreadyParsed := dependencies[cleanedAbsPath]; alreadyParsed {
+			return nil
+		}
+		dependencies[cleanedAbsPath] = struct{}{}
+		err = vm.findDependencies(foundAt, &node, dependencies, stackTrace)
+		if err != nil {
+			*stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...)
+			return err
+		}
+	case *ast.ImportStr:
+		foundAt, err := vm.ResolveImport(filePath, i.File.Value)
+		if err != nil {
+			*stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...)
+			return err
+		}
+		cleanedAbsPath = foundAt
+		if _, isFileImporter := vm.importer.(*FileImporter); isFileImporter {
+			cleanedAbsPath, err = getAbsPath(foundAt)
+			if err != nil {
+				*stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...)
+				return err
+			}
+		}
+		dependencies[cleanedAbsPath] = struct{}{}
+	default:
+		for _, node := range parser.Children(i) {
+			err = vm.findDependencies(filePath, &node, dependencies, stackTrace)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return
+}
+
 // EvaluateSnippet evaluates a string containing Jsonnet code, return a JSON
 // string.
 //
@@ -225,6 +301,60 @@ func (vm *VM) EvaluateSnippetMulti(filename string, snippet string) (files map[s
 	return
 }
 
+// FindDependencies returns a sorted array of unique transitive dependencies (via import or importstr)
+// from all the given `importedPaths` which are themselves excluded from the returned array.
+// The `importedPaths` are parsed as if they were imported from a Jsonnet file located at `importedFrom`.
+func (vm *VM) FindDependencies(importedFrom string, importedPaths []string) ([]string, error) {
+	var nodes []*ast.Node
+	var stackTrace []traceFrame
+	filePaths := make([]string, len(importedPaths))
+	depsToExclude := make([]string, len(importedPaths))
+	deps := make(map[string]struct{})
+
+	for i, filePath := range importedPaths {
+		node, foundAt, err := vm.ImportAST(importedFrom, filePath)
+		if err != nil {
+			return nil, err
+		}
+		cleanedAbsPath := foundAt
+		if _, isFileImporter := vm.importer.(*FileImporter); isFileImporter {
+			cleanedAbsPath, err = getAbsPath(foundAt)
+			if err != nil {
+				return nil, err
+			}
+		}
+		nodes = append(nodes, &node)
+		filePaths[i] = foundAt
+
+		// Add `importedPaths` to the dependencies so that they are not parsed again.
+		// Will be removed before returning.
+		deps[cleanedAbsPath] = struct{}{}
+		depsToExclude[i] = cleanedAbsPath
+	}
+
+	for i, filePath := range filePaths {
+		err := vm.findDependencies(filePath, nodes[i], deps, &stackTrace)
+		if err != nil {
+			err = makeRuntimeError(err.Error(), stackTrace)
+			return nil, errors.New(vm.ErrorFormatter.Format(err))
+		}
+	}
+
+	// Exclude `importedPaths` from the dependencies.
+	for _, dep := range depsToExclude {
+		delete(deps, dep)
+	}
+
+	dependencies, i := make([]string, len(deps)), 0
+	for key := range deps {
+		dependencies[i] = key
+		i++
+	}
+	sort.Strings(dependencies)
+
+	return dependencies, nil
+}
+
 // ResolveImport finds the actual path where the imported file can be found.
 // It will cache the contents of the file immediately as well, to avoid the possibility of the file
 // disappearing after being checked.