From 856bd58872418eee1cede0badea5b7b462c429eb Mon Sep 17 00:00:00 2001 From: Angus Lees Date: Sun, 23 Jan 2022 11:04:40 +1100 Subject: [PATCH] Add 'importbin' statement Add `importbin` statement. Similar to `importstr` but the result is an array of numbers (all integers 0-255). --- ast/ast.go | 8 ++++ ast/clone.go | 7 ++++ imports.go | 37 ++++++++++++++++-- internal/formatter/fix_indentation.go | 5 +++ internal/formatter/unparser.go | 4 ++ internal/parser/context.go | 12 ++---- internal/parser/lexer.go | 4 ++ internal/parser/lexer_test.go | 6 +++ internal/parser/parser.go | 19 ++++++++- internal/parser/parser_test.go | 3 ++ internal/pass/pass.go | 9 +++++ internal/program/desugarer.go | 8 ++++ internal/program/static_analyzer.go | 4 +- interpreter.go | 4 ++ jsonnet_test.go | 11 +++--- linter/internal/types/build_graph.go | 2 + linter/linter.go | 6 +++ testdata/importbin_block_literal.golden | 7 ++++ testdata/importbin_block_literal.jsonnet | 3 ++ .../importbin_block_literal.linter.golden | 7 ++++ testdata/importbin_computed.golden | 5 +++ testdata/importbin_computed.jsonnet | 1 + testdata/importbin_computed.linter.golden | 5 +++ testdata/importbin_nonutf8.golden | 5 +++ testdata/importbin_nonutf8.jsonnet | 1 + testdata/importbin_nonutf8.linter.golden | 0 testdata/nonutf8.bin | Bin 0 -> 3 bytes vm.go | 17 +++++++- 28 files changed, 177 insertions(+), 23 deletions(-) create mode 100644 testdata/importbin_block_literal.golden create mode 100644 testdata/importbin_block_literal.jsonnet create mode 100644 testdata/importbin_block_literal.linter.golden create mode 100644 testdata/importbin_computed.golden create mode 100644 testdata/importbin_computed.jsonnet create mode 100644 testdata/importbin_computed.linter.golden create mode 100644 testdata/importbin_nonutf8.golden create mode 100644 testdata/importbin_nonutf8.jsonnet create mode 100644 testdata/importbin_nonutf8.linter.golden create mode 100644 testdata/nonutf8.bin diff --git a/ast/ast.go b/ast/ast.go index 087e52f..3269f44 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -423,6 +423,14 @@ type ImportStr struct { // --------------------------------------------------------------------------- +// ImportBin represents importbin "file". +type ImportBin struct { + NodeBase + File *LiteralString +} + +// --------------------------------------------------------------------------- + // Index represents both e[e] and the syntax sugar e.f. // // One of index and id will be nil before desugaring. After desugaring id diff --git a/ast/clone.go b/ast/clone.go index ce877b0..3424986 100644 --- a/ast/clone.go +++ b/ast/clone.go @@ -169,6 +169,13 @@ func clone(astPtr *Node) { r.File = new(LiteralString) *r.File = *node.File + case *ImportBin: + r := new(ImportBin) + *astPtr = r + *r = *node + r.File = new(LiteralString) + *r.File = *node.File + case *Index: r := new(Index) *astPtr = r diff --git a/imports.go b/imports.go index cf0e21e..97f748c 100644 --- a/imports.go +++ b/imports.go @@ -21,6 +21,7 @@ import ( "io/ioutil" "os" "path" + "unsafe" "github.com/google/go-jsonnet/ast" "github.com/google/go-jsonnet/internal/program" @@ -58,19 +59,33 @@ type Importer interface { } // Contents is a representation of imported data. It is a simple -// string wrapper, which makes it easier to enforce the caching policy. +// byte wrapper, which makes it easier to enforce the caching policy. type Contents struct { - data *string + data *[]byte } func (c Contents) String() string { + // Construct string without copying underlying bytes. + // NB: This only works because c.data is not modified. + return *(*string)(unsafe.Pointer(c.data)) +} + +func (c Contents) Data() []byte { return *c.data } // MakeContents creates Contents from a string. func MakeContents(s string) Contents { + data := []byte(s) return Contents{ - data: &s, + data: &data, + } +} + +// MakeContentsRaw creates Contents from (possibly non-utf8) []byte data. +func MakeContentsRaw(bytes []byte) Contents { + return Contents{ + data: &bytes, } } @@ -139,6 +154,20 @@ func (cache *importCache) importString(importedFrom, importedPath string, i *int return makeValueString(data.String()), nil } +// ImportString imports an array of bytes, caches it and then returns it. +func (cache *importCache) importBinary(importedFrom, importedPath string, i *interpreter) (*valueArray, error) { + data, _, err := cache.importData(importedFrom, importedPath) + if err != nil { + return nil, i.Error(err.Error()) + } + bytes := data.Data() + elements := make([]*cachedThunk, len(bytes)) + for i := range bytes { + elements[i] = readyThunk(intToValue(int(bytes[i]))) + } + return makeValueArray(elements), nil +} + func nodeToPV(i *interpreter, filename string, node ast.Node) *cachedThunk { env := makeInitialEnv(filename, i.baseStd) return &cachedThunk{ @@ -223,7 +252,7 @@ func (importer *FileImporter) tryPath(dir, importedPath string) (found bool, con } else { entry = &fsCacheEntry{ exists: true, - contents: MakeContents(string(contentBytes)), + contents: MakeContentsRaw(contentBytes), } } importer.fsCache[absPath] = entry diff --git a/internal/formatter/fix_indentation.go b/internal/formatter/fix_indentation.go index 29a39a9..90229d1 100644 --- a/internal/formatter/fix_indentation.go +++ b/internal/formatter/fix_indentation.go @@ -574,6 +574,11 @@ func (c *FixIndentation) Visit(expr ast.Node, currIndent indent, crowded bool) { newIndent := c.newIndent(*openFodder(node.File), currIndent, c.column+1) c.Visit(node.File, newIndent, true) + case *ast.ImportBin: + c.column += 9 // importbin + newIndent := c.newIndent(*openFodder(node.File), currIndent, c.column+1) + c.Visit(node.File, newIndent, true) + case *ast.InSuper: c.Visit(node.Index, currIndent, crowded) c.fill(node.InFodder, true, true, currIndent.lineUp) diff --git a/internal/formatter/unparser.go b/internal/formatter/unparser.go index 532f87f..d3e92cd 100644 --- a/internal/formatter/unparser.go +++ b/internal/formatter/unparser.go @@ -370,6 +370,10 @@ func (u *unparser) unparse(expr ast.Node, crowded bool) { u.write("importstr") u.unparse(node.File, true) + case *ast.ImportBin: + u.write("importbin") + u.unparse(node.File, true) + case *ast.Index: u.unparse(node.Target, crowded) u.fill(node.LeftBracketFodder, false, false) // Can also be DotFodder diff --git a/internal/parser/context.go b/internal/parser/context.go index 9952825..1cca48b 100644 --- a/internal/parser/context.go +++ b/internal/parser/context.go @@ -64,9 +64,7 @@ func DirectChildren(node ast.Node) []ast.Node { return []ast.Node{node.Expr} case *ast.Function: return nil - case *ast.Import: - return nil - case *ast.ImportStr: + case *ast.Import, *ast.ImportStr, *ast.ImportBin: return nil case *ast.Index: if node.Id != nil { @@ -181,9 +179,7 @@ func thunkChildren(node ast.Node) []ast.Node { return nil case *ast.Function: return nil - case *ast.Import: - return nil - case *ast.ImportStr: + case *ast.Import, *ast.ImportStr, *ast.ImportBin: return nil case *ast.Index: return nil @@ -304,9 +300,7 @@ func specialChildren(node ast.Node) []ast.Node { } } return children - case *ast.Import: - return nil - case *ast.ImportStr: + case *ast.Import, *ast.ImportStr, *ast.ImportBin: return nil case *ast.Index: return nil diff --git a/internal/parser/lexer.go b/internal/parser/lexer.go index 352b3df..facc1b2 100644 --- a/internal/parser/lexer.go +++ b/internal/parser/lexer.go @@ -65,6 +65,7 @@ const ( tokenIf tokenImport tokenImportStr + tokenImportBin tokenIn tokenLocal tokenNullLit @@ -112,6 +113,7 @@ var tokenKindStrings = []string{ tokenIf: "if", tokenImport: "import", tokenImportStr: "importstr", + tokenImportBin: "importbin", tokenIn: "in", tokenLocal: "local", tokenNullLit: "null", @@ -580,6 +582,8 @@ func getTokenKindFromID(str string) tokenKind { return tokenImport case "importstr": return tokenImportStr + case "importbin": + return tokenImportBin case "in": return tokenIn case "local": diff --git a/internal/parser/lexer_test.go b/internal/parser/lexer_test.go index 5b7841f..c54ff0e 100644 --- a/internal/parser/lexer_test.go +++ b/internal/parser/lexer_test.go @@ -424,6 +424,12 @@ func TestImportstr(t *testing.T) { }) } +func TestImportbin(t *testing.T) { + SingleTest(t, "importbin", "", Tokens{ + {kind: tokenImportBin, data: "importbin"}, + }) +} + func TestIn(t *testing.T) { SingleTest(t, "in", "", Tokens{ {kind: tokenIn, data: "in"}, diff --git a/internal/parser/parser.go b/internal/parser/parser.go index c576074..7589d67 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -870,7 +870,7 @@ func (p *parser) parseTerminal() (ast.Node, errors.StaticError) { tok := p.pop() switch tok.kind { case tokenAssert, tokenBraceR, tokenBracketR, tokenComma, tokenDot, tokenElse, - tokenError, tokenFor, tokenFunction, tokenIf, tokenIn, tokenImport, tokenImportStr, + tokenError, tokenFor, tokenFunction, tokenIf, tokenIn, tokenImport, tokenImportStr, tokenImportBin, tokenLocal, tokenOperator, tokenParenR, tokenSemicolon, tokenTailStrict, tokenThen: return nil, makeUnexpectedError(tok, "parsing terminal") @@ -1122,6 +1122,23 @@ func (p *parser) parse(prec precedence) (ast.Node, errors.StaticError) { } return nil, errors.MakeStaticError("Computed imports are not allowed", *body.Loc()) + case tokenImportBin: + p.pop() + body, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + if lit, ok := body.(*ast.LiteralString); ok { + if lit.Kind == ast.StringBlock { + return nil, errors.MakeStaticError("Block string literals not allowed in imports", *body.Loc()) + } + return &ast.ImportBin{ + NodeBase: ast.NewNodeBaseLoc(locFromTokenAST(begin, body), begin.fodder), + File: lit, + }, nil + } + return nil, errors.MakeStaticError("Computed imports are not allowed", *body.Loc()) + case tokenLocal: p.pop() var binds ast.LocalBinds diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go index 558fbbf..c58d035 100644 --- a/internal/parser/parser_test.go +++ b/internal/parser/parser_test.go @@ -98,6 +98,7 @@ var tests = []string{ `import 'foo.jsonnet'`, `importstr 'foo.text'`, + `importbin 'foo.bin'`, `{a: b} + {c: d}`, `{a: b}{c: d}`, @@ -230,6 +231,8 @@ var errorTests = []testError{ {`import (a+b)`, `test:1:8-13 Computed imports are not allowed`}, {`importstr (a b)`, `test:1:14-15 Expected token ")" but got (IDENTIFIER, "b")`}, {`importstr (a+b)`, `test:1:11-16 Computed imports are not allowed`}, + {`importbin (a b)`, `test:1:14-15 Expected token ")" but got (IDENTIFIER, "b")`}, + {`importbin (a+b)`, `test:1:11-16 Computed imports are not allowed`}, {`local a = b ()`, `test:1:15 Expected , or ; but got end of file`}, {`local a = b; (a b)`, `test:1:17-18 Expected token ")" but got (IDENTIFIER, "b")`}, diff --git a/internal/pass/pass.go b/internal/pass/pass.go index b3475fe..5d76c68 100644 --- a/internal/pass/pass.go +++ b/internal/pass/pass.go @@ -48,6 +48,7 @@ type ASTPass interface { Function(ASTPass, *ast.Function, Context) Import(ASTPass, *ast.Import, Context) ImportStr(ASTPass, *ast.ImportStr, Context) + ImportBin(ASTPass, *ast.ImportBin, Context) Index(ASTPass, *ast.Index, Context) Slice(ASTPass, *ast.Slice, Context) Local(ASTPass, *ast.Local, Context) @@ -282,6 +283,12 @@ func (*Base) ImportStr(p ASTPass, node *ast.ImportStr, ctx Context) { p.LiteralString(p, node.File, ctx) } +// ImportBin traverses that kind of node +func (*Base) ImportBin(p ASTPass, node *ast.ImportBin, ctx Context) { + p.Fodder(p, &node.File.Fodder, ctx) + p.LiteralString(p, node.File, ctx) +} + // Index traverses that kind of node func (*Base) Index(p ASTPass, node *ast.Index, ctx Context) { p.Visit(p, &node.Target, ctx) @@ -419,6 +426,8 @@ func (*Base) Visit(p ASTPass, node *ast.Node, ctx Context) { p.Import(p, node, ctx) case *ast.ImportStr: p.ImportStr(p, node, ctx) + case *ast.ImportBin: + p.ImportBin(p, node, ctx) case *ast.Index: p.Index(p, node, ctx) case *ast.InSuper: diff --git a/internal/program/desugarer.go b/internal/program/desugarer.go index 38d959f..d36acbe 100644 --- a/internal/program/desugarer.go +++ b/internal/program/desugarer.go @@ -441,6 +441,14 @@ func desugar(astPtr *ast.Node, objLevel int) (err error) { return } + case *ast.ImportBin: + // See comment in ast.Import. + var file ast.Node = node.File + err = desugar(&file, objLevel) + if err != nil { + return + } + case *ast.Index: err = desugar(&node.Target, objLevel) if err != nil { diff --git a/internal/program/static_analyzer.go b/internal/program/static_analyzer.go index e1a11a3..01c93e0 100644 --- a/internal/program/static_analyzer.go +++ b/internal/program/static_analyzer.go @@ -89,9 +89,7 @@ func analyzeVisit(a ast.Node, inObject bool, vars ast.IdentifierSet) error { for _, param := range a.Parameters { s.freeVars.Remove(param.Name) } - case *ast.Import: - //nothing to do here - case *ast.ImportStr: + case *ast.Import, *ast.ImportStr, *ast.ImportBin: //nothing to do here case *ast.InSuper: if !inObject { diff --git a/interpreter.go b/interpreter.go index d3dac69..ca5bdae 100644 --- a/interpreter.go +++ b/interpreter.go @@ -488,6 +488,10 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) { codePath := node.Loc().FileName return i.importCache.importString(codePath, node.File.Value, i) + case *ast.ImportBin: + codePath := node.Loc().FileName + return i.importCache.importBinary(codePath, node.File.Value, i) + case *ast.LiteralBoolean: return makeValueBoolean(node.Value), nil diff --git a/jsonnet_test.go b/jsonnet_test.go index 2287abe..4a86437 100644 --- a/jsonnet_test.go +++ b/jsonnet_test.go @@ -109,10 +109,11 @@ func TestCustomImporter(t *testing.T) { map[string]Contents{ "a.jsonnet": MakeContents("2 + 2"), "b.jsonnet": MakeContents("3 + 3"), + "c.bin": MakeContentsRaw([]byte{0xff, 0xfe, 0xfd}), }, }) - input := `[import "a.jsonnet", importstr "b.jsonnet"]` - expected := `[ 4, "3 + 3" ]` + input := `[import "a.jsonnet", importstr "b.jsonnet", importbin "c.bin"]` + expected := `[ 4, "3 + 3", [ 255, 254, 253 ] ]` actual, err := vm.EvaluateSnippet("custom_import.jsonnet", input) if err != nil { t.Errorf("Unexpected error: %v", err) @@ -159,7 +160,7 @@ func TestExtVarImportedFrom(t *testing.T) { if actual != expected { t.Errorf("Expected %q, but got %q", expected, actual) } - expectedImportHistory := []importHistoryEntry{importHistoryEntry{"", "a.jsonnet"}} + expectedImportHistory := []importHistoryEntry{{"", "a.jsonnet"}} if !reflect.DeepEqual(importer.history, expectedImportHistory) { t.Errorf("Expected %q, but got %q", expectedImportHistory, importer.history) } @@ -186,7 +187,7 @@ func TestTLAImportedFrom(t *testing.T) { if actual != expected { t.Errorf("Expected %q, but got %q", expected, actual) } - expectedImportHistory := []importHistoryEntry{importHistoryEntry{"", "a.jsonnet"}} + expectedImportHistory := []importHistoryEntry{{"", "a.jsonnet"}} if !reflect.DeepEqual(importer.history, expectedImportHistory) { t.Errorf("Expected %q, but got %q", expectedImportHistory, importer.history) } @@ -212,7 +213,7 @@ func TestAnonymousImportedFrom(t *testing.T) { if actual != expected { t.Errorf("Expected %q, but got %q", expected, actual) } - expectedImportHistory := []importHistoryEntry{importHistoryEntry{"", "a.jsonnet"}} + expectedImportHistory := []importHistoryEntry{{"", "a.jsonnet"}} if !reflect.DeepEqual(importer.history, expectedImportHistory) { t.Errorf("Expected %q, but got %q", expectedImportHistory, importer.history) } diff --git a/linter/internal/types/build_graph.go b/linter/internal/types/build_graph.go index 1e0e35b..81a9afd 100644 --- a/linter/internal/types/build_graph.go +++ b/linter/internal/types/build_graph.go @@ -217,6 +217,8 @@ func calcTP(node ast.Node, varAt map[ast.Node]*common.Variable, g *typeGraph) ty return tpRef(g.getExprPlaceholder(imported)) case *ast.ImportStr: return tpRef(stringType) + case *ast.ImportBin: + return tpRef(anyArrayType) case *ast.LiteralBoolean: return tpRef(boolType) case *ast.LiteralNull: diff --git a/linter/linter.go b/linter/linter.go index 4557448..1437ba1 100644 --- a/linter/linter.go +++ b/linter/linter.go @@ -124,6 +124,12 @@ func getImports(vm *jsonnet.VM, node nodeWithLocation, roots map[string]ast.Node if err != nil { errWriter.writeError(vm, errors.MakeStaticError(err.Error(), *node.Loc())) } + case *ast.ImportBin: + p := node.File.Value + _, err := vm.ResolveImport(currentPath, p) + if err != nil { + errWriter.writeError(vm, errors.MakeStaticError(err.Error(), *node.Loc())) + } default: for _, c := range parser.Children(node) { getImports(vm, nodeWithLocation{c, currentPath}, roots, errWriter) diff --git a/testdata/importbin_block_literal.golden b/testdata/importbin_block_literal.golden new file mode 100644 index 0000000..8a0bff6 --- /dev/null +++ b/testdata/importbin_block_literal.golden @@ -0,0 +1,7 @@ +testdata/importbin_block_literal:(1:11)-(3:4) Block string literals not allowed in imports + +importbin ||| + block_literals_for_imports_are_not_allowed_and_make_exactly_zero_sense +||| + + diff --git a/testdata/importbin_block_literal.jsonnet b/testdata/importbin_block_literal.jsonnet new file mode 100644 index 0000000..e5df406 --- /dev/null +++ b/testdata/importbin_block_literal.jsonnet @@ -0,0 +1,3 @@ +importbin ||| + block_literals_for_imports_are_not_allowed_and_make_exactly_zero_sense +||| diff --git a/testdata/importbin_block_literal.linter.golden b/testdata/importbin_block_literal.linter.golden new file mode 100644 index 0000000..f2e8592 --- /dev/null +++ b/testdata/importbin_block_literal.linter.golden @@ -0,0 +1,7 @@ +../testdata/importbin_block_literal:(1:11)-(3:4) Block string literals not allowed in imports + +importbin ||| + block_literals_for_imports_are_not_allowed_and_make_exactly_zero_sense +||| + + diff --git a/testdata/importbin_computed.golden b/testdata/importbin_computed.golden new file mode 100644 index 0000000..dc77920 --- /dev/null +++ b/testdata/importbin_computed.golden @@ -0,0 +1,5 @@ +testdata/importbin_computed:1:11-20 Computed imports are not allowed + +importbin "a" + "b" + + diff --git a/testdata/importbin_computed.jsonnet b/testdata/importbin_computed.jsonnet new file mode 100644 index 0000000..10cbc38 --- /dev/null +++ b/testdata/importbin_computed.jsonnet @@ -0,0 +1 @@ +importbin "a" + "b" diff --git a/testdata/importbin_computed.linter.golden b/testdata/importbin_computed.linter.golden new file mode 100644 index 0000000..3dbdab7 --- /dev/null +++ b/testdata/importbin_computed.linter.golden @@ -0,0 +1,5 @@ +../testdata/importbin_computed:1:11-20 Computed imports are not allowed + +importbin "a" + "b" + + diff --git a/testdata/importbin_nonutf8.golden b/testdata/importbin_nonutf8.golden new file mode 100644 index 0000000..7992121 --- /dev/null +++ b/testdata/importbin_nonutf8.golden @@ -0,0 +1,5 @@ +[ + 255, + 0, + 254 +] diff --git a/testdata/importbin_nonutf8.jsonnet b/testdata/importbin_nonutf8.jsonnet new file mode 100644 index 0000000..e85d5b4 --- /dev/null +++ b/testdata/importbin_nonutf8.jsonnet @@ -0,0 +1 @@ +importbin "nonutf8.bin" diff --git a/testdata/importbin_nonutf8.linter.golden b/testdata/importbin_nonutf8.linter.golden new file mode 100644 index 0000000..e69de29 diff --git a/testdata/nonutf8.bin b/testdata/nonutf8.bin new file mode 100644 index 0000000000000000000000000000000000000000..90db00e1d6cf116716ba949a1cca330ac8c63634 GIT binary patch literal 3 Kcmey*@DBh3{sH~~ literal 0 HcmV?d00001 diff --git a/vm.go b/vm.go index f4e6437..d1fd467 100644 --- a/vm.go +++ b/vm.go @@ -292,6 +292,21 @@ func (vm *VM) findDependencies(filePath string, node *ast.Node, dependencies map } } dependencies[cleanedAbsPath] = struct{}{} + case *ast.ImportBin: + foundAt, err := vm.ResolveImport(filePath, i.File.Value) + if err != nil { + *stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...) + return err + } + cleanedAbsPath = foundAt + if _, isFileImporter := vm.importer.(*FileImporter); isFileImporter { + cleanedAbsPath, err = getAbsPath(foundAt) + if err != nil { + *stackTrace = append([]traceFrame{{Loc: *i.Loc()}}, *stackTrace...) + return err + } + } + dependencies[cleanedAbsPath] = struct{}{} default: for _, node := range parser.Children(i) { err = vm.findDependencies(filePath, &node, dependencies, stackTrace) @@ -435,7 +450,7 @@ func (vm *VM) EvaluateFileMulti(filename string) (files map[string]string, forma return output, nil } -// FindDependencies returns a sorted array of unique transitive dependencies (via import or importstr) +// FindDependencies returns a sorted array of unique transitive dependencies (via import/importstr/importbin) // from all the given `importedPaths` which are themselves excluded from the returned array. // The `importedPaths` are parsed as if they were imported from a Jsonnet file located at `importedFrom`. func (vm *VM) FindDependencies(importedFrom string, importedPaths []string) ([]string, error) {