From 7b6ac5f2d7aeae0109d2df7008c55176cac06b39 Mon Sep 17 00:00:00 2001 From: Joe Beda Date: Tue, 16 Feb 2016 10:46:31 -0800 Subject: [PATCH] Check in work in progress on parser. Doesn't build right now. --- ast.go | 5 +- lexer.go | 73 +++++++++++++++++-- parser.go | 207 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 278 insertions(+), 7 deletions(-) create mode 100644 parser.go diff --git a/ast.go b/ast.go index e9facb9..cd4f9b6 100644 --- a/ast.go +++ b/ast.go @@ -235,7 +235,10 @@ type astDollar struct{ astNodeBase } // --------------------------------------------------------------------------- // astError represents the error e. -type astError struct{ astNodeBase } +type astError struct { + astNodeBase + expr astNode +} // --------------------------------------------------------------------------- diff --git a/lexer.go b/lexer.go index f932ba4..3804c07 100644 --- a/lexer.go +++ b/lexer.go @@ -35,10 +35,8 @@ type fodder []fodderElement type tokenKind int const ( - tokenInvalid tokenKind = iota - // Symbols - tokenBraceL + tokenBraceL tokenKind = iota tokenBraceR tokenBracketL tokenBracketR @@ -54,9 +52,9 @@ const ( tokenIdentifier tokenNumber tokenOperator + tokenStringBlock tokenStringDouble tokenStringSingle - tokenStringBlock // Keywords tokenAssert @@ -71,10 +69,10 @@ const ( tokenIn tokenLocal tokenNullLit - tokenTailStrict - tokenThen tokenSelf tokenSuper + tokenTailStrict + tokenThen tokenTrue // A special token that holds line/column information about the end of the @@ -82,6 +80,59 @@ const ( tokenEndOfFile ) +var tokenKindStrings = []string{ + // Symbols + tokenBraceL: "\"{\"", + tokenBraceR: "\"}\"", + tokenBracketL: "\"[\"", + tokenBracketR: "\"]\"", + tokenColon: "\":\"", + tokenComma: "\",\"", + tokenDollar: "\"$\"", + tokenDot: "\".\"", + tokenParenL: "\"(\"", + tokenParenR: "\")\"", + tokenSemicolon: "\";\"", + + // Arbitrary length lexemes + tokenIdentifier: "IDENTIFIER", + tokenNumber: "NUMBER", + tokenOperator: "OPERATOR", + tokenStringBlock: "STRING_BLOCK", + tokenStringDouble: "STRING_DOUBLE", + tokenStringSingle: "STRING_SINGLE", + + // Keywords + tokenAssert: "assert", + tokenElse: "else", + tokenError: "error", + tokenFalse: "false", + tokenFor: "for", + tokenFunction: "function", + tokenIf: "if", + tokenImport: "import", + tokenImportStr: "importstr", + tokenIn: "in", + tokenLocal: "local", + tokenNullLit: "null", + tokenSelf: "self", + tokenSuper: "super", + tokenTailStrict: "tailstrict", + tokenThen: "then", + tokenTrue: "true", + + // A special token that holds line/column information about the end of the + // file. + tokenEndOfFile: "end of file", +} + +func (tk tokenKind) String() string { + if tk < 0 || int(tk) >= len(tokenKindStrings) { + panic(fmt.Sprintf("INTERNAL ERROR: Unknown token kind:: %v", tk)) + } + return tokenKindStrings[tk] +} + type token struct { kind tokenKind // The type of the token fodder fodder // Any fodder the occurs before this token @@ -96,6 +147,16 @@ type token struct { type tokens []token +func (t *token) String() string { + if t.data == "" { + return t.kind.String() + } else if t.kind == tokenOperator { + return fmt.Sprintf("\"%v\"", t.data) + } else { + return fmt.Sprintf("(%v, \"%v\")", t.kind, t.data) + } +} + // --------------------------------------------------------------------------- // Helpers diff --git a/parser.go b/parser.go new file mode 100644 index 0000000..07e5815 --- /dev/null +++ b/parser.go @@ -0,0 +1,207 @@ +package jsonnet + +import ( + "fmt" +) + +type precedence int + +const ( + applyPrecedence precedence = 2 // Function calls and indexing. + unaryPrecedence precedence = 4 // Logical and bitwise negation, unary + - + beforeElsePrecedence precedence = 15 // True branch of an if. + maxPrecedence precedence = 16 // Local, If, Import, Function, Error +) + +// --------------------------------------------------------------------------- + +func makeUnexpectedError(t token, while string) error { + return makeStaticError( + fmt.Sprintf("Unexpected: %v while %v", t, while), t.loc) +} + +func locFromTokens(begin, end *token) LocationRange { + return makeLocationRange(begin.loc.FileName, begin.loc.Begin, end.loc.End) +} + +func locFromTokenAST(begin *token, end astNode) LocationRange { + return makeLocationRange(begin.loc.FileName, begin.loc.Begin, end.Loc().End) +} + +// --------------------------------------------------------------------------- + +type parser struct { + t tokens + currT int +} + +func makeParser(t tokens) *parser { + return &parser{ + t: t, + } +} + +func (p *parser) pop() *token { + t := &p.t[p.currT] + p.currT++ + return t +} + +func (p *parser) popExpect(tk tokenKind) (*token, error) { + t := p.pop() + if t.kind != tk { + return nil, makeStaticError( + fmt.Sprintf("Expected token %v but got %v", tk, t), t.loc) + } + return t, nil +} + +func (p *parser) popExpectOp(op string) (*token, error) { + t := p.pop() + if t.kind != tokenOperator || t.data != op { + return nil, makeStaticError( + fmt.Sprintf("Expected operator %v but got %v", op, t), t.loc) + } + return t, nil +} + +func (p *parser) peek() *token { + return &p.t[p.currT] +} + +func (p *parser) parseIdentifierList(elementKind string) (identifiers, bool, error) { + exprs, got_comma, err := p.parseCommaList(tokenParenR, elementKind) + if err != nil { + return identifiers{}, false, err + } + var ids identifiers + for n := range exprs { + v, ok := n.(astVar) + if !ok { + return identifiers{}, false, makeStaticError(fmt.Sprintf("Not an identifier: %v", n), n.Loc()) + } + ids = append(ids, v.id) + } +} + +func (p *parser) parseCommaList(tokenKind end, elementkind string) (astNodes, bool, error) { + +} + +func (p *parser) parse(prec precedence) (astNode, error) { + begin := p.peek() + + switch begin.kind { + // These cases have effectively maxPrecedence as the first + // call to parse will parse them. + case tokenAssert: + p.pop() + cond, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + var msg astNode + if p.peek().kind == tokenColon { + p.pop() + msg, err = p.parse(maxPrecedence) + if err != nil { + return nil, err + } + } + _, err = p.popExpect(tokenSemicolon) + if err != nil { + return nil, err + } + rest, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + return &astAssert{ + astNodeBase: astNodeBase{locFromTokenAST(begin, rest)}, + cond: cond, + message: msg, + rest: rest, + }, nil + + case tokenError: + p.pop() + expr, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + return &astError{ + astNodeBase: astNodeBase{locFromTokenAST(begin, expr)}, + expr: expr, + }, nil + + case tokenIf: + p.pop() + cond, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + _, err = p.popExpect(tokenThen) + if err != nil { + return nil, err + } + branchTrue, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + var branchFalse astNode + lr := locFromTokenAST(begin, branchTrue) + if p.peek().kind == tokenElse { + p.pop() + branchFalse, err = p.parse(maxPrecedence) + if err != nil { + return nil, err + } + lr = locFromTokenAST(begin, branchFalse) + } + return &astConditional{ + astNodeBase: astNodeBase{lr}, + cond: cond, + branchTrue: branchTrue, + branchFalse: branchFalse, + }, nil + case tokenFunction: + p.pop() + next := p.pop() + if next.kind == tokenParenL { + params, got_comma, err := p.parseIdentifierList("function parameter") + if err != nil { + return nil, err + } + body, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + return astFunction{ + astNodeBase: astNodeBase{locFromTokenAST(begin, body)}, + parameters: params, + trailingComma, got_comma, + body: body, + } + } else { + return makeStaticError(fmt.Sprintf("Expected ( but got %v", next), next.loc) + } + } + + return nil, nil +} + +// --------------------------------------------------------------------------- + +func parse(t tokens) (astNode, error) { + p := makeParser(t) + expr, err := p.parse(maxPrecedence) + if err != nil { + return nil, err + } + + if p.peek().kind != tokenEndOfFile { + return nil, makeStaticError(fmt.Sprintf("Did not expect: %v", p.peek()), p.peek().loc) + } + + return expr, nil +}