mirrors/go-jsonnet
1
0
Fork 0
mirror of https://github.com/google/go-jsonnet.git synced 2025-12-20 16:41:06 +01:00
Code Issues Packages Projects Releases Wiki Activity

Adaptive string representation

Browse Source 
When adding long strings, don't copy them immediately. Instead
build long strings only when their contents are requested.

This allows to build a long string from parts, using a regular
operator+ in linear time. This lets users to worry much less
about using std.join etc.

If indexing the string is mixed with building it using operator+
the behavior can still be quadratic. We may want to address it in
a later change.
This commit is contained in:
Stanisław Barzowski 2019-02-26 11:45:53 +01:00
parent b44e63e102
commit fe82a78401
4 changed files with 177 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
builtins.go
View File

@ -33,12 +33,12 @@ import (
func builtinPlus(i *interpreter, trace traceElement, x, y value) (value, error) { func builtinPlus(i *interpreter, trace traceElement, x, y value) (value, error) {
// TODO(sbarzowski) perhaps a more elegant way to dispatch // TODO(sbarzowski) perhaps a more elegant way to dispatch
switch right := y.(type) { switch right := y.(type) {
case *valueString: case valueString:
left, err := builtinToString(i, trace, x) left, err := builtinToString(i, trace, x)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return concatStrings(left.(*valueString), right), nil return concatStrings(left.(valueString), right), nil
} }
switch left := x.(type) { switch left := x.(type) {
@ -48,12 +48,12 @@ func builtinPlus(i *interpreter, trace traceElement, x, y value) (value, error)
return nil, err return nil, err
} }
return makeDoubleCheck(i, trace, left.value+right.value) return makeDoubleCheck(i, trace, left.value+right.value)
case *valueString: case valueString:
right, err := builtinToString(i, trace, y) right, err := builtinToString(i, trace, y)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return concatStrings(left, right.(*valueString)), nil return concatStrings(left, right.(valueString)), nil
case *valueObject: case *valueObject:
switch right := y.(type) { switch right := y.(type) {
case *valueObject: case *valueObject:
@ -135,7 +135,7 @@ func valueLess(i *interpreter, trace traceElement, x, yv value) (bool, error) {
return false, err return false, err
} }
return left.value < right.value, nil return left.value < right.value, nil
case *valueString: case valueString:
right, err := i.getString(yv, trace) right, err := i.getString(yv, trace)
if err != nil { if err != nil {
return false, err return false, err
@ -178,7 +178,7 @@ func builtinLength(i *interpreter, trace traceElement, x value) (value, error) {
num = len(objectFields(x, withoutHidden)) num = len(objectFields(x, withoutHidden))
case *valueArray: case *valueArray:
num = len(x.elements) num = len(x.elements)
case *valueString: case valueString:
num = x.length() num = x.length()
case *valueFunction: case *valueFunction:
num = len(x.Parameters().required) num = len(x.Parameters().required)
@ -190,7 +190,7 @@ func builtinLength(i *interpreter, trace traceElement, x value) (value, error) {
func builtinToString(i *interpreter, trace traceElement, x value) (value, error) { func builtinToString(i *interpreter, trace traceElement, x value) (value, error) {
switch x := x.(type) { switch x := x.(type) {
case *valueString: case valueString:
return x, nil return x, nil
} }
var buf bytes.Buffer var buf bytes.Buffer
@ -209,7 +209,7 @@ func builtinTrace(i *interpreter, trace traceElement, x value, y value) (value,
filename := trace.loc.FileName filename := trace.loc.FileName
line := trace.loc.Begin.Line line := trace.loc.Begin.Line
fmt.Fprintf( fmt.Fprintf(
os.Stderr, "TRACE: %s:%d %s\n", filename, line, xStr.getString()) os.Stderr, "TRACE: %s:%d %s\n", filename, line, xStr.getGoString())
return y, nil return y, nil
} }
@ -306,7 +306,7 @@ func joinArrays(i *interpreter, trace traceElement, sep *valueArray, arr *valueA
return makeValueArray(result), nil return makeValueArray(result), nil
} }
func joinStrings(i *interpreter, trace traceElement, sep *valueString, arr *valueArray) (value, error) { func joinStrings(i *interpreter, trace traceElement, sep valueString, arr *valueArray) (value, error) {
result := make([]rune, 0, arr.length()) result := make([]rune, 0, arr.length())
first := true first := true
for _, elem := range arr.elements { for _, elem := range arr.elements {
@ -317,17 +317,17 @@ func joinStrings(i *interpreter, trace traceElement, sep *valueString, arr *valu
switch v := elemValue.(type) { switch v := elemValue.(type) {
case *valueNull: case *valueNull:
continue continue
case *valueString: case valueString:
if !first { if !first {
result = append(result, sep.value...) result = append(result, sep.getRunes()...)
} }
result = append(result, v.value...) result = append(result, v.getRunes()...)
default: default:
return nil, i.typeErrorSpecific(elemValue, &valueString{}, trace) return nil, i.typeErrorSpecific(elemValue, emptyString(), trace)
} }
first = false first = false
} }
return &valueString{value: result}, nil return makeStringFromRunes(result), nil
} }
func builtinJoin(i *interpreter, trace traceElement, sep, arrv value) (value, error) { func builtinJoin(i *interpreter, trace traceElement, sep, arrv value) (value, error) {
@ -336,7 +336,7 @@ func builtinJoin(i *interpreter, trace traceElement, sep, arrv value) (value, er
return nil, err return nil, err
} }
switch sep := sep.(type) { switch sep := sep.(type) {
case *valueString: case valueString:
return joinStrings(i, trace, sep, arr) return joinStrings(i, trace, sep, arr)
case *valueArray: case *valueArray:
return joinArrays(i, trace, sep, arr) return joinArrays(i, trace, sep, arr)
@ -524,7 +524,7 @@ func primitiveEquals(i *interpreter, trace traceElement, x, y value) (value, err
return nil, err return nil, err
} }
return makeValueBoolean(left.value == right.value), nil return makeValueBoolean(left.value == right.value), nil
case *valueString: case valueString:
right, err := i.getString(y, trace) right, err := i.getString(y, trace)
if err != nil { if err != nil {
return nil, err return nil, err
@ -559,7 +559,7 @@ func rawEquals(i *interpreter, trace traceElement, x, y value) (bool, error) {
return false, err return false, err
} }
return left.value == right.value, nil return left.value == right.value, nil
case *valueString: case valueString:
right, err := i.getString(y, trace) right, err := i.getString(y, trace)
if err != nil { if err != nil {
return false, err return false, err
@ -660,7 +660,7 @@ func builtinMd5(i *interpreter, trace traceElement, x value) (value, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
hash := md5.Sum([]byte(string(str.value))) hash := md5.Sum([]byte(str.getGoString()))
return makeValueString(hex.EncodeToString(hash[:])), nil return makeValueString(hex.EncodeToString(hash[:])), nil
} }
@ -669,7 +669,7 @@ func builtinEncodeUTF8(i *interpreter, trace traceElement, x value) (value, erro
if err != nil { if err != nil {
return nil, err return nil, err
} }
s := str.getString() s := str.getGoString()
elems := make([]*cachedThunk, 0, len(s)) // it will be longer if characters fall outside of ASCII elems := make([]*cachedThunk, 0, len(s)) // it will be longer if characters fall outside of ASCII
for _, c := range []byte(s) { for _, c := range []byte(s) {
elems = append(elems, readyThunk(makeValueNumber(float64(c)))) elems = append(elems, readyThunk(makeValueNumber(float64(c))))
@ -721,7 +721,7 @@ func builtinCodepoint(i *interpreter, trace traceElement, x value) (value, error
if str.length() != 1 { if str.length() != 1 {
return nil, i.Error(fmt.Sprintf("codepoint takes a string of length 1, got length %v", str.length()), trace) return nil, i.Error(fmt.Sprintf("codepoint takes a string of length 1, got length %v", str.length()), trace)
} }
return makeValueNumber(float64(str.value[0])), nil return makeValueNumber(float64(str.getRunes()[0])), nil
} }
func makeDoubleCheck(i *interpreter, trace traceElement, x float64) (value, error) { func makeDoubleCheck(i *interpreter, trace traceElement, x float64) (value, error) {
@ -823,7 +823,7 @@ func builtinObjectHasEx(i *interpreter, trace traceElement, objv value, fnamev v
return nil, err return nil, err
} }
h := withHiddenFromBool(includeHidden.value) h := withHiddenFromBool(includeHidden.value)
hasField := objectHasField(objectBinding(obj), string(fname.value), h) hasField := objectHasField(objectBinding(obj), string(fname.getRunes()), h)
return makeValueBoolean(hasField), nil return makeValueBoolean(hasField), nil
} }
@ -855,8 +855,8 @@ func builtinSplitLimit(i *interpreter, trace traceElement, strv, cv, maxSplitsV
if maxSplits < -1 { if maxSplits < -1 {
return nil, i.Error(fmt.Sprintf("std.splitLimit third parameter should be -1 or non-negative, got %v", maxSplits), trace) return nil, i.Error(fmt.Sprintf("std.splitLimit third parameter should be -1 or non-negative, got %v", maxSplits), trace)
} }
sStr := str.getString() sStr := str.getGoString()
sC := c.getString() sC := c.getGoString()
if len(sC) != 1 { if len(sC) != 1 {
return nil, i.Error(fmt.Sprintf("std.splitLimit second parameter should have length 1, got %v", len(sC)), trace) return nil, i.Error(fmt.Sprintf("std.splitLimit second parameter should have length 1, got %v", len(sC)), trace)
} }
@ -889,9 +889,9 @@ func builtinStrReplace(i *interpreter, trace traceElement, strv, fromv, tov valu
if err != nil { if err != nil {
return nil, err return nil, err
} }
sStr := str.getString() sStr := str.getGoString()
sFrom := from.getString() sFrom := from.getGoString()
sTo := to.getString() sTo := to.getGoString()
if len(sFrom) == 0 { if len(sFrom) == 0 {
return nil, i.Error("'from' string must not be zero length.", trace) return nil, i.Error("'from' string must not be zero length.", trace)
} }
@ -965,7 +965,7 @@ func builtinParseJSON(i *interpreter, trace traceElement, str value) (value, err
if err != nil { if err != nil {
return nil, err return nil, err
} }
s := sval.getString() s := sval.getGoString()
var parsedJSON interface{} var parsedJSON interface{}
err = json.Unmarshal([]byte(s), &parsedJSON) err = json.Unmarshal([]byte(s), &parsedJSON)
if err != nil { if err != nil {
@ -979,7 +979,7 @@ func builtinExtVar(i *interpreter, trace traceElement, name value) (value, error
if err != nil { if err != nil {
return nil, err return nil, err
} }
index := str.getString() index := str.getGoString()
if pv, ok := i.extVars[index]; ok { if pv, ok := i.extVars[index]; ok {
return i.evaluatePV(pv, trace) return i.evaluatePV(pv, trace)
} }
@ -991,7 +991,7 @@ func builtinNative(i *interpreter, trace traceElement, name value) (value, error
if err != nil { if err != nil {
return nil, err return nil, err
} }
index := str.getString() index := str.getGoString()
if f, exists := i.nativeFuncs[index]; exists { if f, exists := i.nativeFuncs[index]; exists {
return &valueFunction{ec: f}, nil return &valueFunction{ec: f}, nil
} }

2
imports.go
View File

@ -124,7 +124,7 @@ func (cache *importCache) importAST(importedFrom, importedPath string) (ast.Node
} }
// ImportString imports a string, caches it and then returns it. // ImportString imports a string, caches it and then returns it.
func (cache *importCache) importString(importedFrom, importedPath string, i *interpreter, trace traceElement) (*valueString, error) { func (cache *importCache) importString(importedFrom, importedPath string, i *interpreter, trace traceElement) (valueString, error) {
data, _, err := cache.importData(importedFrom, importedPath) data, _, err := cache.importData(importedFrom, importedPath)
if err != nil { if err != nil {
return nil, i.Error(err.Error(), trace) return nil, i.Error(err.Error(), trace)

30
interpreter.go
View File

@ -379,8 +379,8 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
} }
var fieldName string var fieldName string
switch fieldNameValue := fieldNameValue.(type) { switch fieldNameValue := fieldNameValue.(type) {
case *valueString: case valueString:
fieldName = fieldNameValue.getString() fieldName = fieldNameValue.getGoString()
case *valueNull: case *valueNull:
// Omitted field. // Omitted field.
continue continue
@ -424,7 +424,7 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return nil, i.Error(msg.getString(), trace) return nil, i.Error(msg.getGoString(), trace)
case *ast.Index: case *ast.Index:
targetValue, err := i.evaluate(node.Target, nonTailCall) targetValue, err := i.evaluate(node.Target, nonTailCall)
@ -441,7 +441,7 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return target.index(i, trace, indexString.getString()) return target.index(i, trace, indexString.getGoString())
case *valueArray: case *valueArray:
indexInt, err := i.getNumber(index, trace) indexInt, err := i.getNumber(index, trace)
if err != nil { if err != nil {
@ -450,7 +450,7 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
// TODO(https://github.com/google/jsonnet/issues/377): non-integer indexes should be an error // TODO(https://github.com/google/jsonnet/issues/377): non-integer indexes should be an error
return target.index(i, trace, int(indexInt.value)) return target.index(i, trace, int(indexInt.value))
case *valueString: case valueString:
indexInt, err := i.getNumber(index, trace) indexInt, err := i.getNumber(index, trace)
if err != nil { if err != nil {
return nil, err return nil, err
@ -517,7 +517,7 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return objectIndex(i, trace, i.stack.getSelfBinding().super(), indexStr.getString()) return objectIndex(i, trace, i.stack.getSelfBinding().super(), indexStr.getGoString())
case *ast.InSuper: case *ast.InSuper:
index, err := i.evaluate(node.Index, nonTailCall) index, err := i.evaluate(node.Index, nonTailCall)
@ -528,7 +528,7 @@ func (i *interpreter) evaluate(a ast.Node, tc tailCallStatus) (value, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
hasField := objectHasField(i.stack.getSelfBinding().super(), indexStr.getString(), withHidden) hasField := objectHasField(i.stack.getSelfBinding().super(), indexStr.getGoString(), withHidden)
return makeValueBoolean(hasField), nil return makeValueBoolean(hasField), nil
case *ast.Function: case *ast.Function:
@ -636,8 +636,8 @@ func (i *interpreter) manifestJSON(trace traceElement, v value) (interface{}, er
case *valueNumber: case *valueNumber:
return v.value, nil return v.value, nil
case *valueString: case valueString:
return v.getString(), nil return v.getGoString(), nil
case *valueNull: case *valueNull:
return nil, nil return nil, nil
@ -802,8 +802,8 @@ func (i *interpreter) manifestAndSerializeJSON(
// manifestString expects the value to be a string and returns it. // manifestString expects the value to be a string and returns it.
func (i *interpreter) manifestString(buf *bytes.Buffer, trace traceElement, v value) error { func (i *interpreter) manifestString(buf *bytes.Buffer, trace traceElement, v value) error {
switch v := v.(type) { switch v := v.(type) {
case *valueString: case valueString:
buf.WriteString(v.getString()) buf.WriteString(v.getGoString())
return nil return nil
default: default:
return makeRuntimeError(fmt.Sprintf("expected string result, got: %s", v.getType().name), i.getCurrentStackTrace(trace)) return makeRuntimeError(fmt.Sprintf("expected string result, got: %s", v.getType().name), i.getCurrentStackTrace(trace))
@ -1010,16 +1010,16 @@ func (i *interpreter) evaluateInt64(pv potentialValue, trace traceElement) (int6
return i.getInt64(v, trace) return i.getInt64(v, trace)
} }
func (i *interpreter) getString(val value, trace traceElement) (*valueString, error) { func (i *interpreter) getString(val value, trace traceElement) (valueString, error) {
switch v := val.(type) { switch v := val.(type) {
case *valueString: case valueString:
return v, nil return v, nil
default: default:
return nil, i.typeErrorSpecific(val, &valueString{}, trace) return nil, i.typeErrorSpecific(val, emptyString(), trace)
} }
} }
func (i *interpreter) evaluateString(pv potentialValue, trace traceElement) (*valueString, error) { func (i *interpreter) evaluateString(pv potentialValue, trace traceElement) (valueString, error) {
v, err := i.evaluatePV(pv, trace) v, err := i.evaluatePV(pv, trace)
if err != nil { if err != nil {
return nil, err return nil, err

168
value.go
View File

@ -73,73 +73,165 @@ func (v *valueBase) aValue() {}
// Primitive values // Primitive values
// ------------------------------------- // -------------------------------------
// valueString represents a string value, internally using a []rune for quick type valueString interface {
value
length() int
getRunes() []rune
getGoString() string
index(i *interpreter, trace traceElement, index int) (value, error)
}
// valueFlatString represents a string value, internally using a []rune for quick
// indexing. // indexing.
type valueString struct { type valueFlatString struct {
valueBase valueBase
// We use rune slices instead of strings for quick indexing // We use rune slices instead of strings for quick indexing
value []rune value []rune
} }
func (s *valueString) index(i *interpreter, trace traceElement, index int) (value, error) { func (s *valueFlatString) index(i *interpreter, trace traceElement, index int) (value, error) {
if 0 <= index && index < s.length() { if 0 <= index && index < s.length() {
return makeValueString(string(s.value[index])), nil return makeValueString(string(s.value[index])), nil
} }
return nil, i.Error(fmt.Sprintf("Index %d out of bounds, not within [0, %v)", index, s.length()), trace) return nil, i.Error(fmt.Sprintf("Index %d out of bounds, not within [0, %v)", index, s.length()), trace)
} }
func concatStrings(a, b *valueString) *valueString { func (s *valueFlatString) getRunes() []rune {
result := make([]rune, 0, len(a.value)+len(b.value)) return s.value
for _, r := range a.value {
result = append(result, r)
}
for _, r := range b.value {
result = append(result, r)
}
return &valueString{value: result}
} }
func stringLessThan(a, b *valueString) bool { func (s *valueFlatString) getGoString() string {
var length int return string(s.value)
if len(a.value) < len(b.value) { }
length = len(a.value)
func (s *valueFlatString) length() int {
return len(s.value)
}
func (s *valueFlatString) getType() *valueType {
return stringType
}
// TODO(sbarzowski) this was chosen on a hunch, be more systematic about it
const extendedStringMinLength = 30
// Indirect representation of long strings.
// It consists of two parts, left and right, concatenation of which is the whole string.
type valueStringTree struct {
valueBase
left, right valueString
len int
}
func buildFullString(s valueString, buf *[]rune) {
switch s := s.(type) {
case *valueFlatString:
*buf = append(*buf, s.value...)
case *valueStringTree:
buildFullString(s.left, buf)
buildFullString(s.right, buf)
}
}
func (s *valueStringTree) flattenToLeft() {
if s.right != nil {
result := make([]rune, 0, s.len)
buildFullString(s, &result)
s.left = makeStringFromRunes(result)
s.right = nil
}
}
func (s *valueStringTree) index(i *interpreter, trace traceElement, index int) (value, error) {
if 0 <= index && index < s.len {
s.flattenToLeft()
return s.left.index(i, trace, index)
}
return nil, i.Error(fmt.Sprintf("Index %d out of bounds, not within [0, %v)", index, s.length()), trace)
}
func (s *valueStringTree) getRunes() []rune {
s.flattenToLeft()
return s.left.getRunes()
}
func (s *valueStringTree) getGoString() string {
s.flattenToLeft()
return s.left.getGoString()
}
func (s *valueStringTree) length() int {
return s.len
}
func (s *valueStringTree) getType() *valueType {
return stringType
}
func emptyString() *valueFlatString {
return &valueFlatString{}
}
func makeStringFromRunes(runes []rune) *valueFlatString {
return &valueFlatString{value: runes}
}
func concatStrings(a, b valueString) valueString {
aLen := a.length()
bLen := b.length()
if aLen == 0 {
return b
} else if bLen == 0 {
return a
} else if aLen+bLen < extendedStringMinLength {
runesA := a.getRunes()
runesB := b.getRunes()
result := make([]rune, 0, aLen+bLen)
result = append(result, runesA...)
result = append(result, runesB...)
return makeStringFromRunes(result)
} else { } else {
length = len(b.value) return &valueStringTree{
left: a,
right: b,
len: aLen + bLen,
}
}
}
func stringLessThan(a, b valueString) bool {
runesA := a.getRunes()
runesB := b.getRunes()
var length int
if len(runesA) < len(runesB) {
length = len(runesA)
} else {
length = len(runesB)
} }
for i := 0; i < length; i++ { for i := 0; i < length; i++ {
if a.value[i] != b.value[i] { if runesA[i] != runesB[i] {
return a.value[i] < b.value[i] return runesA[i] < runesB[i]
} }
} }
return len(a.value) < len(b.value) return len(runesA) < len(runesB)
} }
func stringEqual(a, b *valueString) bool { func stringEqual(a, b valueString) bool {
if len(a.value) != len(b.value) { runesA := a.getRunes()
runesB := b.getRunes()
if len(runesA) != len(runesB) {
return false return false
} }
for i := 0; i < len(a.value); i++ { for i := 0; i < len(runesA); i++ {
if a.value[i] != b.value[i] { if runesA[i] != runesB[i] {
return false return false
} }
} }
return true return true
} }
func (s *valueString) length() int { func makeValueString(v string) valueString {
return len(s.value) return &valueFlatString{value: []rune(v)}
}
func (s *valueString) getString() string {
return string(s.value)
}
func makeValueString(v string) *valueString {
return &valueString{value: []rune(v)}
}
func (*valueString) getType() *valueType {
return stringType
} }
type valueBoolean struct { type valueBoolean struct {