Update to latest CPP version

bazel/repositories.bzl

@@ -4,8 +4,8 @@ load(
 )
 
 # NB: update_cpp_jsonnet.sh looks for these.
-CPP_JSONNET_SHA256 = "867fe09842e9868c95d57c19648674241a83c581a780ab4f4a5a37ac2afb06be"
+CPP_JSONNET_SHA256 = "965dac82878ef2c2df5ad69095bfeceb04cbef7ca505ee87c038b2c7fdd54c6c"
-CPP_JSONNET_GITHASH = "34419d2483927ceb17cd506cad77c3c2a96e7b8c"
+CPP_JSONNET_GITHASH = "295345366e1fdc0ee9ab7048c352750d45053efd"
 
 def jsonnet_go_repositories():
     http_archive(

builtins.go

@@ -1211,8 +1211,8 @@ func builtinSplitLimit(i *interpreter, strv, cv, maxSplitsV value) (value, error
 	}
 	sStr := str.getGoString()
 	sC := c.getGoString()
-	if len(sC) != 1 {
+	if len(sC) < 1 {
-		return nil, i.Error(fmt.Sprintf("std.splitLimit second parameter should have length 1, got %v", len(sC)))
+		return nil, i.Error(fmt.Sprintf("std.splitLimit second parameter should have length 1 or greater, got %v", len(sC)))
 	}
 
 	// the convention is slightly different from strings.splitN in Go (the meaning of non-negative values is shifted by one)

c-bindings-tests/compat_test.py

@@ -113,13 +113,13 @@ lib.jsonnet_native_callback.argtypes = [
 lib.jsonnet_native_callback.restype = None
 
 IMPORT_CALLBACK = ctypes.CFUNCTYPE(
-    ctypes.c_char_p,
+    ctypes.c_int,
     ctypes.c_void_p,
     ctypes.POINTER(ctypes.c_char),
     ctypes.POINTER(ctypes.c_char),
-    # we use *int instead of **char to pass the real C allocated pointer, that we have to free
+    ctypes.POINTER(ctypes.c_char_p),
-    ctypes.POINTER(ctypes.c_uint64),
+    ctypes.POINTER(ctypes.c_void_p),
-    ctypes.POINTER(ctypes.c_int)
+    ctypes.POINTER(ctypes.c_uint64)
 )
 
 lib.jsonnet_import_callback.argtypes = [
@@ -338,20 +338,21 @@ def build_native(ctx, argv, success):
     return res
 
 @IMPORT_CALLBACK
-def import_callback(ctx, dir, rel, found_here, success):
+def import_callback(ctx, dir, rel, found_here, buf_ptr, size_ptr):
     full_path, content = jsonnet_try_path(b"jsonnet_import_test/", to_bytes(rel))
 
     bcontent = content.encode()
-    dst = lib.jsonnet_realloc(ctx, None, len(bcontent) + 1)
+    dst = lib.jsonnet_realloc(ctx, None, len(bcontent))
-    ctypes.memmove(ctypes.addressof(dst.contents), bcontent, len(bcontent) + 1)
+    ctypes.memmove(ctypes.addressof(dst.contents), bcontent, len(bcontent))
 
     fdst = lib.jsonnet_realloc(ctx, None, len(full_path) + 1)
     ctypes.memmove(ctypes.addressof(fdst.contents), full_path, len(full_path) + 1)
     found_here[0] = ctypes.addressof(fdst.contents)
 
-    success[0] = ctypes.c_int(1)
+    buf_ptr[0] = ctypes.addressof(dst.contents)
-
+    size_ptr[0] = len(bcontent)
-    return ctypes.addressof(dst.contents)
+    
+    return 0
 
 io_writer_buf = None
 

c-bindings/c-bindings.go

@@ -60,7 +60,9 @@ type importer struct {
 // Import fetches data from a given path by using c.JsonnetImportCallback
 func (i *importer) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) {
 	var (
-		success    = C.int(0)
+		buflen     = C.size_t(0)
+		msgC       *C.char
+		bufPtr       unsafe.Pointer
 		dir, _     = path.Split(importedFrom)
 		foundHereC *C.char
 	)
@@ -71,19 +73,22 @@ func (i *importer) Import(importedFrom, importedPath string) (contents jsonnet.C
 	// returning nothing (NULL), if they know that we have the contents
 	// cached anyway.
 
-	resultC := C.jsonnet_internal_execute_import(i.cb, i.ctx, C.CString(dir), C.CString(importedPath), &foundHereC, &success)
+	success := C.jsonnet_internal_execute_import(i.cb, i.ctx, C.CString(dir), C.CString(importedPath), &foundHereC, &msgC, &bufPtr, &buflen)
-	result := C.GoString(resultC)
+	if success != 0 {
-	C.jsonnet_internal_free_string(resultC)
+		// Failure
+		msg := C.GoString(msgC)
+		C.jsonnet_internal_free_string(msgC)
+		return jsonnet.Contents{}, "", errors.New("importer error: " + msg)
+	}
+	result := C.GoBytes(bufPtr, C.int(buflen))
+	C.jsonnet_internal_free_pointer(bufPtr)
 
 	foundHere := C.GoString(foundHereC)
 	C.jsonnet_internal_free_string(foundHereC)
 
-	if success != 1 {
-		return jsonnet.Contents{}, "", errors.New("importer error: " + result)
-	}
 
 	if _, isCached := i.contentCache[foundHere]; !isCached {
-		i.contentCache[foundHere] = jsonnet.MakeContents(result)
+		i.contentCache[foundHere] = jsonnet.MakeContentsRaw(result)
 	}
 	return i.contentCache[foundHere], foundHere, nil
 }

c-bindings/internal.h

@@ -25,18 +25,16 @@ struct JsonnetJsonValue* jsonnet_internal_execute_native(JsonnetNativeCallback *
                                                   const struct JsonnetJsonValue *const *argv,
                                                   int *success);
 
-typedef char *JsonnetImportCallback(void *ctx,
+typedef int JsonnetImportCallback(void *ctx, const char *base, const char *rel,
+                                  char **found_here, char **buf, size_t *buflen);
+
+int jsonnet_internal_execute_import(JsonnetImportCallback *cb,
+                                    void *ctx,
                                     const char *base,
                                     const char *rel,
                                     char **found_here,
-                                    int *success);
+                                    char **msg,
-
+                                    void **buf, size_t *buflen);
-char* jsonnet_internal_execute_import(JsonnetImportCallback *cb,
-                                      void *ctx,
-                                      const char *base,
-                                      const char *rel,
-                                      char **found_here,
-                                      int *success);
 
 typedef int JsonnetIoWriterCallback(const void *buf, size_t nbytes, int *success);
 
@@ -46,5 +44,6 @@ int jsonnet_internal_execute_writer(JsonnetIoWriterCallback *cb,
                                     int *success);
 
 void jsonnet_internal_free_string(char *str);
+void jsonnet_internal_free_pointer(void *ptr);
 
 char* jsonnet_internal_realloc(struct JsonnetVm *vm, char *str, size_t sz);

c-bindings/libjsonnet.cpp

@@ -34,14 +34,24 @@ struct JsonnetJsonValue* jsonnet_internal_execute_native(JsonnetNativeCallback *
     return (cb)(ctx, argv, success);
 }
 
-char* jsonnet_internal_execute_import(JsonnetImportCallback *cb,
+int jsonnet_internal_execute_import(JsonnetImportCallback *cb,
-                                      void *ctx,
+                                    void *ctx,
-                                      const char *base,
+                                    const char *base,
-                                      const char *rel,
+                                    const char *rel,
-                                      char **found_here,
+                                    char **found_here,
-                                      int *success)
+                                    char **msg,
+                                    void **buf, size_t *buflen)
 {
-    return (cb)(ctx, base, rel, found_here, success);
+    char *char_buf;
+    int success = (cb)(ctx, base, rel, found_here, &char_buf, buflen);
+    if (success == 0) {
+		// Success
+        *buf = char_buf;
+    } else {
+		// Fail
+        *msg = char_buf;
+    }
+    return success;
 }
 
 int jsonnet_internal_execute_writer(JsonnetIoWriterCallback *cb,
@@ -53,9 +63,11 @@ int jsonnet_internal_execute_writer(JsonnetIoWriterCallback *cb,
 }
 
 void jsonnet_internal_free_string(char *str) {
-    if (str != nullptr) {
+    ::free(str);
-        ::free(str);
+}
-    }
+
+void jsonnet_internal_free_pointer(void *ptr) {
+    ::free(ptr);
 }
 
 void jsonnet_gc_min_objects(struct JsonnetVm *vm, unsigned v) {

cpp-jsonnet

@@ -1 +1 @@
-Subproject commit 34419d2483927ceb17cd506cad77c3c2a96e7b8c
+Subproject commit 295345366e1fdc0ee9ab7048c352750d45053efd

python/_jsonnet.c

@@ -23,8 +23,17 @@ limitations under the License.
 
 static char *jsonnet_str(struct JsonnetVm *vm, const char *str)
 {
-    char *out = jsonnet_realloc(vm, NULL, strlen(str) + 1);
+    size_t size = strlen(str) + 1;
-    memcpy(out, str, strlen(str) + 1);
+    char *out = jsonnet_realloc(vm, NULL, size);
+    memcpy(out, str, size);
+    return out;
+}
+
+static char *jsonnet_str_nonull(struct JsonnetVm *vm, const char *str, size_t *buflen)
+{
+    *buflen = strlen(str);
+    char *out = jsonnet_realloc(vm, NULL, *buflen);
+    memcpy(out, str, *buflen);
     return out;
 }
 
@@ -209,12 +218,12 @@ struct ImportCtx {
     PyObject *callback;
 };
 
-static char *cpython_import_callback(void *ctx_, const char *base, const char *rel,
+static int cpython_import_callback(void *ctx_, const char *base, const char *rel,
-                                     char **found_here, int *success)
+                                   char **found_here, char **buf, size_t *buflen)
 {
     const struct ImportCtx *ctx = ctx_;
     PyObject *arglist, *result;
-    char *out;
+    int success;
 
     PyEval_RestoreThread(*ctx->py_thread);
     arglist = Py_BuildValue("(s, s)", base, rel);
@@ -223,47 +232,50 @@ static char *cpython_import_callback(void *ctx_, const char *base, const char *r
 
     if (result == NULL) {
         // Get string from exception
-        char *out = jsonnet_str(ctx->vm, exc_to_str());
+        *buf = jsonnet_str_nonull(ctx->vm, exc_to_str(), buflen);
-        *success = 0;
         PyErr_Clear();
         *ctx->py_thread = PyEval_SaveThread();
-        return out;
+        return 1; // failure
     }
 
     if (!PyTuple_Check(result)) {
-        out = jsonnet_str(ctx->vm, "import_callback did not return a tuple");
+        *buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return a tuple", buflen);
-        *success = 0;
+        success = 0;
     } else if (PyTuple_Size(result) != 2) {
-        out = jsonnet_str(ctx->vm, "import_callback did not return a tuple (size 2)");
+        *buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return a tuple (size 2)", buflen);
-        *success = 0;
+        success = 0;
     } else {
         PyObject *file_name = PyTuple_GetItem(result, 0);
         PyObject *file_content = PyTuple_GetItem(result, 1);
 #if PY_MAJOR_VERSION >= 3
-        if (!PyUnicode_Check(file_name) || !PyUnicode_Check(file_content)) {
+        if (!PyUnicode_Check(file_name) || !PyBytes_Check(file_content)) {
 #else
         if (!PyString_Check(file_name) || !PyString_Check(file_content)) {
 #endif
-            out = jsonnet_str(ctx->vm, "import_callback did not return a pair of strings");
+            *buf = jsonnet_str_nonull(ctx->vm, "import_callback did not return (string, bytes)", buflen);
-            *success = 0;
+            success = 0;
         } else {
+            const char *content_buf;
+            const ssize_t content_len;
 #if PY_MAJOR_VERSION >= 3
             const char *found_here_cstr = PyUnicode_AsUTF8(file_name);
-            const char *content_cstr = PyUnicode_AsUTF8(file_content);
+            PyBytes_AsStringAndSize(file_content, &content_buf, &content_len);
 #else
             const char *found_here_cstr = PyString_AsString(file_name);
-            const char *content_cstr = PyString_AsString(file_content);
+            PyString_AsStringAndSize(file_content, &content_buf, &content_len);
 #endif
             *found_here = jsonnet_str(ctx->vm, found_here_cstr);
-            out = jsonnet_str(ctx->vm, content_cstr);
+            *buflen = content_len - 1; // Python always adds a trailing null
-            *success = 1;
+            *buf = jsonnet_realloc(ctx->vm, NULL, *buflen);
+            memcpy(*buf, content_buf, *buflen);
+            success = 1;
         }
     }
 
     Py_DECREF(result);
     *ctx->py_thread = PyEval_SaveThread();
 
-    return out;
+    return success ? 0 : 1;
 }
 
 static PyObject *handle_result(struct JsonnetVm *vm, char *out, int error)

python/_jsonnet_test.py

@@ -36,7 +36,7 @@ def try_path_cached(cache, dir, rel):
             cache[full_path] = None
         else:
             with open(full_path) as f:
-                cache[full_path] = f.read()
+                cache[full_path] = f.read().encode()
     return full_path, cache[full_path]
 
 

testdata/assert_equal4.golden

@@ -1,6 +1,6 @@
 RUNTIME ERROR: Assertion failed. {"x": 1} != {"x": 2}
 -------------------------------------------------
-	<std>:814:7-50	function <anonymous>
+	<std>:826:7-50	function <anonymous>
 
       error 'Assertion failed. ' + a + ' != ' + b,
 

testdata/assert_equal5.golden

@@ -2,7 +2,7 @@ RUNTIME ERROR: Assertion failed.
   != 
 
 -------------------------------------------------
-	<std>:814:7-50	function <anonymous>
+	<std>:826:7-50	function <anonymous>
 
       error 'Assertion failed. ' + a + ' != ' + b,
 

testdata/assert_equal6.golden

@@ -1,6 +1,6 @@
 RUNTIME ERROR: Assertion failed.  != 
 -------------------------------------------------
-	<std>:814:7-50	function <anonymous>
+	<std>:826:7-50	function <anonymous>
 
       error 'Assertion failed. ' + a + ' != ' + b,
 

testdata/builtinReverse_not_array.golden

@@ -1,8 +1,8 @@
-RUNTIME ERROR: Unexpected type string, expected array
+RUNTIME ERROR: Unexpected type boolean, expected array
 -------------------------------------------------
-	testdata/builtinReverse_not_array:1:1-20	$
+	testdata/builtinReverse_not_array:1:1-19	$
 
-std.reverse("asdf")
+std.reverse(false)
 
 -------------------------------------------------
 	During evaluation	

testdata/builtinReverse_not_array.jsonnet

@@ -1 +1 @@
-std.reverse("asdf")
+std.reverse(false)

testdata/builtinReverse_not_array.linter.golden

@@ -1,10 +1,10 @@
 ../testdata/builtinReverse_not_array:1:1-12 Indexed object has no field "reverse"
 
-std.reverse("asdf")
+std.reverse(false)
 
 
-../testdata/builtinReverse_not_array:1:1-20 Called value must be a function, but it is assumed to be void
+../testdata/builtinReverse_not_array:1:1-19 Called value must be a function, but it is assumed to be void
 
-std.reverse("asdf")
+std.reverse(false)
 
 

testdata/percent_bad.golden

@@ -1,6 +1,6 @@
 RUNTIME ERROR: Operator % cannot be used on types number and string.
 -------------------------------------------------
-	<std>:239:7-94	function <anonymous>
+	<std>:251:7-94	function <anonymous>
 
       error 'Operator % cannot be used on types ' + std.type(a) + ' and ' + std.type(b) + '.',
 

testdata/percent_bad2.golden

@@ -1,21 +1,21 @@
 RUNTIME ERROR: Too many values to format: 1, expected 0
 -------------------------------------------------
-	<std>:684:11-86	function <format_codes_arr>
+	<std>:696:11-86	function <format_codes_arr>
 
           error ('Too many values to format: ' + std.length(arr) + ', expected ' + j)
 
 -------------------------------------------------
-	<std>:690:11-59	function <format_codes_arr>
+	<std>:702:11-59	function <format_codes_arr>
 
           format_codes_arr(codes, arr, i + 1, j, v + code) tailstrict
 
 -------------------------------------------------
-	<std>:781:7-48	function <anonymous>
+	<std>:793:7-48	function <anonymous>
 
       format_codes_arr(codes, [vals], 0, 0, ''),
 
 -------------------------------------------------
-	<std>:237:7-23	function <anonymous>
+	<std>:249:7-23	function <anonymous>
 
       std.format(a, b)
 

testdata/percent_bad3.golden

@@ -1,6 +1,6 @@
 RUNTIME ERROR: Operator % cannot be used on types function and number.
 -------------------------------------------------
-	<std>:239:7-94	function <anonymous>
+	<std>:251:7-94	function <anonymous>
 
       error 'Operator % cannot be used on types ' + std.type(a) + ' and ' + std.type(b) + '.',
 

testdata/percent_format_str4.golden

@@ -1,21 +1,21 @@
 RUNTIME ERROR: Too many values to format: 2, expected 1
 -------------------------------------------------
-	<std>:684:11-86	function <format_codes_arr>
+	<std>:696:11-86	function <format_codes_arr>
 
           error ('Too many values to format: ' + std.length(arr) + ', expected ' + j)
 
 -------------------------------------------------
-	<std>:690:11-59	function <format_codes_arr>
+	<std>:702:11-59	function <format_codes_arr>
 
           format_codes_arr(codes, arr, i + 1, j, v + code) tailstrict
 
 -------------------------------------------------
-	<std>:777:7-46	function <anonymous>
+	<std>:789:7-46	function <anonymous>
 
       format_codes_arr(codes, vals, 0, 0, '')
 
 -------------------------------------------------
-	<std>:237:7-23	function <anonymous>
+	<std>:249:7-23	function <anonymous>
 
       std.format(a, b)
 

testdata/percent_format_str5.golden

@@ -1,38 +1,38 @@
 RUNTIME ERROR: Not enough values to format: 1, expected more than 1
 -------------------------------------------------
-	<std>:717:15-103	thunk <val> from <function <format_codes_arr>>
+	<std>:729:15-103	thunk <val> from <function <format_codes_arr>>
 
               error ('Not enough values to format: ' + std.length(arr) + ', expected more than ' + j2);
 
 -------------------------------------------------
-	<std>:722:27-30	thunk from <thunk <s> from <function <format_codes_arr>>>
+	<std>:734:27-30	thunk from <thunk <s> from <function <format_codes_arr>>>
 
               format_code(val, code, tmp.fw, tmp2.prec, j2);
 
 -------------------------------------------------
-	<std>:592:22-25	thunk from <function <format_code>>
+	<std>:604:22-25	thunk from <function <format_code>>
 
         std.toString(val)
 
 -------------------------------------------------
-	<std>:592:9-26	function <format_code>
+	<std>:604:9-26	function <format_code>
 
         std.toString(val)
 
 -------------------------------------------------
 	... (skipped 10 frames)
 -------------------------------------------------
-	<std>:733:11-64	function <format_codes_arr>
+	<std>:745:11-64	function <format_codes_arr>
 
           format_codes_arr(codes, arr, i + 1, j3, v + s_padded) tailstrict;
 
 -------------------------------------------------
-	<std>:777:7-46	function <anonymous>
+	<std>:789:7-46	function <anonymous>
 
       format_codes_arr(codes, vals, 0, 0, '')
 
 -------------------------------------------------
-	<std>:237:7-23	function <anonymous>
+	<std>:249:7-23	function <anonymous>
 
       std.format(a, b)
 

testdata/percent_format_str6.golden

@@ -1,38 +1,38 @@
 RUNTIME ERROR: Not enough values to format: 1, expected more than 1
 -------------------------------------------------
-	<std>:717:15-103	thunk <val> from <function <format_codes_arr>>
+	<std>:729:15-103	thunk <val> from <function <format_codes_arr>>
 
               error ('Not enough values to format: ' + std.length(arr) + ', expected more than ' + j2);
 
 -------------------------------------------------
-	<std>:722:27-30	thunk from <thunk <s> from <function <format_codes_arr>>>
+	<std>:734:27-30	thunk from <thunk <s> from <function <format_codes_arr>>>
 
               format_code(val, code, tmp.fw, tmp2.prec, j2);
 
 -------------------------------------------------
-	<std>:592:22-25	thunk from <function <format_code>>
+	<std>:604:22-25	thunk from <function <format_code>>
 
         std.toString(val)
 
 -------------------------------------------------
-	<std>:592:9-26	function <format_code>
+	<std>:604:9-26	function <format_code>
 
         std.toString(val)
 
 -------------------------------------------------
 	... (skipped 10 frames)
 -------------------------------------------------
-	<std>:733:11-64	function <format_codes_arr>
+	<std>:745:11-64	function <format_codes_arr>
 
           format_codes_arr(codes, arr, i + 1, j3, v + s_padded) tailstrict;
 
 -------------------------------------------------
-	<std>:781:7-48	function <anonymous>
+	<std>:793:7-48	function <anonymous>
 
       format_codes_arr(codes, [vals], 0, 0, ''),
 
 -------------------------------------------------
-	<std>:237:7-23	function <anonymous>
+	<std>:249:7-23	function <anonymous>
 
       std.format(a, b)
 

testdata/percent_format_str7.golden

@@ -1,39 +1,39 @@
 RUNTIME ERROR: Format required number at 0, got string
 -------------------------------------------------
-	<std>:(595:11)-(596:47)	function <format_code>
+	<std>:(607:11)-(608:47)	function <format_code>
 
           error 'Format required number at '
                 + i + ', got ' + std.type(val)
 
 -------------------------------------------------
-	<std>:722:15-60	thunk <s> from <function <format_codes_arr>>
+	<std>:734:15-60	thunk <s> from <function <format_codes_arr>>
 
               format_code(val, code, tmp.fw, tmp2.prec, j2);
 
 -------------------------------------------------
-	<std>:727:24-25	thunk from <thunk <s_padded> from <function <format_codes_arr>>>
+	<std>:739:24-25	thunk from <thunk <s_padded> from <function <format_codes_arr>>>
 
               pad_left(s, tmp.fw, ' ');
 
 -------------------------------------------------
-	<std>:480:30-33	thunk from <thunk from <function <pad_left>>>
+	<std>:492:30-33	thunk from <thunk from <function <pad_left>>>
 
       padding(w - std.length(str), s) + str;
 
 -------------------------------------------------
 	... (skipped 7 frames)
 -------------------------------------------------
-	<std>:733:11-64	function <format_codes_arr>
+	<std>:745:11-64	function <format_codes_arr>
 
           format_codes_arr(codes, arr, i + 1, j3, v + s_padded) tailstrict;
 
 -------------------------------------------------
-	<std>:777:7-46	function <anonymous>
+	<std>:789:7-46	function <anonymous>
 
       format_codes_arr(codes, vals, 0, 0, '')
 
 -------------------------------------------------
-	<std>:237:7-23	function <anonymous>
+	<std>:249:7-23	function <anonymous>
 
       std.format(a, b)
 

testdata/percent_mod_int5.golden

@@ -1,6 +1,6 @@
 RUNTIME ERROR: Division by zero.
 -------------------------------------------------
-	<std>:235:7-23	function <anonymous>
+	<std>:247:7-23	function <anonymous>
 
       std.modulo(a, b)