diff --git a/Makefile.envs b/Makefile.envs index 3f5a3dc14de..1a05e174ec8 100644 --- a/Makefile.envs +++ b/Makefile.envs @@ -146,6 +146,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \ -lworkerfs.js \ -lwebsocket.js \ -leventloop.js \ + -lhiwire \ \ -lGL \ -legl.js \ @@ -165,7 +166,9 @@ export MAIN_MODULE_CFLAGS= $(CFLAGS_BASE) \ -Werror=int-conversion \ -Werror=incompatible-pointer-types \ -Werror=unused-result \ + -mreference-types \ -I$(PYTHONINCLUDE) \ + -I$(PYTHONINCLUDE)/.. \ -s EXCEPTION_CATCHING_ALLOWED=['we only want to allow exception handling in side modules'] export STDLIB_MODULE_CFLAGS= $(SIDE_MODULE_CFLAGS) -I Include/ -I . -I Include/internal/ diff --git a/cpython/Makefile b/cpython/Makefile index 2eca8da9c33..b3314153934 100644 --- a/cpython/Makefile +++ b/cpython/Makefile @@ -14,11 +14,15 @@ FFIBUILD=$(ROOT)/build/libffi LIBFFIREPO=https://github.com/libffi/libffi LIBFFI_COMMIT=f08493d249d2067c8b3207ba46693dd858f95db3 +HIWIREBUILD=$(ROOT)/build/hiwire +HIWIREREPO=https://github.com/hoodmane/hiwire +HIWIRE_COMMIT=49f3450e34f3f50d4b8296e782dc321bb2e3264e + ifdef CPYTHON_DEBUG MAYBE_WITH_PYDEBUG=--with-pydebug endif -all: $(INSTALL)/lib/$(LIB) $(INSTALL)/lib/libffi.a +all: $(INSTALL)/lib/$(LIB) $(INSTALL)/lib/libffi.a $(INSTALL)/lib/libhiwire.a $(INSTALL)/lib/$(LIB): $(BUILD)/$(LIB) @@ -84,6 +88,20 @@ $(INSTALL)/lib/libffi.a : mkdir -p $(INSTALL)/lib cp $(FFIBUILD)/target/lib/libffi.a $(INSTALL)/lib/ +$(INSTALL)/lib/libhiwire.a : + rm -rf $(HIWIREBUILD) + mkdir $(HIWIREBUILD) + (\ + cd $(HIWIREBUILD) \ + && git init \ + && git fetch --depth 1 $(HIWIREREPO) $(HIWIRE_COMMIT) \ + && git checkout FETCH_HEAD \ + && . $(PYODIDE_ROOT)/emsdk/emsdk/emsdk_env.sh \ + && CC=emcc EMSCRIPTEN_DEDUPLICATE=1 EXTERN_FAIL=1 make \ + ) + cp -r $(HIWIREBUILD)/dist/lib $(INSTALL)/ + cp -r $(HIWIREBUILD)/dist/include $(INSTALL)/include/hiwire + $(BUILD)/Makefile: $(BUILD)/.patched # --enable-big-digits=30 : # Python integers have "digits" of size 15 by default on systems with 32 diff --git a/emsdk/patches/0001-Support-externref-in-EM_JS-functions-with-dynamic-li.patch b/emsdk/patches/0001-Support-externref-in-EM_JS-functions-with-dynamic-li.patch new file mode 100644 index 00000000000..eabc473bf96 --- /dev/null +++ b/emsdk/patches/0001-Support-externref-in-EM_JS-functions-with-dynamic-li.patch @@ -0,0 +1,103 @@ +From 04dd736fe3ea0d469b84800585eaa4ca6648b9f9 Mon Sep 17 00:00:00 2001 +From: Hood Chatham +Date: Thu, 31 Aug 2023 13:53:39 +0200 +Subject: [PATCH] Support externref in EM_JS functions with dynamic linking + +--- + emscripten.py | 1 + + src/library_addfunction.js | 1 + + test/core/test_externref2.c | 22 ++++++++++++++++++++++ + test/core/test_externref2.out | 2 ++ + test/test_core.py | 14 ++++++++++++++ + 5 files changed, 40 insertions(+) + create mode 100644 test/core/test_externref2.c + create mode 100644 test/core/test_externref2.out + +diff --git a/emscripten.py b/emscripten.py +index f011a58b8..74ea6f525 100644 +--- a/emscripten.py ++++ b/emscripten.py +@@ -601,6 +601,7 @@ def type_to_sig(type): + webassembly.Type.I64: 'j', + webassembly.Type.F32: 'f', + webassembly.Type.F64: 'd', ++ webassembly.Type.EXTERNREF: 'e', + webassembly.Type.VOID: 'v' + }[type] + +diff --git a/src/library_addfunction.js b/src/library_addfunction.js +index 39e00b772..1537dca66 100644 +--- a/src/library_addfunction.js ++++ b/src/library_addfunction.js +@@ -29,6 +29,7 @@ addToLibrary({ + 'j': 'i64', + 'f': 'f32', + 'd': 'f64', ++ 'e': 'externref', + #if MEMORY64 + 'p': 'i64', + #else +diff --git a/test/core/test_externref2.c b/test/core/test_externref2.c +new file mode 100644 +index 000000000..47451c260 +--- /dev/null ++++ b/test/core/test_externref2.c +@@ -0,0 +1,22 @@ ++#include "emscripten.h" ++ ++ ++EM_JS(__externref_t, get_ref, (), { ++ return {a: 7, b: 9}; ++}); ++ ++EM_JS(void, modify_ref, (__externref_t arg), { ++ arg.a += 3; ++ arg.b -= 3; ++}); ++ ++EM_JS(void, log_ref, (__externref_t arg), { ++ console.log(arg); ++}); ++ ++int main() { ++ __externref_t a = get_ref(); ++ log_ref(a); ++ modify_ref(a); ++ log_ref(a); ++} +diff --git a/test/core/test_externref2.out b/test/core/test_externref2.out +new file mode 100644 +index 000000000..eaceb4e73 +--- /dev/null ++++ b/test/core/test_externref2.out +@@ -0,0 +1,2 @@ ++{ a: 7, b: 9 } ++{ a: 10, b: 6 } +diff --git a/test/test_core.py b/test/test_core.py +index 2f776068d..8b1933bf9 100644 +--- a/test/test_core.py ++++ b/test/test_core.py +@@ -9699,6 +9699,20 @@ NODEFS is no longer included by default; build with -lnodefs.js + self.emcc_args += ['-mreference-types'] + self.do_core_test('test_externref.c', libraries=['asm.o']) + ++ @parameterized({ ++ '': [False], ++ 'dynlink': [True] ++ }) ++ @requires_node ++ @no_wasm2js('wasm2js does not support reference types') ++ def test_externref2(self, dynlink): ++ self.emcc_args += ['-mreference-types'] ++ self.node_args.append("--experimental-wasm-reftypes") ++ if dynlink: ++ self.set_setting('MAIN_MODULE', 2) ++ self.do_core_test('test_externref2.c') ++ ++ + def test_syscall_intercept(self): + self.do_core_test('test_syscall_intercept.c') + +-- +2.25.1 + diff --git a/src/core/error_handling.h b/src/core/error_handling.h index d1dd88bc333..7a56117bb72 100644 --- a/src/core/error_handling.h +++ b/src/core/error_handling.h @@ -92,7 +92,7 @@ console_error_obj(JsRef obj); #endif // Need an extra layer to expand LOG_EM_JS_ERROR. -#define EM_JS_DEFER(ret, func_name, args, body...) \ +#define EM_JS_MACROS(ret, func_name, args, body...) \ EM_JS(ret, func_name, args, body) #define EM_JS_UNCHECKED(ret, func_name, args, body...) \ @@ -101,7 +101,7 @@ console_error_obj(JsRef obj); #define WARN_UNUSED __attribute__((warn_unused_result)) #define EM_JS_REF(ret, func_name, args, body...) \ - EM_JS_DEFER(ret WARN_UNUSED, func_name, args, { \ + EM_JS_MACROS(ret WARN_UNUSED, func_name, args, { \ try /* intentionally no braces, body already has them */ \ body /* <== body of func */ \ catch (e) { \ @@ -115,7 +115,7 @@ console_error_obj(JsRef obj); }) #define EM_JS_NUM(ret, func_name, args, body...) \ - EM_JS_DEFER(ret WARN_UNUSED, func_name, args, { \ + EM_JS_MACROS(ret WARN_UNUSED, func_name, args, { \ try /* intentionally no braces, body already has them */ \ body /* <== body of func */ \ catch (e) { \ @@ -128,7 +128,7 @@ console_error_obj(JsRef obj); // If there is a Js error, catch it and return false. #define EM_JS_BOOL(ret, func_name, args, body...) \ - EM_JS_DEFER(ret WARN_UNUSED, func_name, args, { \ + EM_JS_MACROS(ret WARN_UNUSED, func_name, args, { \ try /* intentionally no braces, body already has them */ \ body /* <== body of func */ \ catch (e) { \ diff --git a/src/core/hiwire.c b/src/core/hiwire.c index 9b2c3e0c484..b55ddf8313d 100644 --- a/src/core/hiwire.c +++ b/src/core/hiwire.c @@ -6,145 +6,184 @@ #include "hiwire.h" #include "jsmemops.h" +#undef hiwire_incref #define ERROR_REF (0) #define ERROR_NUM (-1) +// For when the return value would be Option +// we use the largest possible immortal reference so that `get_value` on it will +// always raise an error. +const JsRef Js_novalue = ((JsRef)(2147483644)); + #define HIWIRE_INIT_CONSTS() \ - HIWIRE_INIT_CONST(UNDEFINED, undefined, 4); \ - HIWIRE_INIT_CONST(JSNULL, null, 8); \ - HIWIRE_INIT_CONST(TRUE, true, 12); \ - HIWIRE_INIT_CONST(FALSE, false, 16) + HIWIRE_INIT_CONST(undefined) \ + HIWIRE_INIT_CONST(null) \ + HIWIRE_INIT_CONST(true) \ + HIWIRE_INIT_CONST(false) // we use HIWIRE_INIT_CONSTS once in C and once inside JS with different // definitions of HIWIRE_INIT_CONST to ensure everything lines up properly // C definition: -#define HIWIRE_INIT_CONST(hiwire_attr, js_value, id) \ - const JsRef Js_##js_value = ((JsRef)(id)); - +#define HIWIRE_INIT_CONST(js_value) const JsRef Js_##js_value; HIWIRE_INIT_CONSTS(); -// JS definition: #undef HIWIRE_INIT_CONST -#define HIWIRE_INIT_CONST(hiwire_attr, js_value, id) \ - Hiwire.hiwire_attr = DEREF_U8(_Js_##js_value, 0); \ - _hiwire.immortals.push(js_value); \ - _hiwire.obj_to_key.set(js_value, Hiwire.hiwire_attr); -// clang-format off -// JsRefs are: -// * heap if they are odd, -// * immortal if they are divisible by 4 -// * stack references if they are congruent to 2 mod 4 -// -// Note that "NULL" is immortal which is important. -// -// Both immortal and stack indexes are converted to id by bitshifting right by -// two to remove the lower order bits which indicate the reference type. -#define IS_IMMORTAL(idval) (((idval) & 3) === 0) -#define IMMORTAL_REF_TO_INDEX(idval) ((idval) >> 2) -#define IMMORTAL_INDEX_TO_REF(idval) ((idval) << 2) - -#define IS_STACK(idval) (((idval) & 3) === 2) -#define STACK_REF_TO_INDEX(idval) ((idval) >> 2) -#define STACK_INDEX_TO_REF(index) (((index) << 2) | 2) +#define HIWIRE_INIT_CONST(js_value) \ + HEAP32[_Js_##js_value / 4] = _hiwire_intern(js_value); -// For when the return value would be Option -// we use the largest possible immortal reference so that `get_value` on it will -// always raise an error. -const JsRef Js_novalue = ((JsRef)(2147483644)); +EM_JS_NUM(int, hiwire_init_js, (void), { + HIWIRE_INIT_CONSTS(); + // clang-format off + Hiwire.new_value = _hiwire_new; + Hiwire.new_stack = _hiwire_new; + Hiwire.intern_object = _hiwire_intern; + Hiwire.num_keys = _hiwire_num_refs; + Hiwire.stack_length = () => 0; + Hiwire.get_value = _hiwire_get; + Hiwire.incref = (x) => + { + _hiwire_incref(x); + return x; + }; + Hiwire.decref = _hiwire_decref; + Hiwire.pop_value = _hiwire_pop; + // clang-format on + // This is factored out primarily for testing purposes. + Hiwire.isPromise = function(obj) + { + try { + // clang-format off + return !!obj && typeof obj.then === "function"; + // clang-format on + } catch (e) { + return false; + } + }; + + /** + * Turn any ArrayBuffer view or ArrayBuffer into a Uint8Array. + * + * This respects slices: if the ArrayBuffer view is restricted to a slice of + * the backing ArrayBuffer, we return a Uint8Array that shows the same slice. + */ + API.typedArrayAsUint8Array = function(arg) + { + if (ArrayBuffer.isView(arg)) { + return new Uint8Array(arg.buffer, arg.byteOffset, arg.byteLength); + } else { + return new Uint8Array(arg); + } + }; + + { + let dtypes_str = + [ "b", "B", "h", "H", "i", "I", "f", "d" ].join(String.fromCharCode(0), ); + let dtypes_ptr = stringToNewUTF8(dtypes_str); + let dtypes_map = {}; + for (let[idx, val] of Object.entries(dtypes_str)) { + dtypes_map[val] = dtypes_ptr + Number(idx); + } -// Heap slotmap layout macros - -// The idea of a slotmap is that we use a list for storage. we use the empty -// slots in the list to maintain a linked list of freed indices in the same -// place as the values. This means that the next slot we assign is always the -// most recently freed. This leads to the possibility of masking use after free -// errors, since a recently freed reference will likely point to a valid but -// different object. To deal with this, we include as part of the reference a 5 -// bit version for each slot. Only if the same slot is freed and reassigned 32 -// times can the two references be the same. The references look as follows: -// -// [version (5 bits)][index (25 bits)]1 -// -// The highest order 5 bits are the version, the middle 25 bits are the index, -// and the least order bit indicates that it is a heap reference. Since we have -// 25 bits for the index, we can store up to 2^25 = 33,554,432 distinct objects. -// For each slot we associate an 32 bit "info" integer, which we store as part -// of the slotmap state. So references, occupied slot info, and unoccupied slot -// info all look like: -// -// [version (5 bits)][multipurpose field (25 bits)][1 bit] -// -// The least significant bit is set in the references to indicate that they are -// heap references. The least significant bit is set in the info if the slot is -// occupied and unset if the slot is unoccupied. -// -// In a reference, the mulipurpose field contains the slot index. -// -// reference: [version (5 bits)][index (25 bits)]1 -// -// If a slot is unoccupied, the multipurpose field of the slotInfo contains the -// index of the next free slot in the free list or zero if this is the last free -// slot (for this reason, we do not use slot 0). -// -// unoccupied slot: [version (5 bits)][next free index (25 bits)]0 -// -// If a slot is occupied, the multipurpose field of the slotInfo contains a 24 -// bit reference count and an IS_DEDUPLICATED bit. -// -// occupied slot: [version (5 bits)][refcount (24 bits)][IS_DEDUPLICATED bit]1 -// -// References used by JsProxies are deduplicated which makes allocating/freeing -// them more expensive. - - -#define VERSION_SHIFT 26 // 1 occupied bit, 25 bits of index/nextfree/refcount, then the version -#define INDEX_MASK 0x03FFFFFE // mask for index/nextfree -#define REFCOUNT_MASK 0x03FFFFFC // mask for refcount -#define VERSION_OCCUPIED_MASK 0xFc000001 // mask for version and occupied bit -#define VERSION_MASK 0xFc000000 // mask for version -#define OCCUPIED_BIT 1 // occupied bit mask -#define DEDUPLICATED_BIT 2 // is it deduplicated? (used for JsRefs) -#define REFCOUNT_INTERVAL 4 // The refcount starts after OCCUPIED_BIT and DEDUPLICATED_BIT -#define NEW_INFO_FLAGS 5 // REFCOUNT_INTERVAL | OCCUPIED_BIT - -// Check that the constants are internally consistent -_Static_assert(INDEX_MASK == ((1 << VERSION_SHIFT) - 2), "Oops!"); -_Static_assert((REFCOUNT_MASK | DEDUPLICATED_BIT) == INDEX_MASK, "Oops!"); -_Static_assert(VERSION_OCCUPIED_MASK == (~INDEX_MASK), "Oops!"); -_Static_assert(VERSION_OCCUPIED_MASK == (VERSION_MASK | OCCUPIED_BIT), "Oops!"); -_Static_assert(NEW_INFO_FLAGS == (REFCOUNT_INTERVAL | OCCUPIED_BIT), "Oops"); - -#define HEAP_REF_TO_INDEX(ref) (((ref) & INDEX_MASK) >> 1) -#define HEAP_INFO_TO_NEXTFREE(info) HEAP_REF_TO_INDEX(info) - -// The ref is always odd so this is truthy if info is even (meaning unoccupied) -// or info has a different version than ref. Masking removes the bits that form -// the index in the reference and the refcount/next free index in the info. -#define HEAP_REF_IS_OUT_OF_DATE(ref, info) \ - (((ref) ^ (info)) & VERSION_OCCUPIED_MASK) - -#define HEAP_IS_REFCNT_ZERO(info) (!((info) & REFCOUNT_MASK)) -#define HEAP_IS_DEDUPLICATED(info) ((info) & DEDUPLICATED_BIT) - -#define HEAP_INCREF(info) info += REFCOUNT_INTERVAL -#define HEAP_DECREF(info) info -= REFCOUNT_INTERVAL - -// increment the version in info. -#define _NEXT_VERSION(info) (info + (1 << VERSION_SHIFT)) -// assemble version, field, and occupied -#define _NEW_INFO(version, field_and_flag) \ - (((version) & VERSION_MASK) | (field_and_flag)) - -// make a new reference with the same version as info and the given index. -#define HEAP_NEW_REF(index, info) _NEW_INFO(info, ((index) << 1) | 1) -// new occupied info: same version as argument info, NEW_INFO_FLAGS says occupied with refcount 1 -#define HEAP_NEW_OCCUPIED_INFO(info) _NEW_INFO(info, NEW_INFO_FLAGS) -// new unoccupied info, increment version and nextfree in the field -#define FREE_LIST_INFO(info, nextfree) _NEW_INFO(_NEXT_VERSION(info), (nextfree) << 1) + let buffer_datatype_map = new Map([ + [ "Int8Array", [ dtypes_map["b"], 1, true ] ], + [ "Uint8Array", [ dtypes_map["B"], 1, true ] ], + [ "Uint8ClampedArray", [ dtypes_map["B"], 1, true ] ], + [ "Int16Array", [ dtypes_map["h"], 2, true ] ], + [ "Uint16Array", [ dtypes_map["H"], 2, true ] ], + [ "Int32Array", [ dtypes_map["i"], 4, true ] ], + [ "Uint32Array", [ dtypes_map["I"], 4, true ] ], + [ "Float32Array", [ dtypes_map["f"], 4, true ] ], + [ "Float64Array", [ dtypes_map["d"], 8, true ] ], + // These last two default to Uint8. They have checked : false to allow use + // with other types. + [ "DataView", [ dtypes_map["B"], 1, false ] ], + [ "ArrayBuffer", [ dtypes_map["B"], 1, false ] ], + ]); + + /** + * This gets the dtype of a ArrayBuffer or ArrayBuffer view. We return a + * triple: [char* format_ptr, int itemsize, bool checked] If argument is + * untyped (a DataView or ArrayBuffer) then we say it's a Uint8, but we set + * the flag checked to false in that case so we allow assignment to/from + * anything. + * + * This is the API for use from JavaScript, there's also an EM_JS + * hiwire_get_buffer_datatype wrapper for use from C. Used in js2python and + * in jsproxy.c for buffers. + */ + Module.get_buffer_datatype = function(jsobj) + { + return buffer_datatype_map.get(jsobj.constructor.name) || [ 0, 0, false ]; + }; + } + Module.iterObject = function * (object) + { + for (let k in object) { + if (Object.prototype.hasOwnProperty.call(object, k)) { + yield k; + } + } + }; + return 0; +}); + +int +hiwire_init() +{ + return hiwire_init_js(); +} + +HwRef +wrapped_hiwire_incref(HwRef ref) +{ + hiwire_incref(ref); + return ref; +} + +// Called by libhiwire if an invalid ID is dereferenced. +// clang-format off +EM_JS_MACROS(void, hiwire_invalid_ref, (int type, JsRef ref), { + API.fail_test = true; + if (type === HIWIRE_FAIL_GET && !ref) { + // hiwire_get on NULL. + // This might have happened because the error indicator is set. Let's + // check. + if (_PyErr_Occurred()) { + // This will lead to a more helpful error message. + let exc = _wrap_exception(); + let e = Hiwire.pop_value(exc); + console.error( + "Pyodide internal error: Argument to hiwire_get is falsy. This was " + + "probably because the Python error indicator was set when get_value was " + + "called. The Python error that caused this was:", + e + ); + throw e; + } else { + const msg = ( + "Pyodide internal error: Argument to hiwire_get is falsy (but error " + + "indicator is not set)." + ); + console.error(msg); + throw new Error(msg); + } + } + const typestr = { + [HIWIRE_FAIL_GET]: "get", + [HIWIRE_FAIL_INCREF]: "incref", + [HIWIRE_FAIL_DECREF]: "decref", + }[type]; + const msg = ( + `hiwire_${typestr} on invalid reference ${ref}. This is most likely due ` + + "to use after free. It may also be due to memory corruption." + ); + console.error(msg); + throw new Error(msg); +}); // clang-format on JsRef @@ -159,39 +198,6 @@ EM_JS(bool, hiwire_to_bool, (JsRef val), { }); // clang-format on -#ifdef DEBUG_F -bool tracerefs; - -#define TRACEREFS(args...) \ - if (DEREF_U8(_tracerefs, 0)) { \ - console.warn(args); \ - } - -#define DEBUG_INIT(cb) (cb)() - -#else - -#define TRACEREFS(args...) -#define DEBUG_INIT(cb) - -#endif - -#include - -#include "hiwire.js" - -EM_JS(JsRef WARN_UNUSED, hiwire_incref, (JsRef idval), { - return Hiwire.incref(idval); -}); - -EM_JS(JsRef WARN_UNUSED, hiwire_incref_deduplicate, (JsRef idval), { - return Hiwire.incref_deduplicate(idval); -}); - -// clang-format off -EM_JS(void, hiwire_decref, (JsRef idval), { - Hiwire.decref(idval); -}); // clang-format on // clang-format off diff --git a/src/core/hiwire.h b/src/core/hiwire.h index 4babe066912..37dd4a82dd5 100644 --- a/src/core/hiwire.h +++ b/src/core/hiwire.h @@ -1,43 +1,14 @@ -#ifndef HIWIRE_H -#define HIWIRE_H +#ifndef PYODIDE_HIWIRE_H +#define PYODIDE_HIWIRE_H #define PY_SSIZE_T_CLEAN #include "Python.h" +#include "hiwire/hiwire.h" #include "stdalign.h" #include "types.h" #define WARN_UNUSED __attribute__((warn_unused_result)) +#define hiwire_incref wrapped_hiwire_incref -/** - * hiwire: A super-simple framework for converting values between C and - * JavaScript. - * - * Arbitrary JavaScript objects are referenced from C using an opaque JsRef - * value. - * - * JavaScript objects passed to the C side must be manually reference-counted. - * Use `hiwire_incref` if you plan to store the object on the C side. Use - * `hiwire_decref` when done. Internally, the objects are stored in a global - * object. There may be one or more keys pointing to the same object. - */ - -// JsRef is a NewType of int. We need -// -// alignof(JsRef) = alignof(int) = 4 and -// sizeof(JsRef) = sizeof(int) = 4 -// -// To be future proof, we have _Static_asserts for this. -// I also added -// -Werror=int-conversion -Werror=incompatible-pointer-types -// to the compile flags, to ensure that no implicit casts will happen between -// JsRef and any other type. -struct _JsRefStruct -{}; - -typedef struct _JsRefStruct* JsRef; - -_Static_assert(alignof(JsRef) == alignof(int), - "JsRef should have the same alignment as int."); -_Static_assert(sizeof(JsRef) == sizeof(int), - "JsRef should have the same size as int."); +typedef HwRef JsRef; // Error handling will want to see JsRef. #include "error_handling.h" diff --git a/src/core/hiwire.js b/src/core/hiwire.js deleted file mode 100644 index 84f851be64d..00000000000 --- a/src/core/hiwire.js +++ /dev/null @@ -1,279 +0,0 @@ -JS_FILE(hiwire_init, () => { - 0, 0; /* Magic, see include_js_file.h */ - - // See the macros and extensive comment in hiwire.c for more info. - const _hiwire = { - // next free = 0 means that all slots are full, so we don't use slot 0. - objects: [null], - slotInfo: new Uint32Array(0), - slotInfoSize: 0, - freeHead: 1, - numKeys: 0, - // The reverse of the object maps, needed to deduplicate keys so that key - // equality is object identity. - obj_to_key: new Map(), - stack: [], - // Actual 0 is reserved for NULL so we have to leave a space for it. - immortals: [null], - }; - HIWIRE_INIT_CONSTS(); - - DEBUG_INIT(() => { - Hiwire._hiwire = _hiwire; - }); - Hiwire.new_stack = function (jsval) { - const idx = _hiwire.stack.push(jsval) - 1; - TRACEREFS("hw.new_stack", STACK_INDEX_TO_REF(idx), idx, jsval); - return STACK_INDEX_TO_REF(idx); - }; - - Hiwire.new_value = function (jsval) { - const index = _hiwire.freeHead; - const info = _hiwire.slotInfo[index]; - _hiwire.objects[index] = jsval; - // if nextfree is 0 then we'll add a new entry to the list next - _hiwire.freeHead = HEAP_INFO_TO_NEXTFREE(info) || _hiwire.objects.length; - if (index >= _hiwire.slotInfoSize) { - // we ran out of space in the slotInfo map, reallocate. - _hiwire.slotInfoSize += 1024; - const old = _hiwire.slotInfo; - _hiwire.slotInfo = new Uint32Array(_hiwire.slotInfoSize); - _hiwire.slotInfo.set(old); - } - _hiwire.slotInfo[index] = HEAP_NEW_OCCUPIED_INFO(info); - const idval = HEAP_NEW_REF(index, info); - _hiwire.numKeys++; - TRACEREFS("hw.new_value", index, idval, jsval); - return idval; - }; - - /** - * Increase the reference count on an object and return a JsRef which is unique - * to the object. - * - * I.e., if `Hiwire.get_value(id1) === Hiwire.get_value(id2)` then - * hiwire_incref_deduplicate(id1) == hiwire_incref_deduplicate(id2). - * - * This is used for the id for JsProxies so that equality checks work correctly. - */ - Hiwire.incref_deduplicate = function (idval) { - const obj = Hiwire.get_value(idval); - let result = _hiwire.obj_to_key.get(obj); - if (result) { - if (!IS_IMMORTAL(result)) { - HEAP_INCREF(_hiwire.slotInfo[HEAP_REF_TO_INDEX(result)]); - } - return result; - } - // Either not present or key is out of date. - // Use incref to force possible stack reference to heap reference. - result = Hiwire.incref(idval); - _hiwire.obj_to_key.set(obj, result); - // Record that we need to remove this entry from obj_to_key when the - // reference is freed. (Touching a map is expensive, avoid if possible!) - _hiwire.slotInfo[HEAP_REF_TO_INDEX(result)] |= DEDUPLICATED_BIT; - return result; - }; - - Hiwire.intern_object = function (obj) { - const id = IMMORTAL_INDEX_TO_REF(_hiwire.immortals.push(obj) - 1); - _hiwire.obj_to_key.set(obj, id); - return id; - }; - - // for testing purposes. - Hiwire.num_keys = function () { - return _hiwire.numKeys; - }; - - Hiwire.stack_length = () => _hiwire.stack.length; - - Hiwire.get_value = function (idval) { - if (!idval) { - API.fail_test = true; - // This might have happened because the error indicator is set. Let's - // check. - if (_PyErr_Occurred()) { - // This will lead to a more helpful error message. - let exc = _wrap_exception(); - let e = Hiwire.pop_value(exc); - console.error( - `Pyodide internal error: Argument '${idval}' to hiwire.get_value is falsy. ` + - "This was probably because the Python error indicator was set when get_value was called. " + - "The Python error that caused this was:", - e, - ); - throw e; - } else { - const msg = - `Pyodide internal error: Argument '${idval}' to hiwire.get_value is falsy` + - " (but error indicator is not set)."; - console.error(msg); - throw new Error(msg); - } - } - if (IS_IMMORTAL(idval)) { - return _hiwire.immortals[IMMORTAL_REF_TO_INDEX(idval)]; - } - if (IS_STACK(idval)) { - const idx = STACK_REF_TO_INDEX(idval); - if (idx >= _hiwire.stack.length) { - API.fail_test = true; - const msg = `Pyodide internal error : Invalid stack reference handling`; - console.error(msg); - throw new Error(msg); - } - return _hiwire.stack[idx]; - } - const index = HEAP_REF_TO_INDEX(idval); - const info = _hiwire.slotInfo[index]; - if (HEAP_REF_IS_OUT_OF_DATE(idval, info)) { - API.fail_test = true; - console.error(`Pyodide internal error: Undefined id ${idval}`); - throw new Error(`Undefined id ${idval}`); - } - return _hiwire.objects[index]; - }; - - Hiwire.decref = function (idval) { - if (IS_IMMORTAL(idval)) { - return; - } - if (IS_STACK(idval)) { - const idx = STACK_REF_TO_INDEX(idval); - TRACEREFS("hw.decref.stack", idval, idx, _hiwire.stack[idx]); - if (idx + 1 !== _hiwire.stack.length) { - API.fail_test = true; - const msg = `Pyodide internal error: Invalid stack reference handling: decref index ${idx} stack size ${_hiwire.stack.length}`; - console.error(msg); - throw new Error(msg); - } - _hiwire.stack.pop(); - return; - } - // heap reference - const index = HEAP_REF_TO_INDEX(idval); - TRACEREFS("hw.decref", index, idval, _hiwire.objects[index]); - let info = _hiwire.slotInfo[index]; - if (HEAP_REF_IS_OUT_OF_DATE(idval, info)) { - API.fail_test = true; - console.error(`Pyodide internal error: Undefined id ${idval}`); - throw new Error(`Undefined id ${idval}`); - } - HEAP_DECREF(info); - if (HEAP_IS_REFCNT_ZERO(info)) { - if (HEAP_IS_DEDUPLICATED(info)) { - _hiwire.obj_to_key.delete(_hiwire.objects[index]); - } - // Note: it's 100x faster to set the value to `undefined` than to `delete` it. - _hiwire.objects[index] = undefined; - _hiwire.numKeys--; - info = FREE_LIST_INFO(info, _hiwire.freeHead); - _hiwire.freeHead = index; - } - _hiwire.slotInfo[index] = info; - }; - - Hiwire.incref = function (idval) { - if (IS_IMMORTAL(idval)) { - return idval; - } - if (IS_STACK(idval)) { - const idx = STACK_REF_TO_INDEX(idx); - TRACEREFS("hw.incref.stack", idval, idx, _hiwire.stack[idx]); - // stack reference ==> move to heap - return Hiwire.new_value(_hiwire.stack[idx]); - } - // heap reference - const index = HEAP_REF_TO_INDEX(idval); - TRACEREFS("hw.incref", index, idval, _hiwire.objects[index]); - const info = _hiwire.slotInfo[index]; - if (HEAP_REF_IS_OUT_OF_DATE(idval, info)) { - API.fail_test = true; - console.error(`Pyodide internal error: Undefined id ${idval}`); - throw new Error(`Undefined id ${idval}`); - } - HEAP_INCREF(_hiwire.slotInfo[index]); - return idval; - }; - - Hiwire.pop_value = function (idval) { - let result = Hiwire.get_value(idval); - Hiwire.decref(idval); - return result; - }; - - // This is factored out primarily for testing purposes. - Hiwire.isPromise = function (obj) { - try { - return !!obj && typeof obj.then === "function"; - } catch (e) { - return false; - } - }; - - /** - * Turn any ArrayBuffer view or ArrayBuffer into a Uint8Array. - * - * This respects slices: if the ArrayBuffer view is restricted to a slice of - * the backing ArrayBuffer, we return a Uint8Array that shows the same slice. - */ - API.typedArrayAsUint8Array = function (arg) { - if (ArrayBuffer.isView(arg)) { - return new Uint8Array(arg.buffer, arg.byteOffset, arg.byteLength); - } else { - return new Uint8Array(arg); - } - }; - - { - let dtypes_str = ["b", "B", "h", "H", "i", "I", "f", "d"].join( - String.fromCharCode(0), - ); - let dtypes_ptr = stringToNewUTF8(dtypes_str); - let dtypes_map = {}; - for (let [idx, val] of Object.entries(dtypes_str)) { - dtypes_map[val] = dtypes_ptr + Number(idx); - } - - let buffer_datatype_map = new Map([ - ["Int8Array", [dtypes_map["b"], 1, true]], - ["Uint8Array", [dtypes_map["B"], 1, true]], - ["Uint8ClampedArray", [dtypes_map["B"], 1, true]], - ["Int16Array", [dtypes_map["h"], 2, true]], - ["Uint16Array", [dtypes_map["H"], 2, true]], - ["Int32Array", [dtypes_map["i"], 4, true]], - ["Uint32Array", [dtypes_map["I"], 4, true]], - ["Float32Array", [dtypes_map["f"], 4, true]], - ["Float64Array", [dtypes_map["d"], 8, true]], - // These last two default to Uint8. They have checked : false to allow use - // with other types. - ["DataView", [dtypes_map["B"], 1, false]], - ["ArrayBuffer", [dtypes_map["B"], 1, false]], - ]); - - /** - * This gets the dtype of a ArrayBuffer or ArrayBuffer view. We return a - * triple: [char* format_ptr, int itemsize, bool checked] If argument is - * untyped (a DataView or ArrayBuffer) then we say it's a Uint8, but we set - * the flag checked to false in that case so we allow assignment to/from - * anything. - * - * This is the API for use from JavaScript, there's also an EM_JS - * hiwire_get_buffer_datatype wrapper for use from C. Used in js2python and - * in jsproxy.c for buffers. - */ - Module.get_buffer_datatype = function (jsobj) { - return buffer_datatype_map.get(jsobj.constructor.name) || [0, 0, false]; - }; - } - - Module.iterObject = function* (object) { - for (let k in object) { - if (Object.prototype.hasOwnProperty.call(object, k)) { - yield k; - } - } - }; - return 0; -}); diff --git a/src/tests/test_pyodide.py b/src/tests/test_pyodide.py index ad8435be25c..4a401e34d4d 100644 --- a/src/tests/test_pyodide.py +++ b/src/tests/test_pyodide.py @@ -1731,3 +1731,55 @@ def f2(): assert dedent("\n".join(msg.splitlines()[1:-1])) == "\n".join( expected.splitlines()[2:] ) + + +@run_in_pyodide +def test_hiwire_invalid_ref(selenium): + import pytest + + import pyodide_js + from pyodide.code import run_js + from pyodide.ffi import JsException + + _hiwire_get = pyodide_js._module._hiwire_get + _hiwire_incref = pyodide_js._module._hiwire_incref + _hiwire_decref = pyodide_js._module._hiwire_decref + _api = pyodide_js._api + + _hiwire_incref(0) + assert not _api.fail_test + _hiwire_decref(0) + assert not _api.fail_test + expected = r"Pyodide internal error: Argument to hiwire_get is falsy \(but error indicator is not set\)\." + with pytest.raises(JsException, match=expected): + _hiwire_get(0) + assert _api.fail_test + _api.fail_test = False + + with pytest.raises(AssertionError, match="This is a message"): + run_js( + """ + const msgptr = pyodide._module.stringToNewUTF8("This is a message"); + const AssertionError = pyodide._module.HEAP32[pyodide._module._PyExc_AssertionError/4]; + pyodide._module._PyErr_SetString(AssertionError, msgptr); + pyodide._module._free(msgptr); + try { + pyodide._module._hiwire_get(0); + } finally { + pyodide._module._PyErr_Clear(); + } + """ + ) + msg = "hiwire_{} on invalid reference 77. This is most likely due to use after free. It may also be due to memory corruption." + with pytest.raises(JsException, match=msg.format("get")): + _hiwire_get(77) + assert _api.fail_test + _api.fail_test = False + with pytest.raises(JsException, match=msg.format("incref")): + _hiwire_incref(77) + assert _api.fail_test + _api.fail_test = False + with pytest.raises(JsException, match=msg.format("decref")): + _hiwire_decref(77) + assert _api.fail_test + _api.fail_test = False