diff --git a/.circleci/config.yml b/.circleci/config.yml index c0536063facaa..61d7cb912a278 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -419,12 +419,14 @@ jobs: - run: make -C site text - run: tools/maint/check_emcc_help_text.py - run: make -C site html - flake8: + ruff: executor: bionic steps: - checkout - pip-install - - run: python3 -m flake8 --show-source --statistics + - run: ruff check + # TODO (cclauss): When ruff supports rule E303 without --preview, remove following line + - run: ruff check --preview --select=E303 mypy: executor: bionic steps: @@ -970,7 +972,7 @@ jobs: workflows: build-test: jobs: - - flake8 + - ruff - mypy - eslint - build-docs diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index e8ca40a3d059c..0000000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -source = . -omit = ./test/* - ./third_party/* - ./tools/emcoverage.py - test.py - diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 865ca1e004956..0000000000000 --- a/.flake8 +++ /dev/null @@ -1,19 +0,0 @@ -[flake8] -ignore = E111,E114,E501,E261,E266,E121,E402,E241,W504,E741,B011,B023,U101 -exclude = - ./node_modules/, # third-party code - ./third_party/, # third-party code - ./tools/filelock.py, # third-party code - ./tools/scons/, # third-party code - ./test/third_party/, # third-party code - ./site/source/conf.py, - ./site/source/_themes/, - ./system/lib/, # system libraries - ./cache/, # download/built content - .git -# The ports plugins have a lot of unused imports because -# they need to implement the specific plugin APIs -per-file-ignores = - ./tools/ports/*.py: U100 - ./test/*.py: U100 - ./tools/toolchain_profiler.py: U100 diff --git a/.github/workflows/archive.yml b/.github/workflows/archive.yml deleted file mode 100644 index 480bf446da5c9..0000000000000 --- a/.github/workflows/archive.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: CI - -on: - create: - tags: - push: - branches: - - main - pull_request: - -permissions: - contents: read - -jobs: - archive: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - - name: make dist - run: | - make dist - version=`cat emscripten-version.txt | sed s/\"//g` - echo "VERSION=$version" >> $GITHUB_ENV - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 - with: - name: emscripten-${{ env.VERSION }} - path: emscripten-${{ env.VERSION }}.tar.bz2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000000..f6b1abea01ce7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,86 @@ +name: CI + +on: + create: + tags: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +jobs: + archive: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - name: make dist + run: | + make dist + version=`cat emscripten-version.txt | sed s/\"//g` + echo "VERSION=$version" >> $GITHUB_ENV + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: emscripten-${{ env.VERSION }} + path: emscripten-${{ env.VERSION }}.tar.bz2 + + check-expectations: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # We want access to other branches, specifically `main` + - name: pip install + run: | + which python3 + python3 --version + python3 -m pip install -r requirements-dev.txt + - name: Install emsdk + run: | + EM_CONFIG=$HOME/emsdk/.emscripten + echo $EM_CONFIG + echo "EM_CONFIG=$EM_CONFIG" >> $GITHUB_ENV + curl -# -L -o ~/emsdk-main.tar.gz https://github.com/emscripten-core/emsdk/archive/main.tar.gz + tar -C ~ -xf ~/emsdk-main.tar.gz + mv ~/emsdk-main ~/emsdk + cd ~/emsdk + ./emsdk install tot + ./emsdk activate tot + echo "JS_ENGINES = [NODE_JS]" >> $EM_CONFIG + echo "final config:" + cat $EM_CONFIG + - name: Check test expectations on target branch + run: | + echo "Checking out ${{ github.base_ref }}" + git checkout ${{ github.base_ref }} + git rev-parse HEAD + # Hack to honor changes to rebaseline_tests.py in the current PR + git checkout - ./tools/maint/rebaseline_tests.py + ./bootstrap + if ! ./tools/maint/rebaseline_tests.py --check-only; then + echo "Test expectations are out-of-date on the target branch." + echo "You can run './tools/maint/rebaseline_tests.py --new-branch'" + echo "and use it to create a seperate PR." + echo "-- This failure is only a warning and can be ignored" + exit 1 + fi + - name: Check test expectations on PR branch + run: | + echo "Checking out ${{ github.ref }} (${{ github.sha }})" + # For some reason we cannot pass ${{ github.ref }} direclty to git + # since it doesn't recognise it. + git checkout ${{ github.sha }} + git rev-parse HEAD + ./bootstrap + if ! ./tools/maint/rebaseline_tests.py --check-only --clear-cache; then + echo "Test expectations are out-of-date on the PR branch." + echo "You can run './tools/maint/rebaseline_tests.py' to" + echo "create a commit updating the expectations." + echo "Be sure to have `emsdk install tot` first." + echo "-- This failure is only a warning and can be ignored" + exit 1 + fi diff --git a/.github/workflows/tag-release.yml b/.github/workflows/tag-release.yml index fd44eb7c80551..4618416d87ef8 100644 --- a/.github/workflows/tag-release.yml +++ b/.github/workflows/tag-release.yml @@ -9,42 +9,44 @@ on: type: string jobs: - create-release: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - # Updates changelog and writes the release version into the environment - - name: Update Changelog - run: python3 tools/maint/create_release.py --action - - name: Create Changelog PR - id: cpr - uses: peter-evans/create-pull-request@v6 - with: - token: ${{ secrets.EMSCRIPTEN_BOT_TOKEN }} - title: Mark ${{ env.RELEASE_VERSION }} as released - team-reviewers: release-reviewers - delete-branch: true - - name: Enable auto-merge - run: gh pr merge --squash --auto "${{ steps.cpr.outputs.pull-request-number }}" - env: - GH_TOKEN: ${{ secrets.EMSCRIPTEN_BOT_TOKEN }} - - name: Tag release sha - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.EMSCRIPTEN_BOT_TOKEN }} - script: | - const tag_sha = '${{ inputs.release-sha }}'; - const release_version = '${{ env.RELEASE_VERSION }}'; - console.log(`Version ${release_version} at SHA ${tag_sha}`); - const regex = /^[0-9]+.[0-9]+.[0-9]+$/; - const match = release_version.match(regex); - if (!match) { - throw new Error('Malformed release version'); - } - await github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: `refs/tags/${release_version}`, - sha: tag_sha - }); + create-release: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + # Updates changelog and writes the release version into the environment + - name: Update Changelog + run: python3 tools/maint/create_release.py --action + - name: Create Changelog PR + id: cpr + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.EMSCRIPTEN_BOT_TOKEN }} + title: Mark ${{ env.RELEASE_VERSION }} as released + body: Update changelog and emscripten-version.txt [ci skip] + team-reviewers: release-reviewers + delete-branch: true + - name: Enable auto-merge + run: gh pr merge --squash --auto "${{ steps.cpr.outputs.pull-request-number }}" + env: + GH_TOKEN: ${{ secrets.EMSCRIPTEN_BOT_TOKEN }} + - name: Tag release sha + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.EMSCRIPTEN_BOT_TOKEN }} + script: | + const tag_sha = '${{ inputs.release-sha }}'; + const release_version = '${{ env.RELEASE_VERSION }}'; + console.log(`Version ${release_version} at SHA ${tag_sha}`); + const regex = /^[0-9]+.[0-9]+.[0-9]+$/; + const match = release_version.match(regex); + if (!match) { + throw new Error('Malformed release version'); + } + await github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/${release_version}`, + sha: tag_sha + }); + diff --git a/.mypy.ini b/.mypy.ini deleted file mode 100644 index 9bbc785692470..0000000000000 --- a/.mypy.ini +++ /dev/null @@ -1,16 +0,0 @@ -[mypy] -mypy_path = third_party/,third_party/ply,third_party/websockify -files = . -exclude = (?x)( - cache | - third_party | - conf\.py | - emrun\.py | - site/source/_themes/ | - tools/scons/site_scons/site_tools/emscripten/__init__\.py | - site/source/get_wiki\.py | - test/parse_benchmark_output\.py - ) - -[mypy-tools.create_dom_pk_codes,tools.webidl_binder,tools.toolchain_profiler,tools.filelock,tools.find_bigvars,leb128,ply.*] -ignore_errors = True diff --git a/ChangeLog.md b/ChangeLog.md index edefea9330cfc..785f80d53087d 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -4,7 +4,7 @@ Note that version numbers do not necessarily reflect the amount of changes between versions. A version number reflects a release that is known to pass all tests, and versions may be tagged more or less frequently at different times. -Note that there is *no* ABI compatibility guarantee between versions - the ABI +nNote that there is *no* ABI compatibility guarantee between versions - the ABI may change, so that we can keep improving and optimizing it. The compiler will automatically invalidate system caches when the version number updates, so that libc etc. are rebuilt for you. You should also rebuild object files and @@ -18,12 +18,58 @@ to browse the changes between the tags. See docs/process.md for more on how version tagging works. -3.1.74 (in development) +3.1.75 (in development) ----------------------- +- compiler-rt was updated to LLVM 19.1.4. (#22937) +- The Wasm nontrapping-fptoint feature has been enabled by default. clang will + generate nontrapping (saturating) float-to-int conversion instructions for + C typecasts. This should have no effect on programs that do not have + undefined behavior but if the casted floating-point value is outside the range + of the target integer type, the result will be a number of the max or min value + instead of a trap. This also results in a small code size improvement because + of details of the LLVM IR semantics. This feature can be disabled in clang with + the -mno-nontrapping-fptoint flag. (#23007) +- The `WASM_BIGINT` feature has been enabled by default. This has the effect that + Wasm i64 values are passed and returned between Wasm and JS as BigInt values + rather than being split by Binaryen into pairs of Numbers. (#22993) +- When using `-sMODULARIZE` we now assert if the factory function is called with + the JS `new` keyword. e.g. `a = new Module()` rather than `b = Module()`. + This paves the way for marking the function as `async` which does not allow + `new` to be used. This usage of `new` here was never documented and is + considered and antipattern. (#23210) +- `PATH.basename()` no longer calls `PATH.normalize()`, so that + `PATH.basename("a/.")` returns `"."` instead of `"a"` and + `PATH.basename("a/b/..")` returns `".."` instead of `"a"`. This is in line with + the behaviour of both node and coreutils, and is already the case when using + NODERAWFS". (#23180) +- The factory function exposed in `-sMODULARIZE` mode is now marked as `async` + when `WASM_ASYNC_COMPILATION` is enabled (the default). This allows us to use + `await` during module creation. One side effect of this is that code in + `--post-js` files will now be delayed until after module creation and after + `main` runs. This matches the existing behaviour when using sync instantation + (`-sWASM_ASYNC_COMPILATION=0`) but is an observable difference. (#23157) + +3.1.74 - 12/14/24 +----------------- +- The file system was updated to independently track atime, mtime and ctime + instead of using the same time for all three. (#22998) +- Emscripten-generated code will now use async/await internally when loading + the Wasm module. This will be lowered away by babel when targeting older + browsers. (#23068) +- Due to the discontinued support for invalid specializations of + `std::basic_string` (https://github.com/llvm/llvm-project/pull/72694), the + support for `std::basic_string` was removed from embind. + (#23070) +- The minimum supported versions of browser engines that we support were updated + to versions that support Promise, Fetch and Object.asign APIs, allowing the + polyfills for these to be removed. Chrome 32 -> 45, Firefox 34 -> 40, Safari + 9.0 -> 10.1. These browser engines version are all over 8 years old now. + (#23077, #23118) 3.1.73 - 11/28/24 ----------------- -- libunwind was updated to LLVM 19.1.4. (#22394) +- libunwind was updated to LLVM 19.1.4. (#22934) +- mimalloc was updated to 2.1.7. (#21548) 3.1.72 - 11/19/24 ----------------- diff --git a/docs/emcc.txt b/docs/emcc.txt index 560542e38a35f..10b216fde55f2 100644 --- a/docs/emcc.txt +++ b/docs/emcc.txt @@ -252,7 +252,7 @@ Options that are modified or new in *emcc* are listed below: "--closure 0|1|2" [link] Runs the *Closure Compiler*. Possible values are: - * "0": No closure compiler (default in "-O2" and below). + * "0": No closure compiler (default). * "1": Run closure compiler. This greatly reduces the size of the support JavaScript code (everything but the WebAssembly or @@ -277,9 +277,6 @@ Options that are modified or new in *emcc* are listed below: before the closure-compiled code runs, because then it will reuse that variable. - * Closure is only run if JavaScript opts are being done ("-O2" or - above). - "--closure-args=" [link] Pass arguments to the *Closure compiler*. This is an alternative to "EMCC_CLOSURE_ARGS". diff --git a/docs/process.md b/docs/process.md index 0c3b23b1062d7..f1bf04a714a11 100644 --- a/docs/process.md +++ b/docs/process.md @@ -55,8 +55,8 @@ pre-processor. See [`.clang-format`][clang-format] for more details. ### Python Code We generally follow the pep8 standard with the major exception that we use 2 -spaces for indentation. `flake8` is run on all PRs to ensure that python code -conforms to this style. See [`.flake8`][flake8] for more details. +spaces for indentation. `ruff` is run on all PRs to ensure that Python code +conforms to this style. See [`pyproject.toml`][pyproject.toml] for more details. #### Static Type Checking @@ -304,7 +304,7 @@ To update our libraries to a newer musl release: [emsdk_tags]: https://github.com/emscripten-core/emsdk/tags [emscripten_tags]: https://github.com/emscripten-core/emscripten/tags [clang-format]: https://github.com/emscripten-core/emscripten/blob/main/.clang-format -[flake8]: https://github.com/emscripten-core/emscripten/blob/main/.flake8 +[pyproject.toml]: https://github.com/emscripten-core/emscripten/blob/main/pyproject.toml [mypy]: https://github.com/emscripten-core/emscripten/blob/main/.mypy [update_docs]: https://github.com/emscripten-core/emscripten/blob/main/tools/maint/update_docs.py [llvm_repo]: https://github.com/llvm/llvm-project diff --git a/em-config.py b/em-config.py index 2708d18ca6dd2..bcbd9abe1cf8d 100755 --- a/em-config.py +++ b/em-config.py @@ -24,7 +24,7 @@ def main(): not re.match(r"^[\w\W_][\w\W_\d]*$", sys.argv[1]) or \ not hasattr(config, sys.argv[1]): print('Usage: em-config VAR_NAME', file=sys.stderr) - exit(1) + sys.exit(1) print(getattr(config, sys.argv[1])) return 0 diff --git a/emcc.py b/emcc.py index dfe088e7b75c0..1ae6a24f94477 100644 --- a/emcc.py +++ b/emcc.py @@ -391,8 +391,7 @@ def get_clang_flags(user_args): # Bulk memory may be enabled via threads or directly via -s. if not settings.BULK_MEMORY: flags.append('-mno-bulk-memory') - if '-mnontrapping-fptoint' not in user_args and '-mno-nontrapping-fptoint' not in user_args: - flags.append('-mno-nontrapping-fptoint') + flags.append('-mno-bulk-memory-opt') if settings.RELOCATABLE and '-fPIC' not in user_args: flags.append('-fPIC') @@ -1023,7 +1022,7 @@ def get_clang_command_asm(): if state.mode == Mode.PCH: inputs = [i[1] for i in input_files] for header in inputs: - if not shared.suffix(header) in HEADER_ENDINGS: + if shared.suffix(header) not in HEADER_ENDINGS: exit_with_error(f'cannot mix precompiled headers with non-header inputs: {inputs} : {header}') cmd = get_clang_command() + inputs if options.output_file: @@ -1127,7 +1126,15 @@ def version_string(): return f'emcc (Emscripten gcc/clang-like replacement + linker emulating GNU ld) {utils.EMSCRIPTEN_VERSION}{revision_suffix}' -def parse_args(newargs): +def parse_args(newargs): # noqa: C901, PLR0912, PLR0915 + """Future modifications should consider refactoring to reduce complexity. + + * The McCabe cyclomatiic complexity is currently 117 vs 10 recommended. + * There are currently 115 branches vs 12 recommended. + * There are currently 302 statements vs 50 recommended. + + To revalidate these numbers, run `ruff check --select=C901,PLR091`. + """ options = EmccOptions() settings_changes = [] user_js_defines = [] @@ -1259,6 +1266,13 @@ def consume_arg_file(): if is_int(requested_level): # the -gX value is the debug level (-g1, -g2, etc.) settings.DEBUG_LEVEL = validate_arg_level(requested_level, 4, 'invalid debug level: ' + arg) + if settings.DEBUG_LEVEL == 0: + # Set these explicitly so -g0 overrides previous -g on the cmdline + settings.GENERATE_DWARF = 0 + settings.GENERATE_SOURCE_MAP = 0 + settings.EMIT_NAME_SECTION = 0 + elif settings.DEBUG_LEVEL > 1: + settings.EMIT_NAME_SECTION = 1 # if we don't need to preserve LLVM debug info, do not keep this flag # for clang if settings.DEBUG_LEVEL < 3: @@ -1289,17 +1303,20 @@ def consume_arg_file(): settings.GENERATE_DWARF = 1 elif requested_level == 'source-map': settings.GENERATE_SOURCE_MAP = 1 + settings.EMIT_NAME_SECTION = 1 newargs[i] = '-g' else: # Other non-integer levels (e.g. -gline-tables-only or -gdwarf-5) are # usually clang flags that emit DWARF. So we pass them through to # clang and make the emscripten code treat it like any other DWARF. settings.GENERATE_DWARF = 1 + settings.EMIT_NAME_SECTION = 1 # In all cases set the emscripten debug level to 3 so that we do not # strip during link (during compile, this does not make a difference). settings.DEBUG_LEVEL = 3 elif check_flag('-profiling') or check_flag('--profiling'): settings.DEBUG_LEVEL = max(settings.DEBUG_LEVEL, 2) + settings.EMIT_NAME_SECTION = 1 elif check_flag('-profiling-funcs') or check_flag('--profiling-funcs'): settings.EMIT_NAME_SECTION = 1 elif newargs[i] == '--tracing' or newargs[i] == '--memoryprofiler': @@ -1416,6 +1433,12 @@ def consume_arg_file(): override=True) elif arg == '-mno-sign-ext': feature_matrix.disable_feature(feature_matrix.Feature.SIGN_EXT) + elif arg == '-mnontrappting-fptoint': + feature_matrix.enable_feature(feature_matrix.Feature.NON_TRAPPING_FPTOINT, + '-mnontrapping-fptoint', + override=True) + elif arg == '-mno-nontrapping-fptoint': + feature_matrix.disable_feature(feature_matrix.Feature.NON_TRAPPING_FPTOINT) elif arg == '-fexceptions': # TODO Currently -fexceptions only means Emscripten EH. Switch to wasm # exception handling by default when -fexceptions is given when wasm diff --git a/emrun.py b/emrun.py index 90812813b4d48..0d33218c2d747 100644 --- a/emrun.py +++ b/emrun.py @@ -1583,7 +1583,15 @@ def parse_args(args): return parser.parse_args(args) -def run(args): +def run(args): # noqa: C901, PLR0912, PLR0915 + """Future modifications should consider refactoring to reduce complexity. + + * The McCabe cyclomatiic complexity is currently 74 vs 10 recommended. + * There are currently 86 branches vs 12 recommended. + * There are currently 202 statements vs 50 recommended. + + To revalidate these numbers, run `ruff check --select=C901,PLR091`. + """ global browser_process, browser_exe, processname_killed_atexit, emrun_options, emrun_not_enabled_nag_printed options = emrun_options = parse_args(args) diff --git a/emscripten-version.txt b/emscripten-version.txt index f26702764ab34..00f17b8f33993 100644 --- a/emscripten-version.txt +++ b/emscripten-version.txt @@ -1 +1 @@ -3.1.74-git +3.1.75-git diff --git a/emsymbolizer.py b/emsymbolizer.py index 1a9cf27f25d56..7ba3b951c852b 100755 --- a/emsymbolizer.py +++ b/emsymbolizer.py @@ -121,10 +121,8 @@ def parse(self, filename): self.version = source_map_json['version'] self.sources = source_map_json['sources'] - vlq_map = {} chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=' - for i, c in enumerate(chars): - vlq_map[c] = i + vlq_map = {c: i for i, c in enumerate(chars)} def decodeVLQ(string): result = [] diff --git a/package-lock.json b/package-lock.json index cec4f1c933bbb..464d0293f217c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,7 @@ "requires": true, "packages": { "": { + "name": "emscripten", "dependencies": { "@babel/cli": "^7.25.6", "@babel/core": "^7.25.2", @@ -2600,9 +2601,9 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "dependencies": { "path-key": "^3.1.0", @@ -7029,9 +7030,9 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" }, "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dev": true, "requires": { "path-key": "^3.1.0", diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000..874462532f8e3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,99 @@ +[tool.ruff] +exclude = [ + "./cache/", + "./node_modules/", + "./site/source/_themes/", + "./site/source/conf.py", + "./system/lib/", + "./test/third_party/", + "./third_party/", + "./tools/filelock.py", + "./tools/scons/", + ".git", +] + +lint.select = [ + "ARG", + "ASYNC", + "B", + "C90", + "E", + "F", + "PERF", + "PL", + "W", + "YTT", +] + +lint.ignore = [ + "ARG001", + "ARG002", + "ARG005", + "B006", + "B011", + "B018", + "B023", + "B026", + "B904", + "E402", + "E501", + "E721", + "E741", + "PERF203", + "PERF401", + "PLR1704", + "PLR1714", + "PLR5501", + "PLW0602", + "PLW0603", + "PLW1510", + "PLW2901", +] +lint.per-file-ignores."emrun.py" = [ "PLE0704" ] +lint.mccabe.max-complexity = 48 # Recommended: 10 +lint.pylint.allow-magic-value-types = [ + "bytes", + "float", + "int", + "str", +] +lint.pylint.max-args = 15 # Recommended: 5 +lint.pylint.max-branches = 49 # Recommended: 12 +lint.pylint.max-returns = 16 # Recommended: 6 +lint.pylint.max-statements = 142 # Recommended: 50 + +[tool.coverage.run] +source = [ "." ] +omit = [ + "./test/*", + "./third_party/*", + "./tools/emcoverage.py", + "test.py", +] + +[tool.mypy] +mypy_path = "third_party/,third_party/ply,third_party/websockify" +files = [ "." ] +exclude = ''' +(?x)( +cache | +third_party | +conf\.py | +emrun\.py | +site/source/_themes/ | +tools/scons/site_scons/site_tools/emscripten/__init__\.py | +site/source/get_wiki\.py | +test/parse_benchmark_output\.py +)''' + +[[tool.mypy.overrides]] +module = [ + "tools.create_dom_pk_codes", + "tools.webidl_binder", + "tools.toolchain_profiler", + "tools.filelock", + "tools.find_bigvars", + "leb128", + "ply.*", +] +ignore_errors = true diff --git a/requirements-dev.txt b/requirements-dev.txt index 4e0ddc450aecd..84c824c5ee1b6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,20 +1,15 @@ # TODO(sbc): switch to using Pipenv since it seems like that way to go # these day managing python deps. -# These requirements are only needed for developers who want to run flake8 on +# These requirements are only needed for developers who want to run ruff on # the codebase and generate docs using Sphinx, not for users of emscripten. # Install with `pip3 install -r requirements-dev.txt` -flake8==5.0.4 -flake8-bugbear==22.9.23 -flake8-unused-arguments==0.0.11 -coverage==5.5 +coverage[toml]==5.5 mypy==0.971 +ruff==0.8.2 types-requests==2.27.14 unittest-xml-reporting==3.1.0 -# See https://github.com/emscripten-core/emscripten/issues/19785 -lxml==4.9.2 - # This version is mentioned in `site/source/docs/site/about.rst`. # Please keep them in sync. sphinx==7.1.2 diff --git a/site/source/docs/porting/files/Synchronous-Virtual-XHR-Backed-File-System-Usage.rst b/site/source/docs/porting/files/Synchronous-Virtual-XHR-Backed-File-System-Usage.rst index c8904f3f530e1..af5f07b372dfa 100644 --- a/site/source/docs/porting/files/Synchronous-Virtual-XHR-Backed-File-System-Usage.rst +++ b/site/source/docs/porting/files/Synchronous-Virtual-XHR-Backed-File-System-Usage.rst @@ -63,6 +63,6 @@ Instructions .. include:: ../../../../../test/test_browser.py :literal: - :start-after: create_file('main.html', - :end-before: """ % (worker_filename, self.port)) + :start-after: create_file('main.html', ''' + :end-before: ''' % self.PORT) :code: html diff --git a/site/source/docs/tools_reference/emcc.rst b/site/source/docs/tools_reference/emcc.rst index 557eb6daa850a..8f107b8f73255 100644 --- a/site/source/docs/tools_reference/emcc.rst +++ b/site/source/docs/tools_reference/emcc.rst @@ -262,7 +262,7 @@ Options that are modified or new in *emcc* are listed below: [link] Runs the :term:`Closure Compiler`. Possible values are: - - ``0``: No closure compiler (default in ``-O2`` and below). + - ``0``: No closure compiler (default). - ``1``: Run closure compiler. This greatly reduces the size of the support JavaScript code (everything but the WebAssembly or asm.js). Note that this increases compile time significantly. - ``2``: Run closure compiler on *all* the emitted code, even on **asm.js** output in **asm.js** mode. This can further reduce code size, but does prevent a significant amount of **asm.js** optimizations, so it is not recommended unless you want to reduce code size at all costs. @@ -270,7 +270,6 @@ Options that are modified or new in *emcc* are listed below: - Consider using ``-sMODULARIZE`` when using closure, as it minifies globals to names that might conflict with others in the global scope. ``MODULARIZE`` puts all the output into a function (see ``src/settings.js``). - Closure will minify the name of `Module` itself, by default! Using ``MODULARIZE`` will solve that as well. Another solution is to make sure a global variable called `Module` already exists before the closure-compiled code runs, because then it will reuse that variable. - - Closure is only run if JavaScript opts are being done (``-O2`` or above). ``--closure-args=`` [link] diff --git a/site/source/docs/tools_reference/settings_reference.rst b/site/source/docs/tools_reference/settings_reference.rst index b8a971f917e71..f89d622dcf579 100644 --- a/site/source/docs/tools_reference/settings_reference.rst +++ b/site/source/docs/tools_reference/settings_reference.rst @@ -2189,7 +2189,7 @@ legalize i64s into pairs of i32s, as the wasm VM will use a BigInt where an i64 is used. If WASM_BIGINT is present, the default minimum supported browser versions will be increased to the min version that supports BigInt. -Default value: false +Default value: true .. _emit_producers_section: @@ -2939,7 +2939,7 @@ This setting also applies to modern Chromium-based Edge, which shares version numbers with Chrome. Chrome 85 was released on 2020-08-25. MAX_INT (0x7FFFFFFF, or -1) specifies that target is not supported. -Minimum supported value is 32, which was released on 2014-01-04. +Minimum supported value is 33, which was released on 2014-02-18. Default value: 85 diff --git a/src/closure-externs/closure-externs.js b/src/closure-externs/closure-externs.js index 86495adcc15a4..1346cac539330 100644 --- a/src/closure-externs/closure-externs.js +++ b/src/closure-externs/closure-externs.js @@ -15,9 +15,6 @@ var EMSCRIPTEN$IMPORT$META; var EMSCRIPTEN$AWAIT$IMPORT; -// Don't minify createRequire -var createRequire; - // Don't minify startWorker which we use to start workers once the runtime is ready. /** * @param {Object} Module diff --git a/src/closure-externs/node-externs.js b/src/closure-externs/node-externs.js index ce995014c3179..859658c05b2e5 100644 --- a/src/closure-externs/node-externs.js +++ b/src/closure-externs/node-externs.js @@ -86,6 +86,12 @@ Buffer.from = function(arrayBufferOrString, byteOffsetOrEncoding, length) {}; */ Buffer.alloc = function(size, fill, encoding) {}; +/** + * @return {boolean} + * @nosideeffects + */ +Buffer.isBuffer = function(obj) {}; + /** * @param {number=} start * @param {number=} end @@ -119,3 +125,15 @@ fs.Stats.prototype.mtimeMs; * @type {number} */ fs.Stats.prototype.ctimeMs; + +/** + * @param {string} p + * @return {boolean} + * @nosideeffects + */ +path.isAbsolute; + +/** + * @type {Object.} + */ +path.posix; diff --git a/src/embind/embind.js b/src/embind/embind.js index 64b6fd437d36a..941d9cf54d366 100644 --- a/src/embind/embind.js +++ b/src/embind/embind.js @@ -496,8 +496,7 @@ var LibraryEmbind = { name = readLatin1String(name); var stdStringIsUTF8 #if EMBIND_STD_STRING_IS_UTF8 - //process only std::string bindings with UTF8 support, in contrast to e.g. std::basic_string - = (name === "std::string"); + = true; #else = false; #endif diff --git a/src/embind/emval.js b/src/embind/emval.js index 41f5ca5656924..675db1b2ac0b9 100644 --- a/src/embind/emval.js +++ b/src/embind/emval.js @@ -447,9 +447,9 @@ var LibraryEmVal = { _emval_await__deps: ['$Emval', '$Asyncify'], _emval_await__async: true, _emval_await: (promise) => { - return Asyncify.handleAsync(() => { - promise = Emval.toValue(promise); - return promise.then(Emval.toHandle); + return Asyncify.handleAsync(async () => { + var value = await Emval.toValue(promise); + return Emval.toHandle(value); }); }, #endif @@ -468,10 +468,9 @@ var LibraryEmVal = { }, _emval_coro_suspend__deps: ['$Emval', '_emval_coro_resume'], - _emval_coro_suspend: (promiseHandle, awaiterPtr) => { - Emval.toValue(promiseHandle).then(result => { - __emval_coro_resume(awaiterPtr, Emval.toHandle(result)); - }); + _emval_coro_suspend: async (promiseHandle, awaiterPtr) => { + var result = await Emval.toValue(promiseHandle); + __emval_coro_resume(awaiterPtr, Emval.toHandle(result)); }, _emval_coro_make_promise__deps: ['$Emval', '__cxa_rethrow'], diff --git a/src/jsifier.mjs b/src/jsifier.mjs index 5e103c348561e..157f4538d46b9 100644 --- a/src/jsifier.mjs +++ b/src/jsifier.mjs @@ -338,7 +338,7 @@ function(${args}) { MEMORY64 && rtnType == 'p' ? 'proxyToMainThreadPtr' : 'proxyToMainThread'; deps.push('$' + proxyFunc); return ` -function(${args}) { +${async_}function(${args}) { if (ENVIRONMENT_IS_PTHREAD) return ${proxyFunc}(${proxiedFunctionTable.length}, 0, ${+sync}, 0${args ? ', ' : ''}${args}); ${body} diff --git a/src/library.js b/src/library.js index e72476c6441a4..94e275839c765 100644 --- a/src/library.js +++ b/src/library.js @@ -2194,20 +2194,12 @@ addToLibrary({ return x.startsWith('dynCall_') ? x : '_' + x; }, - $asyncLoad__docs: '/** @param {boolean=} noRunDep */', - $asyncLoad: (url, noRunDep) => { - return new Promise((resolve, reject) => { - var dep = !noRunDep ? getUniqueRunDependency(`al ${url}`) : ''; - if (dep) addRunDependency(dep); - readAsync(url).then( - (arrayBuffer) => { + $asyncLoad: async (url) => { + var arrayBuffer = await readAsync(url); #if ASSERTIONS - assert(arrayBuffer, `Loading data file "${url}" failed (no arrayBuffer).`); + assert(arrayBuffer, `Loading data file "${url}" failed (no arrayBuffer).`); #endif - resolve(new Uint8Array(arrayBuffer)); - if (dep) removeRunDependency(dep); - }, reject); - }); + return new Uint8Array(arrayBuffer); }, $alignMemory: (size, alignment) => { diff --git a/src/library_async.js b/src/library_async.js index 8d694439e147c..9327bc3b495ff 100644 --- a/src/library_async.js +++ b/src/library_async.js @@ -471,7 +471,7 @@ addToLibrary({ emscripten_wget_data: (url, pbuffer, pnum, perror) => { return Asyncify.handleSleep((wakeUp) => { /* no need for run dependency, this is async but will not do any prepare etc. step */ - asyncLoad(UTF8ToString(url), /*noRunDep=*/true).then((byteArray) => { + asyncLoad(UTF8ToString(url)).then((byteArray) => { // can only allocate the buffer after the wakeUp, not during an asyncing var buffer = _malloc(byteArray.length); // must be freed by caller! HEAPU8.set(byteArray, buffer); @@ -520,9 +520,7 @@ addToLibrary({ var imports = {'primary': wasmExports}; // Replace '.wasm' suffix with '.deferred.wasm'. var deferred = wasmBinaryFile.slice(0, -5) + '.deferred.wasm'; - await new Promise((resolve) => { - instantiateAsync(null, deferred, imports, resolve); - }); + await instantiateAsync(null, deferred, imports); }, $Fibers__deps: ['$Asyncify', 'emscripten_stack_set_limits', '$stackRestore'], diff --git a/src/library_browser.js b/src/library_browser.js index 6a098a48813d4..721a27df7f996 100644 --- a/src/library_browser.js +++ b/src/library_browser.js @@ -652,7 +652,7 @@ var LibraryBrowser = { // TODO: currently not callable from a pthread, but immediately calls onerror() if not on main thread. emscripten_async_load_script__deps: ['$UTF8ToString'], - emscripten_async_load_script: (url, onload, onerror) => { + emscripten_async_load_script: async (url, onload, onerror) => { url = UTF8ToString(url); #if PTHREADS if (ENVIRONMENT_IS_PTHREAD) { @@ -687,10 +687,13 @@ var LibraryBrowser = { #if ENVIRONMENT_MAY_BE_NODE && DYNAMIC_EXECUTION if (ENVIRONMENT_IS_NODE) { - readAsync(url, false).then((data) => { + try { + var data = await readAsync(url, false); eval(data); loadDone(); - }, loadError); + } catch { + loadError(); + } return; } #endif diff --git a/src/library_fs.js b/src/library_fs.js index 575bf8feaac6b..5fb60d1fb5aee 100644 --- a/src/library_fs.js +++ b/src/library_fs.js @@ -146,6 +146,7 @@ FS.staticInit(); this.name = name; this.mode = mode; this.rdev = rdev; + this.atime = this.mtime = this.ctime = Date.now(); } get read() { return (this.mode & this.readMode) === this.readMode; @@ -171,11 +172,13 @@ FS.staticInit(); // paths // lookupPath(path, opts = {}) { - path = PATH_FS.resolve(path); - if (!path) return { path: '', node: null }; opts.follow_mount ??= true + if (!PATH.isAbs(path)) { + path = FS.cwd() + '/' + path; + } + // limit max consecutive symlinks to 40 (SYMLOOP_MAX). linkloop: for (var nlinks = 0; nlinks < 40; nlinks++) { // split the absolute path @@ -192,8 +195,28 @@ FS.staticInit(); break; } - current = FS.lookupNode(current, parts[i]); + if (parts[i] === '.') { + continue; + } + + if (parts[i] === '..') { + current_path = PATH.dirname(current_path); + current = current.parent; + continue; + } + current_path = PATH.join2(current_path, parts[i]); + try { + current = FS.lookupNode(current, parts[i]); + } catch (e) { + // if noent_okay is true, suppress a ENOENT in the last component + // and return an object with an undefined node. This is needed for + // resolving symlinks in the path when creating a file. + if ((e?.errno === {{{ cDefs.ENOENT }}}) && islast && opts.noent_okay) { + return { path: current_path }; + } + throw e; + } // jump to the mount's root node if this is a mountpoint if (FS.isMountpoint(current) && (!islast || opts.follow_mount)) { @@ -207,7 +230,10 @@ FS.staticInit(); throw new FS.ErrnoError({{{ cDefs.ENOSYS }}}); } var link = current.node_ops.readlink(current); - path = PATH_FS.resolve(PATH.dirname(current_path), link, ...parts.slice(i + 1)); + if (!PATH.isAbs(link)) { + link = PATH.dirname(current_path) + '/' + link; + } + path = link + '/' + parts.slice(i + 1).join('/'); continue linkloop; } } @@ -400,8 +426,8 @@ FS.staticInit(); if (FS.isLink(node.mode)) { return {{{ cDefs.ELOOP }}}; } else if (FS.isDir(node.mode)) { - if (FS.flagsToPermissionString(flags) !== 'r' || // opening for write - (flags & {{{ cDefs.O_TRUNC }}})) { // TODO: check for O_SEARCH? (== search for dir only) + if (FS.flagsToPermissionString(flags) !== 'r' // opening for write + || (flags & ({{{ cDefs.O_TRUNC }}} | {{{ cDefs.O_CREAT }}}))) { // TODO: check for O_SEARCH? (== search for dir only) return {{{ cDefs.EISDIR }}}; } } @@ -643,9 +669,12 @@ FS.staticInit(); var lookup = FS.lookupPath(path, { parent: true }); var parent = lookup.node; var name = PATH.basename(path); - if (!name || name === '.' || name === '..') { + if (!name) { throw new FS.ErrnoError({{{ cDefs.EINVAL }}}); } + if (name === '.' || name === '..') { + throw new FS.ErrnoError({{{ cDefs.EEXIST }}}); + } var errCode = FS.mayCreate(parent, name); if (errCode) { throw new FS.ErrnoError(errCode); @@ -943,7 +972,7 @@ FS.staticInit(); } node.node_ops.setattr(node, { mode: (mode & {{{ cDefs.S_IALLUGO }}}) | (node.mode & ~{{{ cDefs.S_IALLUGO }}}), - timestamp: Date.now() + ctime: Date.now() }); }, lchmod(path, mode) { @@ -1016,7 +1045,8 @@ FS.staticInit(); var lookup = FS.lookupPath(path, { follow: true }); var node = lookup.node; node.node_ops.setattr(node, { - timestamp: Math.max(atime, mtime) + atime: atime, + mtime: mtime }); }, open(path, flags, mode = 0o666) { @@ -1030,18 +1060,20 @@ FS.staticInit(); mode = 0; } var node; + var isDirPath; if (typeof path == 'object') { node = path; } else { - path = PATH.normalize(path); - try { - var lookup = FS.lookupPath(path, { - follow: !(flags & {{{ cDefs.O_NOFOLLOW }}}) - }); - node = lookup.node; - } catch (e) { - // ignore - } + isDirPath = path.endsWith("/"); + // noent_okay makes it so that if the final component of the path + // doesn't exist, lookupPath returns `node: undefined`. `path` will be + // updated to point to the target of all symlinks. + var lookup = FS.lookupPath(path, { + follow: !(flags & {{{ cDefs.O_NOFOLLOW }}}), + noent_okay: true + }); + node = lookup.node; + path = lookup.path; } // perhaps we need to create the node var created = false; @@ -1051,9 +1083,14 @@ FS.staticInit(); if ((flags & {{{ cDefs.O_EXCL }}})) { throw new FS.ErrnoError({{{ cDefs.EEXIST }}}); } + } else if (isDirPath) { + throw new FS.ErrnoError({{{ cDefs.EISDIR }}}); } else { // node doesn't exist, try to create it - node = FS.mknod(path, mode, 0); + // Ignore the permission bits here to ensure we can `open` this new + // file below. We use chmod below the apply the permissions once the + // file is open. + node = FS.mknod(path, mode | 0o777, 0); created = true; } } @@ -1103,6 +1140,9 @@ FS.staticInit(); if (stream.stream_ops.open) { stream.stream_ops.open(stream); } + if (created) { + FS.chmod(node, mode & 0o777); + } #if expectToReceiveOnModule('logReadFiles') if (Module['logReadFiles'] && !(flags & {{{ cDefs.O_WRONLY}}})) { if (!(path in FS.readFiles)) { @@ -1391,6 +1431,9 @@ FS.staticInit(); FS.mount({ mount() { var node = FS.createNode(proc_self, 'fd', {{{ cDefs.S_IFDIR | 0o777 }}}, {{{ cDefs.S_IXUGO }}}); + node.stream_ops = { + llseek: MEMFS.stream_ops.llseek, + }; node.node_ops = { lookup(parent, name) { var fd = +name; @@ -1399,9 +1442,15 @@ FS.staticInit(); parent: null, mount: { mountpoint: 'fake' }, node_ops: { readlink: () => stream.path }, + id: fd + 1, }; ret.parent = ret; // make it look like a simple root node return ret; + }, + readdir() { + return Array.from(FS.streams.entries()) + .filter(([k, v]) => v) + .map(([k, v]) => k.toString()); } }; return node; @@ -1618,7 +1667,7 @@ FS.staticInit(); buffer[offset+i] = result; } if (bytesRead) { - stream.node.timestamp = Date.now(); + stream.node.atime = Date.now(); } return bytesRead; }, @@ -1631,7 +1680,7 @@ FS.staticInit(); } } if (length) { - stream.node.timestamp = Date.now(); + stream.node.mtime = stream.node.ctime = Date.now(); } return i; } diff --git a/src/library_lz4.js b/src/library_lz4.js index 061af9faf72a7..d989d99322009 100644 --- a/src/library_lz4.js +++ b/src/library_lz4.js @@ -70,7 +70,7 @@ addToLibrary({ node.mode = mode; node.node_ops = LZ4.node_ops; node.stream_ops = LZ4.stream_ops; - node.timestamp = (mtime || new Date).getTime(); + this.atime = this.mtime = this.ctime = (mtime || new Date).getTime(); assert(LZ4.FILE_MODE !== LZ4.DIR_MODE); if (mode === LZ4.FILE_MODE) { node.size = contents.end - contents.start; @@ -95,19 +95,18 @@ addToLibrary({ gid: 0, rdev: 0, size: node.size, - atime: new Date(node.timestamp), - mtime: new Date(node.timestamp), - ctime: new Date(node.timestamp), + atime: new Date(node.atime), + mtime: new Date(node.mtime), + ctime: new Date(node.ctime), blksize: 4096, blocks: Math.ceil(node.size / 4096), }; }, setattr(node, attr) { - if (attr.mode !== undefined) { - node.mode = attr.mode; - } - if (attr.timestamp !== undefined) { - node.timestamp = attr.timestamp; + for (const key of ['mode', 'atime', 'mtime', 'ctime']) { + if (attr[key]) { + node[key] = attr[key]; + } } }, lookup(parent, name) { diff --git a/src/library_memfs.js b/src/library_memfs.js index b4beccd08aadf..09a994f7f5b3e 100644 --- a/src/library_memfs.js +++ b/src/library_memfs.js @@ -92,11 +92,11 @@ addToLibrary({ node.node_ops = MEMFS.ops_table.chrdev.node; node.stream_ops = MEMFS.ops_table.chrdev.stream; } - node.timestamp = Date.now(); + node.atime = node.mtime = node.ctime = Date.now(); // add the new node to the parent if (parent) { parent.contents[name] = node; - parent.timestamp = node.timestamp; + parent.atime = parent.mtime = parent.ctime = node.atime; } return node; }, @@ -161,9 +161,9 @@ addToLibrary({ } else { attr.size = 0; } - attr.atime = new Date(node.timestamp); - attr.mtime = new Date(node.timestamp); - attr.ctime = new Date(node.timestamp); + attr.atime = new Date(node.atime); + attr.mtime = new Date(node.mtime); + attr.ctime = new Date(node.ctime); // NOTE: In our implementation, st_blocks = Math.ceil(st_size/st_blksize), // but this is not required by the standard. attr.blksize = 4096; @@ -171,11 +171,10 @@ addToLibrary({ return attr; }, setattr(node, attr) { - if (attr.mode !== undefined) { - node.mode = attr.mode; - } - if (attr.timestamp !== undefined) { - node.timestamp = attr.timestamp; + for (const key of ["mode", "atime", "mtime", "ctime"]) { + if (attr[key]) { + node[key] = attr[key]; + } } if (attr.size !== undefined) { MEMFS.resizeFileStorage(node, attr.size); @@ -192,29 +191,28 @@ addToLibrary({ return MEMFS.createNode(parent, name, mode, dev); }, rename(old_node, new_dir, new_name) { - // if we're overwriting a directory at new_name, make sure it's empty. - if (FS.isDir(old_node.mode)) { - var new_node; - try { - new_node = FS.lookupNode(new_dir, new_name); - } catch (e) { - } - if (new_node) { + var new_node; + try { + new_node = FS.lookupNode(new_dir, new_name); + } catch (e) {} + if (new_node) { + if (FS.isDir(old_node.mode)) { + // if we're overwriting a directory at new_name, make sure it's empty. for (var i in new_node.contents) { throw new FS.ErrnoError({{{ cDefs.ENOTEMPTY }}}); } } + FS.hashRemoveNode(new_node); } // do the internal rewiring delete old_node.parent.contents[old_node.name]; - old_node.parent.timestamp = Date.now() - old_node.name = new_name; new_dir.contents[new_name] = old_node; - new_dir.timestamp = old_node.parent.timestamp; + old_node.name = new_name; + new_dir.ctime = new_dir.mtime = old_node.parent.ctime = old_node.parent.mtime = Date.now(); }, unlink(parent, name) { delete parent.contents[name]; - parent.timestamp = Date.now(); + parent.ctime = parent.mtime = Date.now(); }, rmdir(parent, name) { var node = FS.lookupNode(parent, name); @@ -222,14 +220,10 @@ addToLibrary({ throw new FS.ErrnoError({{{ cDefs.ENOTEMPTY }}}); } delete parent.contents[name]; - parent.timestamp = Date.now(); + parent.ctime = parent.mtime = Date.now(); }, readdir(node) { - var entries = ['.', '..']; - for (var key of Object.keys(node.contents)) { - entries.push(key); - } - return entries; + return ['.', '..', ...Object.keys(node.contents)]; }, symlink(parent, newname, oldpath) { var node = MEMFS.createNode(parent, newname, 0o777 | {{{ cDefs.S_IFLNK }}}, 0); @@ -282,7 +276,7 @@ addToLibrary({ if (!length) return 0; var node = stream.node; - node.timestamp = Date.now(); + node.mtime = node.ctime = Date.now(); if (buffer.subarray && (!node.contents || node.contents.subarray)) { // This write is from a typed array to a typed array? if (canOwn) { diff --git a/src/library_nodefs.js b/src/library_nodefs.js index 6d0544c8527cc..21e0561b783d9 100644 --- a/src/library_nodefs.js +++ b/src/library_nodefs.js @@ -138,7 +138,7 @@ addToLibrary({ } return { dev: stat.dev, - ino: stat.ino, + ino: node.id, mode: stat.mode, nlink: stat.nlink, uid: stat.uid, @@ -166,9 +166,10 @@ addToLibrary({ // update the common node structure mode as well node.mode = attr.mode; } - if (attr.timestamp !== undefined) { - var date = new Date(attr.timestamp); - fs.utimesSync(path, date, date); + if (attr.atime || attr.mtime) { + var atime = attr.atime && new Date(attr.atime); + var mtime = attr.mtime && new Date(attr.mtime); + fs.utimesSync(path, atime, mtime); } if (attr.size !== undefined) { fs.truncateSync(path, attr.size); @@ -196,6 +197,9 @@ addToLibrary({ rename(oldNode, newDir, newName) { var oldPath = NODEFS.realPath(oldNode); var newPath = PATH.join2(NODEFS.realPath(newDir), newName); + try { + FS.unlink(newPath); + } catch(e) {} NODEFS.tryFSOperation(() => fs.renameSync(oldPath, newPath)); oldNode.name = newName; }, diff --git a/src/library_nodepath.js b/src/library_nodepath.js index c42820f36e289..09d4c48a4662a 100644 --- a/src/library_nodepath.js +++ b/src/library_nodepath.js @@ -12,14 +12,14 @@ // operations. Hence, using `nodePath` should be safe here. addToLibrary({ - $PATH: { - isAbs: (path) => nodePath['isAbsolute'](path), - normalize: (path) => nodePath['normalize'](path), - dirname: (path) => nodePath['dirname'](path), - basename: (path) => nodePath['basename'](path), - join: (...args) => nodePath['join'](...args), - join2: (l, r) => nodePath['join'](l, r), - }, + $PATH: `{ + isAbs: nodePath.isAbsolute, + normalize: nodePath.normalize, + dirname: nodePath.dirname, + basename: nodePath.basename, + join: nodePath.join, + join2: nodePath.join, + }`, // The FS-using parts are split out into a separate object, so simple path // usage does not require the FS. $PATH_FS__deps: ['$FS'], @@ -27,8 +27,8 @@ addToLibrary({ $PATH_FS: { resolve: (...paths) => { paths.unshift(FS.cwd()); - return nodePath['posix']['resolve'](...paths); + return nodePath.posix.resolve(...paths); }, - relative: (from, to) => nodePath['posix']['relative'](from || FS.cwd(), to || FS.cwd()), + relative: (from, to) => nodePath.posix.relative(from || FS.cwd(), to || FS.cwd()), } }); diff --git a/src/library_noderawfs.js b/src/library_noderawfs.js index 55b9e97a4c716..81ab16962a89c 100644 --- a/src/library_noderawfs.js +++ b/src/library_noderawfs.js @@ -40,7 +40,7 @@ addToLibrary({ }, lookupPath(path, opts = {}) { if (opts.parent) { - path = nodePath.dirname(path); + path = PATH.dirname(path); } var st = fs.lstatSync(path); var mode = NODEFS.getMode(path); @@ -112,13 +112,13 @@ addToLibrary({ fs.ftruncateSync(stream.nfd, len); }, utime(path, atime, mtime) { - // -1 here for atime or mtime means UTIME_OMIT was passed. Since node + // null here for atime or mtime means UTIME_OMIT was passed. Since node // doesn't support this concept we need to first find the existing // timestamps in order to preserve them. - if (atime == -1 || mtime == -1) { + if ((atime === null) || (mtime === null)) { var st = fs.statSync(path); - if (atime == -1) atime = st.atimeMs; - if (mtime == -1) mtime = st.mtimeMs; + atime ||= st.atimeMs; + mtime ||= st.mtimeMs; } fs.utimesSync(path, atime/1000, mtime/1000); }, diff --git a/src/library_path.js b/src/library_path.js index ab3112503a202..b1ff8364afb76 100644 --- a/src/library_path.js +++ b/src/library_path.js @@ -63,15 +63,9 @@ addToLibrary({ } return root + dir; }, - basename: (path) => { - // EMSCRIPTEN return '/'' for '/', not an empty string - if (path === '/') return '/'; - path = PATH.normalize(path); - path = path.replace(/\/$/, ""); - var lastSlash = path.lastIndexOf('/'); - if (lastSlash === -1) return path; - return path.substr(lastSlash+1); - }, + // This differs from node's path.basename in that it returns '/' for '/' + // rather than the empty string. + basename: (path) => path && path.match(/([^\/]+|\/)\/*$/)[1], join: (...paths) => PATH.normalize(paths.join('/')), join2: (l, r) => PATH.normalize(l + '/' + r), }, diff --git a/src/library_pipefs.js b/src/library_pipefs.js index 88a0ededa9c8d..48c0d06757593 100644 --- a/src/library_pipefs.js +++ b/src/library_pipefs.js @@ -77,6 +77,9 @@ addToLibrary({ return 0; }, + dup(stream) { + stream.node.pipe.refcnt++; + }, ioctl(stream, request, varargs) { return {{{ cDefs.EINVAL }}}; }, diff --git a/src/library_proxyfs.js b/src/library_proxyfs.js index c98fe3755eadd..6a2a9d35efe49 100644 --- a/src/library_proxyfs.js +++ b/src/library_proxyfs.js @@ -63,9 +63,10 @@ addToLibrary({ // update the common node structure mode as well node.mode = attr.mode; } - if (attr.timestamp !== undefined) { - var date = new Date(attr.timestamp); - node.mount.opts.fs.utime(path, date, date); + if (attr.atime || attr.mtime) { + var atime = new Date(attr.atime || attr.mtime); + var mtime = new Date(attr.mtime || attr.atime); + node.mount.opts.fs.utime(path, atime, mtime); } if (attr.size !== undefined) { node.mount.opts.fs.truncate(path, attr.size); diff --git a/src/library_pthread.js b/src/library_pthread.js index 80d4f73e19180..ccb5f7de85265 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -1066,7 +1066,7 @@ var LibraryPThread = { '$runtimeKeepaliveCounter', #endif ], - $invokeEntryPoint: {{{ asyncIf(ASYNCIFY == 2) }}} (ptr, arg) => { + $invokeEntryPoint: {{{ asyncIf(ASYNCIFY == 2) }}}(ptr, arg) => { #if PTHREADS_DEBUG dbg(`invokeEntryPoint: ${ptrToString(ptr)}`); #endif diff --git a/src/library_syscall.js b/src/library_syscall.js index fde42e8a48428..80e616408bac6 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -38,7 +38,7 @@ var SyscallsLibrary = { } return dir; } - return PATH.join2(dir, path); + return dir + '/' + path; }, doStat(func, path, buf) { @@ -695,9 +695,9 @@ var SyscallsLibrary = { var pos = 0; var off = FS.llseek(stream, 0, {{{ cDefs.SEEK_CUR }}}); - var idx = Math.floor(off / struct_size); - - while (idx < stream.getdents.length && pos + struct_size <= count) { + var startIdx = Math.floor(off / struct_size); + var endIdx = Math.min(stream.getdents.length, startIdx + Math.floor(count/struct_size)) + for (var idx = startIdx; idx < endIdx; idx++) { var id; var type; var name = stream.getdents[idx]; @@ -711,7 +711,17 @@ var SyscallsLibrary = { type = 4; // DT_DIR } else { - var child = FS.lookupNode(stream.node, name); + var child; + try { + child = FS.lookupNode(stream.node, name); + } catch (e) { + // If the entry is not a directory, file, or symlink, nodefs + // lookupNode will raise EINVAL. Skip these and continue. + if (e?.errno === {{{ cDefs.EINVAL }}}) { + continue; + } + throw e; + } id = child.id; type = FS.isChrdev(child.mode) ? 2 : // DT_CHR, character device. FS.isDir(child.mode) ? 4 : // DT_DIR, directory. @@ -727,7 +737,6 @@ var SyscallsLibrary = { {{{ makeSetValue('dirp + pos', C_STRUCTS.dirent.d_type, 'type', 'i8') }}}; stringToUTF8(name, dirp + pos + {{{ C_STRUCTS.dirent.d_name }}}, 256); pos += struct_size; - idx += 1; } FS.llseek(stream, idx * struct_size, {{{ cDefs.SEEK_SET }}}); return pos; @@ -824,10 +833,6 @@ var SyscallsLibrary = { __syscall_mkdirat: (dirfd, path, mode) => { path = SYSCALLS.getStr(path); path = SYSCALLS.calculateAt(dirfd, path); - // remove a trailing slash, if one - /a/b/ has basename of '', but - // we want to create b in the context of this function - path = PATH.normalize(path); - if (path[path.length-1] === '/') path = path.substr(0, path.length-1); FS.mkdir(path, mode, 0); return 0; }, @@ -959,7 +964,7 @@ var SyscallsLibrary = { if (nanoseconds == {{{ cDefs.UTIME_NOW }}}) { atime = now; } else if (nanoseconds == {{{ cDefs.UTIME_OMIT }}}) { - atime = -1; + atime = null; } else { atime = (seconds*1000) + (nanoseconds/(1000*1000)); } @@ -969,15 +974,14 @@ var SyscallsLibrary = { if (nanoseconds == {{{ cDefs.UTIME_NOW }}}) { mtime = now; } else if (nanoseconds == {{{ cDefs.UTIME_OMIT }}}) { - mtime = -1; + mtime = null; } else { mtime = (seconds*1000) + (nanoseconds/(1000*1000)); } } - // -1 here means UTIME_OMIT was passed. FS.utime tables the max of these - // two values and sets the timestamp to that single value. If both were - // set to UTIME_OMIT then we can skip the call completely. - if (mtime != -1 || atime != -1) { + // null here means UTIME_OMIT was passed. If both were set to UTIME_OMIT then + // we can skip the call completely. + if ((mtime ?? atime) !== null) { FS.utime(path, atime, mtime); } return 0; diff --git a/src/library_trace.js b/src/library_trace.js index f45ea94a01c7e..e356881d0de00 100644 --- a/src/library_trace.js +++ b/src/library_trace.js @@ -58,29 +58,29 @@ var LibraryTracing = { }, // Work around CORS issues ... - fetchBlob: (url) => { - return fetch(url).then((rsp) => rsp.blob()); + fetchBlob: async (url) => { + var rsp = await fetch(url); + return rsp.blob(); }, - configure: (collector_url, application) => { + configure: async (collector_url, application) => { EmscriptenTrace.now = _emscripten_get_now; var now = new Date(); var session_id = now.getTime().toString() + '_' + Math.floor((Math.random() * 100) + 1).toString(); - EmscriptenTrace.fetchBlob(collector_url + 'worker.js').then((blob) => { - EmscriptenTrace.worker = new Worker(window.URL.createObjectURL(blob)); - EmscriptenTrace.worker.addEventListener('error', (e) => { - out('TRACE WORKER ERROR:'); - out(e); - }, false); - EmscriptenTrace.worker.postMessage({ 'cmd': 'configure', - 'data_version': EmscriptenTrace.DATA_VERSION, - 'session_id': session_id, - 'url': collector_url }); - EmscriptenTrace.configured = true; - EmscriptenTrace.collectorEnabled = true; - EmscriptenTrace.postEnabled = true; - }); + var blob = await EmscriptenTrace.fetchBlob(collector_url + 'worker.js'); + EmscriptenTrace.worker = new Worker(window.URL.createObjectURL(blob)); + EmscriptenTrace.worker.addEventListener('error', (e) => { + out('TRACE WORKER ERROR:'); + out(e); + }, false); + EmscriptenTrace.worker.postMessage({ 'cmd': 'configure', + 'data_version': EmscriptenTrace.DATA_VERSION, + 'session_id': session_id, + 'url': collector_url }); + EmscriptenTrace.configured = true; + EmscriptenTrace.collectorEnabled = true; + EmscriptenTrace.postEnabled = true; EmscriptenTrace.post([EmscriptenTrace.EVENT_APPLICATION_NAME, application]); EmscriptenTrace.post([EmscriptenTrace.EVENT_SESSION_NAME, now.toISOString()]); }, diff --git a/src/library_tty.js b/src/library_tty.js index 28c1d5232d5a7..6889cf157588a 100644 --- a/src/library_tty.js +++ b/src/library_tty.js @@ -79,7 +79,7 @@ addToLibrary({ buffer[offset+i] = result; } if (bytesRead) { - stream.node.timestamp = Date.now(); + stream.node.atime = Date.now(); } return bytesRead; }, @@ -95,7 +95,7 @@ addToLibrary({ throw new FS.ErrnoError({{{ cDefs.EIO }}}); } if (length) { - stream.node.timestamp = Date.now(); + stream.node.mtime = stream.node.ctime = Date.now(); } return i; } diff --git a/src/library_wasmfs.js b/src/library_wasmfs.js index 645f3ac597342..fd238bc2767bd 100644 --- a/src/library_wasmfs.js +++ b/src/library_wasmfs.js @@ -198,7 +198,7 @@ FS.init(); var bytesRead; if (seeking) { - bytesRead = __wasmfs_pread(stream.fd, dataBuffer, length, position); + bytesRead = __wasmfs_pread(stream.fd, dataBuffer, length, {{{ splitI64('position') }}}); } else { bytesRead = __wasmfs_read(stream.fd, dataBuffer, length); } @@ -222,7 +222,7 @@ FS.init(); var bytesRead; if (seeking) { - bytesRead = __wasmfs_pwrite(stream.fd, dataBuffer, length, position); + bytesRead = __wasmfs_pwrite(stream.fd, dataBuffer, length, {{{ splitI64('position') }}}); } else { bytesRead = __wasmfs_write(stream.fd, dataBuffer, length); } diff --git a/src/library_wasmfs_jsimpl.js b/src/library_wasmfs_jsimpl.js index f6293f3d460ea..63b3e77ec2aee 100644 --- a/src/library_wasmfs_jsimpl.js +++ b/src/library_wasmfs_jsimpl.js @@ -53,6 +53,7 @@ addToLibrary({ return wasmFS$backends[backend].getSize(file); }, + _wasmfs_jsimpl_set_size__i53abi: true, _wasmfs_jsimpl_set_size: (backend, file, size) => { #if ASSERTIONS assert(wasmFS$backends[backend]); diff --git a/src/library_wasmfs_opfs.js b/src/library_wasmfs_opfs.js index 00f71e09cbf65..4dd7598226298 100644 --- a/src/library_wasmfs_opfs.js +++ b/src/library_wasmfs_opfs.js @@ -319,11 +319,11 @@ addToLibrary({ _wasmfs_opfs_read_access__i53abi: true, _wasmfs_opfs_read_access__deps: ['$wasmfsOPFSAccessHandles'], - _wasmfs_opfs_read_access: {{{ asyncIf(!PTHREADS) }}} function(accessID, bufPtr, len, pos) { + _wasmfs_opfs_read_access: {{{ asyncIf(!PTHREADS) }}}function(accessID, bufPtr, len, pos) { let accessHandle = wasmfsOPFSAccessHandles.get(accessID); let data = HEAPU8.subarray(bufPtr, bufPtr + len); try { - return {{{ awaitIf(!PTHREADS) }}} accessHandle.read(data, {at: pos}); + return {{{ awaitIf(!PTHREADS) }}}accessHandle.read(data, {at: pos}); } catch (e) { if (e.name == "TypeError") { return -{{{ cDefs.EINVAL }}}; @@ -367,11 +367,11 @@ addToLibrary({ _wasmfs_opfs_write_access__i53abi: true, _wasmfs_opfs_write_access__deps: ['$wasmfsOPFSAccessHandles'], - _wasmfs_opfs_write_access: {{{ asyncIf(!PTHREADS) }}} function(accessID, bufPtr, len, pos) { + _wasmfs_opfs_write_access: {{{ asyncIf(!PTHREADS) }}}function(accessID, bufPtr, len, pos) { let accessHandle = wasmfsOPFSAccessHandles.get(accessID); let data = HEAPU8.subarray(bufPtr, bufPtr + len); try { - return {{{ awaitIf(!PTHREADS) }}} accessHandle.write(data, {at: pos}); + return {{{ awaitIf(!PTHREADS) }}}accessHandle.write(data, {at: pos}); } catch (e) { if (e.name == "TypeError") { return -{{{ cDefs.EINVAL }}}; diff --git a/src/library_webgl.js b/src/library_webgl.js index 5171541e4e116..d7d1e856c232e 100644 --- a/src/library_webgl.js +++ b/src/library_webgl.js @@ -940,7 +940,6 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}}; gl.uniform1i(gl.getUniformLocation(blitProgram, "sampler"), 0); gl.useProgram(null); - context.defaultVao = undefined; if (gl.createVertexArray) { context.defaultVao = gl.createVertexArray(); gl.bindVertexArray(context.defaultVao); @@ -1020,17 +1019,13 @@ for (/**@suppress{duplicate}*/var i = 0; i <= {{{ GL_POOL_TEMP_BUFFERS_SIZE }}}; gl.drawArrays(5/*GL_TRIANGLE_STRIP*/, 0, 4); } -#if !OFFSCREEN_FRAMEBUFFER_FORBID_VAO_PATH if (context.defaultVao) { // WebGL 2 or OES_vertex_array_object var prevVAO = gl.getParameter(0x85B5 /*GL_VERTEX_ARRAY_BINDING*/); gl.bindVertexArray(context.defaultVao); draw(); gl.bindVertexArray(prevVAO); - } - else -#endif - { + } else { var prevVertexAttribPointer = { buffer: gl.getVertexAttrib(context.blitPosLoc, 0x889F /*GL_VERTEX_ATTRIB_ARRAY_BUFFER_BINDING*/), size: gl.getVertexAttrib(context.blitPosLoc, 0x8623 /*GL_VERTEX_ATTRIB_ARRAY_SIZE*/), diff --git a/src/library_wget.js b/src/library_wget.js index fe6416a7391f0..578ae81129e65 100644 --- a/src/library_wget.js +++ b/src/library_wget.js @@ -62,10 +62,11 @@ var LibraryWget = { emscripten_async_wget_data__deps: ['$asyncLoad', 'malloc', 'free', '$callUserCallback'], emscripten_async_wget_data__proxy: 'sync', - emscripten_async_wget_data: (url, userdata, onload, onerror) => { + emscripten_async_wget_data: async (url, userdata, onload, onerror) => { {{{ runtimeKeepalivePush() }}} /* no need for run dependency, this is async but will not do any prepare etc. step */ - asyncLoad(UTF8ToString(url), /*noRunDep=*/true).then((byteArray) => { + try { + var byteArray = await asyncLoad(UTF8ToString(url)); {{{ runtimeKeepalivePop() }}} callUserCallback(() => { var buffer = _malloc(byteArray.length); @@ -73,14 +74,14 @@ var LibraryWget = { {{{ makeDynCall('vppi', 'onload') }}}(userdata, buffer, byteArray.length); _free(buffer); }); - }, () => { + } catch (e) { if (onerror) { {{{ runtimeKeepalivePop() }}} callUserCallback(() => { {{{ makeDynCall('vp', 'onerror') }}}(userdata); }); } - }); + } }, emscripten_async_wget2__deps: ['$PATH_FS', '$wget', '$stackRestore', '$stringToUTF8OnStack'], diff --git a/src/library_workerfs.js b/src/library_workerfs.js index 28020bfac7e4d..b5245d201f181 100644 --- a/src/library_workerfs.js +++ b/src/library_workerfs.js @@ -57,7 +57,7 @@ addToLibrary({ node.mode = mode; node.node_ops = WORKERFS.node_ops; node.stream_ops = WORKERFS.stream_ops; - node.timestamp = (mtime || new Date).getTime(); + node.atime = node.mtime = node.ctime = (mtime || new Date).getTime(); assert(WORKERFS.FILE_MODE !== WORKERFS.DIR_MODE); if (mode === WORKERFS.FILE_MODE) { node.size = contents.size; @@ -82,19 +82,18 @@ addToLibrary({ gid: 0, rdev: 0, size: node.size, - atime: new Date(node.timestamp), - mtime: new Date(node.timestamp), - ctime: new Date(node.timestamp), + atime: new Date(node.atime), + mtime: new Date(node.mtime), + ctime: new Date(node.ctime), blksize: 4096, blocks: Math.ceil(node.size / 4096), }; }, setattr(node, attr) { - if (attr.mode !== undefined) { - node.mode = attr.mode; - } - if (attr.timestamp !== undefined) { - node.timestamp = attr.timestamp; + for (const key of ["mode", "atime", "mtime", "ctime"]) { + if (attr[key]) { + node[key] = attr[key]; + } } }, lookup(parent, name) { diff --git a/src/node_shell_read.js b/src/node_shell_read.js index 46a736659bae4..8fafd66b81f36 100644 --- a/src/node_shell_read.js +++ b/src/node_shell_read.js @@ -5,23 +5,21 @@ */ readBinary = (filename) => { - // We need to re-wrap `file://` strings to URLs. Normalizing isn't - // necessary in that case, the path should already be absolute. - filename = isFileURI(filename) ? new URL(filename) : nodePath.normalize(filename); + // We need to re-wrap `file://` strings to URLs. + filename = isFileURI(filename) ? new URL(filename) : filename; var ret = fs.readFileSync(filename); #if ASSERTIONS - assert(ret.buffer); + assert(Buffer.isBuffer(ret)); #endif return ret; }; -readAsync = (filename, binary = true) => { +readAsync = async (filename, binary = true) => { // See the comment in the `readBinary` function. - filename = isFileURI(filename) ? new URL(filename) : nodePath.normalize(filename); - return new Promise((resolve, reject) => { - fs.readFile(filename, binary ? undefined : 'utf8', (err, data) => { - if (err) reject(err); - else resolve(binary ? data.buffer : data); - }); - }); + filename = isFileURI(filename) ? new URL(filename) : filename; + var ret = fs.readFileSync(filename, binary ? undefined : 'utf8'); +#if ASSERTIONS + assert(binary ? Buffer.isBuffer(ret) : typeof ret == 'string'); +#endif + return ret; }; diff --git a/src/parseTools.mjs b/src/parseTools.mjs index ca474affa3978..509ff01727b44 100644 --- a/src/parseTools.mjs +++ b/src/parseTools.mjs @@ -785,24 +785,16 @@ export function modifyJSFunction(text, func) { } export function runIfMainThread(text) { - if (WASM_WORKERS && PTHREADS) { - return `if (!ENVIRONMENT_IS_WASM_WORKER && !ENVIRONMENT_IS_PTHREAD) { ${text} }`; - } else if (WASM_WORKERS) { - return `if (!ENVIRONMENT_IS_WASM_WORKER) { ${text} }`; - } else if (PTHREADS) { - return `if (!ENVIRONMENT_IS_PTHREAD) { ${text} }`; + if (WASM_WORKERS || PTHREADS) { + return `if (${ENVIRONMENT_IS_MAIN_THREAD()}) { ${text} }`; } else { return text; } } function runIfWorkerThread(text) { - if (WASM_WORKERS && PTHREADS) { - return `if (ENVIRONMENT_IS_WASM_WORKER || ENVIRONMENT_IS_PTHREAD) { ${text} }`; - } else if (WASM_WORKERS) { - return `if (ENVIRONMENT_IS_WASM_WORKER) { ${text} }`; - } else if (PTHREADS) { - return `if (ENVIRONMENT_IS_PTHREAD) { ${text} }`; + if (WASM_WORKERS || PTHREADS) { + return `if (${ENVIRONMENT_IS_WORKER_THREAD()}) { ${text} }`; } else { return ''; } @@ -979,40 +971,12 @@ function to64(x) { return `BigInt(${x})`; } -// Add assertions to catch common errors when using the Promise object we -// return from MODULARIZE Module() invocations. -function addReadyPromiseAssertions() { - // Warn on someone doing - // - // var instance = Module(); - // ... - // instance._main(); - const properties = Array.from(EXPORTED_FUNCTIONS.values()); - // Also warn on onRuntimeInitialized which might be a common pattern with - // older MODULARIZE-using codebases. - properties.push('onRuntimeInitialized'); - const warningEnding = - ' on the Promise object, instead of the instance. Use .then() to get called back with the instance, see the MODULARIZE docs in src/settings.js'; - const res = JSON.stringify(properties); - return ( - res + - `.forEach((prop) => { - if (!Object.getOwnPropertyDescriptor(readyPromise, prop)) { - Object.defineProperty(readyPromise, prop, { - get: () => abort('You are getting ' + prop + '${warningEnding}'), - set: () => abort('You are setting ' + prop + '${warningEnding}'), - }); - } -});` - ); -} - function asyncIf(condition) { - return condition ? 'async' : ''; + return condition ? 'async ' : ''; } function awaitIf(condition) { - return condition ? 'await' : ''; + return condition ? 'await ' : ''; } // Adds a call to runtimeKeepalivePush, if needed by the current build @@ -1094,12 +1058,15 @@ function implicitSelf() { } function ENVIRONMENT_IS_MAIN_THREAD() { + return `(!${ENVIRONMENT_IS_WORKER_THREAD()})`; +} + +function ENVIRONMENT_IS_WORKER_THREAD() { + assert(PTHREADS || WASM_WORKERS); var envs = []; if (PTHREADS) envs.push('ENVIRONMENT_IS_PTHREAD'); if (WASM_WORKERS) envs.push('ENVIRONMENT_IS_WASM_WORKER'); - if (AUDIO_WORKLET) envs.push('ENVIRONMENT_IS_AUDIO_WORKLET'); - if (envs.length == 0) return 'true'; - return '(!(' + envs.join('||') + '))'; + return '(' + envs.join('||') + ')'; } addToCompileTimeContext({ @@ -1120,9 +1087,9 @@ addToCompileTimeContext({ TARGET_NOT_SUPPORTED, WASM_PAGE_SIZE, ENVIRONMENT_IS_MAIN_THREAD, + ENVIRONMENT_IS_WORKER_THREAD, addAtExit, addAtInit, - addReadyPromiseAssertions, asyncIf, awaitIf, buildStringArray, diff --git a/src/polyfill/atob.js b/src/polyfill/atob.js index 5de121d14a13f..4626cb72a1f1f 100644 --- a/src/polyfill/atob.js +++ b/src/polyfill/atob.js @@ -14,6 +14,7 @@ if (typeof atob == 'undefined') { if (typeof global != 'undefined' && typeof globalThis == 'undefined') { + /** @suppress{checkTypes} */ globalThis = global; } diff --git a/src/polyfill/fetch.js b/src/polyfill/fetch.js deleted file mode 100644 index 979335699a84e..0000000000000 --- a/src/polyfill/fetch.js +++ /dev/null @@ -1,80 +0,0 @@ -// Fetch polyfill from https://github.com/developit/unfetch -// License: -//============================================================================== -// Copyright (c) 2017 Jason Miller -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -//============================================================================== - -#if !POLYFILL -#error "this file should never be included unless POLYFILL is set" -#endif - -if (typeof globalThis.fetch == 'undefined') { - globalThis.fetch = function (url, options) { - options = options || {}; - return new Promise((resolve, reject) => { - const request = new XMLHttpRequest(); - const keys = []; - const headers = {}; - - request.responseType = 'arraybuffer'; - - const response = () => ({ - ok: ((request.status / 100) | 0) == 2, // 200-299 - statusText: request.statusText, - status: request.status, - url: request.responseURL, - text: () => Promise.resolve(request.responseText), - json: () => Promise.resolve(request.responseText).then(JSON.parse), - blob: () => Promise.resolve(new Blob([request.response])), - arrayBuffer: () => Promise.resolve(request.response), - clone: response, - headers: { - keys: () => keys, - entries: () => keys.map((n) => [n, request.getResponseHeader(n)]), - get: (n) => request.getResponseHeader(n), - has: (n) => request.getResponseHeader(n) != null, - }, - }); - - request.open(options.method || "get", url, true); - - request.onload = () => { - request - .getAllResponseHeaders() - .toLowerCase() - .replace(/^(.+?):/gm, (m, key) => { - headers[key] || keys.push((headers[key] = key)); - }); - resolve(response()); - }; - - request.onerror = reject; - - request.withCredentials = options.credentials == "include"; - - for (const i in options.headers) { - request.setRequestHeader(i, options.headers[i]); - } - - request.send(options.body || null); - }); - } -} diff --git a/src/polyfill/objassign.js b/src/polyfill/objassign.js deleted file mode 100644 index e9aafe02d3274..0000000000000 --- a/src/polyfill/objassign.js +++ /dev/null @@ -1,34 +0,0 @@ -// Object.assign polyfill from: -// https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/js/es6/util/assign.js - -#if !POLYFILL -#error "this file should never be included unless POLYFILL is set" -#endif - -if (typeof Object.assign == 'undefined') { - /** - * Equivalent to the Object.assign() method, but guaranteed to be available for use in code - * generated by the compiler. - * - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/assign - * - * Copies values of all enumerable own properties from one or more - * sources to the given target object, and returns the target. - * - * @final - * @param {!Object} target The target object onto which to copy. - * @param {...?Object} source The source objects. - * @return {!Object} The target object is returned. - * @suppress {visibility, duplicate, checkTypes} - */ - Object.assign = function(target, source) { - for (var i = 1; i < arguments.length; i++) { - var source = arguments[i]; - if (!source) continue; - for (var key in source) { - if (source.hasOwnProperty(key)) target[key] = source[key]; - } - } - return target; - }; -} diff --git a/src/polyfill/promise.js b/src/polyfill/promise.js deleted file mode 100644 index 461be88333461..0000000000000 --- a/src/polyfill/promise.js +++ /dev/null @@ -1,274 +0,0 @@ -// Promise polyfill from https://github.com/taylorhakes/promise-polyfill -// License: -//============================================================================== -// Copyright (c) 2014 Taylor Hakes -// Copyright (c) 2014 Forbes Lindesay -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. -//============================================================================== - -#if !POLYFILL -#error "this file should never be included unless POLYFILL is set" -#endif - -/** @suppress{duplicate} This is already defined in from Closure's built-in - externs.zip//es6.js, Closure should not yell when seeing this again. */ -var Promise = (function() { - function noop() {} - - // Polyfill for Function.prototype.bind - function bind(fn, thisArg) { - return function() { - fn.apply(thisArg, arguments); - }; - } - - /** - * @constructor - * @param {Function} fn - */ - function Promise(fn) { - if (!(this instanceof Promise)) - throw new TypeError('Promises must be constructed via new'); - if (typeof fn != 'function') throw new TypeError('not a function'); - /** @type {!number} */ - this._state = 0; - /** @type {!boolean} */ - this._handled = false; - /** @type {Promise|undefined} */ - this._value = undefined; - /** @type {!Array} */ - this._deferreds = []; - - doResolve(fn, this); - } - - function handle(self, deferred) { - while (self._state === 3) { - self = self._value; - } - if (self._state === 0) { - self._deferreds.push(deferred); - return; - } - self._handled = true; - Promise._immediateFn(function() { - var cb = self._state === 1 ? deferred.onFulfilled : deferred.onRejected; - if (cb === null) { - (self._state === 1 ? resolve : reject)(deferred.promise, self._value); - return; - } - var ret; - try { - ret = cb(self._value); - } catch (e) { - reject(deferred.promise, e); - return; - } - resolve(deferred.promise, ret); - }); - } - - function resolve(self, newValue) { - try { - // Promise Resolution Procedure: https://github.com/promises-aplus/promises-spec#the-promise-resolution-procedure - if (newValue === self) - throw new TypeError('A promise cannot be resolved with itself.'); - if ( - newValue && - (typeof newValue == 'object' || typeof newValue == 'function') - ) { - var then = newValue.then; - if (newValue instanceof Promise) { - self._state = 3; - self._value = newValue; - finale(self); - return; - } else if (typeof then == 'function') { - doResolve(bind(then, newValue), self); - return; - } - } - self._state = 1; - self._value = newValue; - finale(self); - } catch (e) { - reject(self, e); - } - } - - function reject(self, newValue) { - self._state = 2; - self._value = newValue; - finale(self); - } - - function finale(self) { - if (self._state === 2 && self._deferreds.length === 0) { - Promise._immediateFn(function() { - if (!self._handled) { - Promise._unhandledRejectionFn(self._value); - } - }); - } - - for (var i = 0, len = self._deferreds.length; i < len; i++) { - handle(self, self._deferreds[i]); - } - self._deferreds = null; - } - - /** - * @constructor - */ - function Handler(onFulfilled, onRejected, promise) { - this.onFulfilled = typeof onFulfilled == 'function' ? onFulfilled : null; - this.onRejected = typeof onRejected == 'function' ? onRejected : null; - this.promise = promise; - } - - /** - * Take a potentially misbehaving resolver function and make sure - * onFulfilled and onRejected are only called once. - * - * Makes no guarantees about asynchrony. - */ - function doResolve(fn, self) { - var done = false; - try { - fn( - function(value) { - if (done) return; - done = true; - resolve(self, value); - }, - function(reason) { - if (done) return; - done = true; - reject(self, reason); - } - ); - } catch (ex) { - if (done) return; - done = true; - reject(self, ex); - } - } - - Promise.prototype['catch'] = function(onRejected) { - return this.then(null, onRejected); - }; - - Promise.prototype.then = function(onFulfilled, onRejected) { - // @ts-ignore - var prom = new this.constructor(noop); - - handle(this, new Handler(onFulfilled, onRejected, prom)); - return prom; - }; - - Promise.all = function(arr) { - return new Promise(function(resolve, reject) { - if (!Array.isArray(arr)) { - return reject(new TypeError('Promise.all accepts an array')); - } - - var args = Array.prototype.slice.call(arr); - if (args.length === 0) return resolve([]); - var remaining = args.length; - - function res(i, val) { - try { - if (val && (typeof val == 'object' || typeof val == 'function')) { - var then = val.then; - if (typeof then == 'function') { - then.call( - val, - function(val) { - res(i, val); - }, - reject - ); - return; - } - } - args[i] = val; - if (--remaining === 0) { - resolve(args); - } - } catch (ex) { - reject(ex); - } - } - - for (var i = 0; i < args.length; i++) { - res(i, args[i]); - } - }); - }; - - Promise.resolve = function(value) { - if (value && typeof value == 'object' && value.constructor == Promise) { - return value; - } - - return new Promise(function(resolve) { - resolve(value); - }); - }; - - Promise.reject = function(value) { - return new Promise(function(resolve, reject) { - reject(value); - }); - }; - - Promise.race = function(arr) { - return new Promise(function(resolve, reject) { - if (!Array.isArray(arr)) { - return reject(new TypeError('Promise.race accepts an array')); - } - - for (var i = 0, len = arr.length; i < len; i++) { - Promise.resolve(arr[i]).then(resolve, reject); - } - }); - }; - - // Use polyfill for setImmediate for performance gains - Promise._immediateFn = - // @ts-ignore - (typeof setImmediate == 'function' && - function(fn) { - // @ts-ignore - setImmediate(fn); - }) || - function(fn) { - setTimeout(fn, 0); // XXX EMSCRIPTEN: just use setTimeout - }; - - Promise._unhandledRejectionFn = function _unhandledRejectionFn(err) { - if (typeof console != 'undefined' && console) { - console.warn('Possible Unhandled Promise Rejection:', err); // eslint-disable-line no-console - } - }; - - return Promise; -})(); - diff --git a/src/postamble.js b/src/postamble.js index 075ed1048e48d..b40dc8992f5f7 100644 --- a/src/postamble.js +++ b/src/postamble.js @@ -26,23 +26,19 @@ if (ENVIRONMENT_IS_WORKER) { {{{ exportRuntime() }}} +#if ASSERTIONS var calledRun; +#endif #if STANDALONE_WASM && MAIN_READS_PARAMS var mainArgs = undefined; #endif -dependenciesFulfilled = function runCaller() { - // If run has never been called, and we should call run (INVOKE_RUN is true, and Module.noInitialRun is not false) - if (!calledRun) run(); - if (!calledRun) dependenciesFulfilled = runCaller; // try this again later, after new deps are fulfilled -}; - #if HAS_MAIN #if MAIN_READS_PARAMS -function callMain(args = []) { +{{{ asyncIf(ASYNCIFY == 2) }}}function callMain(args = []) { #else -function callMain() { +{{{ asyncIf(ASYNCIFY == 2) }}}function callMain() { #endif #if ASSERTIONS assert(runDependencies == 0, 'cannot call main when async dependencies remain! (listen on Module["onRuntimeInitialized"])'); @@ -100,18 +96,12 @@ function callMain() { // The current spec of JSPI returns a promise only if the function suspends // and a plain value otherwise. This will likely change: // https://github.com/WebAssembly/js-promise-integration/issues/11 - Promise.resolve(ret).then((result) => { - exitJS(result, /* implicit = */ true); - }).catch((e) => { - handleException(e); - }); -#else + ret = await ret; +#endif // ASYNCIFY == 2 // if we're not running an evented main loop, it's time to exit exitJS(ret, /* implicit = */ true); -#endif // ASYNCIFY == 2 return ret; - } - catch (e) { + } catch (e) { return handleException(e); } #if ABORT_ON_WASM_EXCEPTIONS @@ -158,28 +148,16 @@ function run() { #if RUNTIME_DEBUG dbg('run() called, but dependencies remain, so not running'); #endif + dependenciesFulfilled = run; return; } -#if WASM_WORKERS - if (ENVIRONMENT_IS_WASM_WORKER) { +#if PTHREADS || WASM_WORKERS + if ({{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) { #if MODULARIZE readyPromiseResolve(Module); -#endif // MODULARIZE - return initRuntime(); - } #endif - -#if PTHREADS - if (ENVIRONMENT_IS_PTHREAD) { -#if MODULARIZE - // The promise resolve function typically gets called as part of the execution - // of `doRun` below. The workers/pthreads don't execute `doRun` so the - // creation promise can be resolved, marking the pthread-Module as initialized. - readyPromiseResolve(Module); -#endif // MODULARIZE initRuntime(); - startWorker(Module); return; } #endif @@ -195,14 +173,17 @@ function run() { #if RUNTIME_DEBUG dbg('run() called, but dependencies remain, so not running'); #endif + dependenciesFulfilled = run; return; } function doRun() { // run may have just been called through dependencies being fulfilled just in this very frame, // or while the async setStatus time below was happening - if (calledRun) return; +#if ASSERTIONS + assert(!calledRun); calledRun = true; +#endif Module['calledRun'] = true; if (ABORT) return; @@ -221,10 +202,11 @@ function run() { #endif #if HAS_MAIN + {{{ makeModuleReceiveWithVar('noInitialRun', undefined, !INVOKE_RUN) }}} #if MAIN_READS_PARAMS - if (shouldRunNow) callMain(args); + if (!noInitialRun) callMain(args); #else - if (shouldRunNow) callMain(); + if (!noInitialRun) callMain(); #endif #else #if ASSERTIONS @@ -317,20 +299,6 @@ if (Module['preInit']) { } #endif -#if HAS_MAIN -// shouldRunNow refers to calling main(), not run(). -#if INVOKE_RUN -var shouldRunNow = true; -#else -var shouldRunNow = false; -#endif - -#if expectToReceiveOnModule('noInitialRun') -if (Module['noInitialRun']) shouldRunNow = false; -#endif - -#endif // HAS_MAIN - run(); #if BUILD_AS_WORKER diff --git a/src/preamble.js b/src/preamble.js index d924c4ab9f8a2..7ba3754e88051 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -219,7 +219,7 @@ function initRuntime() { #endif #if PTHREADS - if (ENVIRONMENT_IS_PTHREAD) return; + if (ENVIRONMENT_IS_PTHREAD) return startWorker(Module); #endif #if STACK_OVERFLOW_CHECK @@ -334,10 +334,10 @@ function addOnPostRun(cb) { // it happens right before run - run will be postponed until // the dependencies are met. var runDependencies = 0; -var runDependencyWatcher = null; var dependenciesFulfilled = null; // overridden to take different actions when all run dependencies are fulfilled #if ASSERTIONS var runDependencyTracking = {}; +var runDependencyWatcher = null; #endif function getUniqueRunDependency(id) { @@ -406,10 +406,12 @@ function removeRunDependency(id) { } #endif if (runDependencies == 0) { +#if ASSERTIONS if (runDependencyWatcher !== null) { clearInterval(runDependencyWatcher); runDependencyWatcher = null; } +#endif if (dependenciesFulfilled) { var callback = dependenciesFulfilled; dependenciesFulfilled = null; @@ -601,8 +603,13 @@ function instrumentWasmTableWithAbort() { } #endif +#if SINGLE_FILE +// In SINGLE_FILE mode the wasm binary is encoded inline here as a data: URL. +var wasmBinaryFile = '{{{ WASM_BINARY_FILE }}}'; +#else +var wasmBinaryFile; function findWasmBinary() { -#if EXPORT_ES6 && USE_ES6_IMPORT_META && !SINGLE_FILE && !AUDIO_WORKLET +#if EXPORT_ES6 && USE_ES6_IMPORT_META && !AUDIO_WORKLET if (Module['locateFile']) { #endif var f = '{{{ WASM_BINARY_FILE }}}'; @@ -612,7 +619,7 @@ function findWasmBinary() { } #endif return f; -#if EXPORT_ES6 && USE_ES6_IMPORT_META && !SINGLE_FILE && !AUDIO_WORKLET // In single-file mode, repeating WASM_BINARY_FILE would emit the contents again. For an Audio Worklet, we cannot use `new URL()`. +#if EXPORT_ES6 && USE_ES6_IMPORT_META && !AUDIO_WORKLET // In single-file mode, repeating WASM_BINARY_FILE would emit the contents again. For an Audio Worklet, we cannot use `new URL()`. } #if ENVIRONMENT_MAY_BE_SHELL if (ENVIRONMENT_IS_SHELL) @@ -622,8 +629,7 @@ function findWasmBinary() { return new URL('{{{ WASM_BINARY_FILE }}}', import.meta.url).href; #endif } - -var wasmBinaryFile; +#endif function getBinarySync(file) { if (file == wasmBinaryFile && wasmBinary) { @@ -645,7 +651,7 @@ function getBinarySync(file) { #endif } -function getBinaryPromise(binaryFile) { +async function getWasmBinary(binaryFile) { #if !SINGLE_FILE // If we don't have the binary yet, load it asynchronously using readAsync. if (!wasmBinary @@ -654,16 +660,17 @@ function getBinaryPromise(binaryFile) { #endif ) { // Fetch the binary using readAsync - return readAsync(binaryFile).then( - (response) => new Uint8Array(/** @type{!ArrayBuffer} */(response)), - // Fall back to getBinarySync if readAsync fails - () => getBinarySync(binaryFile) - ); + try { + var response = await readAsync(binaryFile); + return new Uint8Array(response); + } catch { + // Fall back to getBinarySync below; + } } #endif // Otherwise, getBinarySync should be able to get it synchronously - return Promise.resolve().then(() => getBinarySync(binaryFile)); + return getBinarySync(binaryFile); } #if LOAD_SOURCE_MAP @@ -771,25 +778,18 @@ function resetPrototype(constructor, attrs) { #endif #if WASM_ASYNC_COMPILATION -function instantiateArrayBuffer(binaryFile, imports, receiver) { -#if USE_OFFSET_CONVERTER - var savedBinary; -#endif - return getBinaryPromise(binaryFile).then((binary) => { +async function instantiateArrayBuffer(binaryFile, imports) { + try { + var binary = await getWasmBinary(binaryFile); + var instance = await WebAssembly.instantiate(binary, imports); #if USE_OFFSET_CONVERTER - savedBinary = binary; + // wasmOffsetConverter needs to be assigned before calling resolve. + // See comments below in instantiateAsync. + wasmOffsetConverter = new WasmOffsetConverter(binary, instance.module); #endif - return WebAssembly.instantiate(binary, imports); -#if USE_OFFSET_CONVERTER - }).then((instance) => { - // wasmOffsetConverter needs to be assigned before calling the receiver - // (receiveInstantiationResult). See comments below in instantiateAsync. - wasmOffsetConverter = new WasmOffsetConverter(savedBinary, instance.module); return instance; -#endif - }).then(receiver, (reason) => { + } catch (reason) { err(`failed to asynchronously prepare wasm: ${reason}`); - #if WASM == 2 #if ENVIRONMENT_MAY_BE_NODE || ENVIRONMENT_MAY_BE_SHELL if (typeof location != 'undefined') { @@ -797,11 +797,11 @@ function instantiateArrayBuffer(binaryFile, imports, receiver) { // WebAssembly compilation failed, try running the JS fallback instead. var search = location.search; if (search.indexOf('_rwasm=0') < 0) { + // Reload the page with the `_rwasm=0` argument location.href += (search ? search + '&' : '?') + '_rwasm=0'; - // Return here to avoid calling abort() below. The application - // still has a chance to start successfully do we don't want to - // trigger onAbort or onExit handlers. - return; + // Return a promise that never resolves. We don't want to + // call abort below, or return an error to our caller. + return new Promise(() => {}); } #if ENVIRONMENT_MAY_BE_NODE || ENVIRONMENT_MAY_BE_SHELL } @@ -815,17 +815,17 @@ function instantiateArrayBuffer(binaryFile, imports, receiver) { } #endif abort(reason); - }); + } } -function instantiateAsync(binary, binaryFile, imports, callback) { +async function instantiateAsync(binary, binaryFile, imports) { #if !SINGLE_FILE if (!binary && typeof WebAssembly.instantiateStreaming == 'function' && - !isDataURI(binaryFile) && + !isDataURI(binaryFile) #if ENVIRONMENT_MAY_BE_WEBVIEW // Don't use streaming for file:// delivered objects in a webview, fetch them synchronously. - !isFileURI(binaryFile) && + && !isFileURI(binaryFile) #endif #if ENVIRONMENT_MAY_BE_NODE // Avoid instantiateStreaming() on Node.js environment for now, as while @@ -834,17 +834,15 @@ function instantiateAsync(binary, binaryFile, imports, callback) { // // Reference: // https://github.com/emscripten-core/emscripten/pull/16917 - !ENVIRONMENT_IS_NODE && + && !ENVIRONMENT_IS_NODE #endif - typeof fetch == 'function') { - return fetch(binaryFile, {{{ makeModuleReceiveExpr('fetchSettings', "{ credentials: 'same-origin' }") }}}).then((response) => { - // Suppress closure warning here since the upstream definition for - // instantiateStreaming only allows Promise rather than - // an actual Response. - // TODO(https://github.com/google/closure-compiler/pull/3913): Remove if/when upstream closure is fixed. - /** @suppress {checkTypes} */ - var result = WebAssembly.instantiateStreaming(response, imports); - +#if ENVIRONMENT_MAY_BE_SHELL + // Shell environments don't have fetch. + && !ENVIRONMENT_IS_SHELL +#endif + ) { + try { + var response = fetch(binaryFile, {{{ makeModuleReceiveExpr('fetchSettings', "{ credentials: 'same-origin' }") }}}); #if USE_OFFSET_CONVERTER // We need the wasm binary for the offset converter. Clone the response // in order to get its arrayBuffer (cloning should be more efficient @@ -852,41 +850,35 @@ function instantiateAsync(binary, binaryFile, imports, callback) { // (We must clone the response now in order to use it later, as if we // try to clone it asynchronously lower down then we will get a // "response was already consumed" error.) - var clonedResponsePromise = response.clone().arrayBuffer(); + var clonedResponse = (await response).clone(); #endif - - return result.then( -#if !USE_OFFSET_CONVERTER - callback, -#else - function(instantiationResult) { - // When using the offset converter, we must interpose here. First, - // the instantiation result must arrive (if it fails, the error - // handling later down will handle it). Once it arrives, we can - // initialize the offset converter. And only then is it valid to - // call receiveInstantiationResult, as that function will use the - // offset converter (in the case of pthreads, it will create the - // pthreads and send them the offsets along with the wasm instance). - - clonedResponsePromise.then((arrayBufferResult) => { - wasmOffsetConverter = new WasmOffsetConverter(new Uint8Array(arrayBufferResult), instantiationResult.module); - callback(instantiationResult); - }, - (reason) => err(`failed to initialize offset-converter: ${reason}`) - ); - }, -#endif - function(reason) { - // We expect the most common failure cause to be a bad MIME type for the binary, - // in which case falling back to ArrayBuffer instantiation should work. - err(`wasm streaming compile failed: ${reason}`); - err('falling back to ArrayBuffer instantiation'); - return instantiateArrayBuffer(binaryFile, imports, callback); - }); - }); + var instantiationResult = await WebAssembly.instantiateStreaming(response, imports); +#if USE_OFFSET_CONVERTER + // When using the offset converter, we must interpose here. First, + // the instantiation result must arrive (if it fails, the error + // handling later down will handle it). Once it arrives, we can + // initialize the offset converter. And only then is it valid to + // call receiveInstantiationResult, as that function will use the + // offset converter (in the case of pthreads, it will create the + // pthreads and send them the offsets along with the wasm instance). + var arrayBufferResult = await clonedResponse.arrayBuffer(); + try { + wasmOffsetConverter = new WasmOffsetConverter(new Uint8Array(arrayBufferResult), instantiationResult.module); + } catch (reason) { + err(`failed to initialize offset-converter: ${reason}`); + } +#endif + return instantiationResult; + } catch (reason) { + // We expect the most common failure cause to be a bad MIME type for the binary, + // in which case falling back to ArrayBuffer instantiation should work. + err(`wasm streaming compile failed: ${reason}`); + err('falling back to ArrayBuffer instantiation'); + // fall back of instantiateArrayBuffer below + }; } #endif - return instantiateArrayBuffer(binaryFile, imports, callback); + return instantiateArrayBuffer(binaryFile, imports); } #endif // WASM_ASYNC_COMPILATION @@ -929,7 +921,7 @@ function getWasmImports() { // Create the wasm instance. // Receives the wasm imports, returns the exports. -function createWasm() { +{{{ asyncIf(WASM_ASYNC_COMPILATION) }}}function createWasm() { // Load the wasm module and create an instance of using native support in the JS engine. // handle a generated wasm instance, receiving its exports and // performing other necessary setup @@ -1044,11 +1036,11 @@ function createWasm() { trueModule = null; #endif #if SHARED_MEMORY || RELOCATABLE - receiveInstance(result['instance'], result['module']); + return receiveInstance(result['instance'], result['module']); #else // TODO: Due to Closure regression https://github.com/google/closure-compiler/issues/3193, the above line no longer optimizes out down to the following line. // When the regression is fixed, can restore the above PTHREADS-enabled path. - receiveInstance(result['instance']); + return receiveInstance(result['instance']); #endif } #endif // WASM_ASYNC_COMPILATION @@ -1084,30 +1076,37 @@ function createWasm() { // Instantiate from the module posted from the main thread. // We can just use sync instantiation in the worker. var instance = new WebAssembly.Instance(module, getWasmImports()); - receiveInstance(instance, module); - resolve(); + resolve(receiveInstance(instance, module)); }; }); } #endif +#if !SINGLE_FILE wasmBinaryFile ??= findWasmBinary(); +#endif #if WASM_ASYNC_COMPILATION #if RUNTIME_DEBUG dbg('asynchronously preparing wasm'); #endif #if MODULARIZE - // If instantiation fails, reject the module ready promise. - instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult).catch(readyPromiseReject); -#else - instantiateAsync(wasmBinary, wasmBinaryFile, info, receiveInstantiationResult); + try { #endif + var result = await instantiateAsync(wasmBinary, wasmBinaryFile, info); + var exports = receiveInstantiationResult(result); #if LOAD_SOURCE_MAP - getSourceMapPromise().then(receiveSourceMapJSON); + receiveSourceMapJSON(await getSourceMapAsync()); #endif - return {}; // no exports yet; we'll fill them in later -#else + return exports; +#if MODULARIZE + } catch (e) { + // If instantiation fails, reject the module ready promise. + readyPromiseReject(e); + return Promise.reject(e); + } +#endif +#else // WASM_ASYNC_COMPILATION var result = instantiateSync(wasmBinaryFile, info); #if PTHREADS || MAIN_MODULE return receiveInstance(result[0], result[1]); @@ -1117,7 +1116,7 @@ function createWasm() { // When the regression is fixed, we can remove this if/else. return receiveInstance(result[0]); #endif -#endif +#endif // WASM_ASYNC_COMPILATION } #if !WASM_BIGINT diff --git a/src/runtime_pthread.js b/src/runtime_pthread.js index b9563279b7ba3..484ecb59050dd 100644 --- a/src/runtime_pthread.js +++ b/src/runtime_pthread.js @@ -66,7 +66,7 @@ if (ENVIRONMENT_IS_PTHREAD) { // notified about them. self.onunhandledrejection = (e) => { throw e.reason || e; }; - {{{ asyncIf(ASYNCIFY == 2) }}} function handleMessage(e) { + {{{ asyncIf(ASYNCIFY == 2) }}}function handleMessage(e) { try { var msgData = e['data']; //dbg('msgData: ' + Object.keys(msgData)); @@ -177,7 +177,7 @@ if (ENVIRONMENT_IS_PTHREAD) { } try { - {{{ awaitIf(ASYNCIFY == 2) }}} invokeEntryPoint(msgData.start_routine, msgData.arg); + {{{ awaitIf(ASYNCIFY == 2) }}}invokeEntryPoint(msgData.start_routine, msgData.arg); } catch(ex) { if (ex != 'unwind') { // The pthread "crashed". Do not call `_emscripten_thread_exit` (which diff --git a/src/settings.js b/src/settings.js index e009a98c907e7..c838823a12e18 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1498,7 +1498,7 @@ var DYNCALLS = false; // i64 is used. If WASM_BIGINT is present, the default minimum supported browser // versions will be increased to the min version that supports BigInt. // [link] -var WASM_BIGINT = false; +var WASM_BIGINT = true; // WebAssembly defines a "producers section" which compilers and tools can // annotate themselves in, and LLVM emits this by default. @@ -1919,7 +1919,7 @@ var MIN_SAFARI_VERSION = 140100; // numbers with Chrome. // Chrome 85 was released on 2020-08-25. // MAX_INT (0x7FFFFFFF, or -1) specifies that target is not supported. -// Minimum supported value is 32, which was released on 2014-01-04. +// Minimum supported value is 33, which was released on 2014-02-18. // [link] var MIN_CHROME_VERSION = 85; @@ -2186,14 +2186,6 @@ var LEGACY_RUNTIME = false; // [link] var SIGNATURE_CONVERSIONS = []; -//=========================================== -// Internal, used for testing only, from here -//=========================================== - -// Internal (testing only): Disables the blitOffscreenFramebuffer VAO path. -// [link] -var OFFSCREEN_FRAMEBUFFER_FORBID_VAO_PATH = false; - // For renamed settings the format is: // [OLD_NAME, NEW_NAME] // For removed settings (which now effectively have a fixed value and can no diff --git a/src/shell.js b/src/shell.js index 6d4b2431b247f..f0bd6fba612b4 100644 --- a/src/shell.js +++ b/src/shell.js @@ -34,28 +34,11 @@ var Module = typeof {{{ EXPORT_NAME }}} != 'undefined' ? {{{ EXPORT_NAME }}} : { #endif // USE_CLOSURE_COMPILER #if POLYFILL -#if ((MAYBE_WASM2JS && WASM != 2) || MODULARIZE) && (MIN_CHROME_VERSION < 33 || MIN_FIREFOX_VERSION < 29 || MIN_SAFARI_VERSION < 80000) -// Include a Promise polyfill for legacy browsers. This is needed either for -// wasm2js, where we polyfill the wasm API which needs Promises, or when using -// modularize which creates a Promise for when the module is ready. -// See https://caniuse.com/#feat=promises -#include "polyfill/promise.js" -#endif - -#if MIN_CHROME_VERSION < 45 || MIN_FIREFOX_VERSION < 34 || MIN_SAFARI_VERSION < 90000 -// See https://caniuse.com/mdn-javascript_builtins_object_assign -#include "polyfill/objassign.js" -#endif - -#if WASM_BIGINT && MIN_SAFARI_VERSION < 150000 +#if WASM_BIGINT && MIN_SAFARI_VERSION < 140100 +// TODO(https://github.com/emscripten-core/emscripten/issues/23184): Fix this back to 150000 // See https://caniuse.com/mdn-javascript_builtins_bigint64array #include "polyfill/bigint64array.js" #endif - -#if MIN_CHROME_VERSION < 40 || MIN_FIREFOX_VERSION < 39 || MIN_SAFARI_VERSION < 103000 -// See https://caniuse.com/fetch -#include "polyfill/fetch.js" -#endif #endif // POLYFILL #if MODULARIZE @@ -65,9 +48,6 @@ var readyPromise = new Promise((resolve, reject) => { readyPromiseResolve = resolve; readyPromiseReject = reject; }); -#if ASSERTIONS -{{{ addReadyPromiseAssertions() }}} -#endif #endif // Determine the runtime environment we are in. You can customize this by @@ -121,22 +101,12 @@ if (ENVIRONMENT_IS_PTHREAD) { #if ENVIRONMENT_MAY_BE_NODE if (ENVIRONMENT_IS_NODE) { - // `require()` is no-op in an ESM module, use `createRequire()` to construct - // the require()` function. This is only necessary for multi-environment - // builds, `-sENVIRONMENT=node` emits a static import declaration instead. - // TODO: Swap all `require()`'s with `import()`'s? -#if EXPORT_ES6 && ENVIRONMENT_MAY_BE_WEB - const { createRequire } = await import('module'); - let dirname = import.meta.url; - if (dirname.startsWith("data:")) { - dirname = '/'; - } - /** @suppress{duplicate} */ - var require = createRequire(dirname); -#endif - #if PTHREADS || WASM_WORKERS +#if EXPORT_ES6 + var worker_threads = await import('worker_threads'); +#else var worker_threads = require('worker_threads'); +#endif global.Worker = worker_threads.Worker; ENVIRONMENT_IS_WORKER = !worker_threads.isMainThread; #if PTHREADS @@ -221,19 +191,21 @@ if (ENVIRONMENT_IS_NODE) { } #endif - // These modules will usually be used on Node.js. Load them eagerly to avoid - // the complexity of lazy-loading. - var fs = require('fs'); - var nodePath = require('path'); - #if EXPORT_ES6 + var fs = await import('fs'); + var nodePath = await import('path'); + var url = await import('url'); // EXPORT_ES6 + ENVIRONMENT_IS_NODE always requires use of import.meta.url, // since there's no way getting the current absolute path of the module when // support for that is not available. if (!import.meta.url.startsWith('data:')) { - scriptDirectory = nodePath.dirname(require('url').fileURLToPath(import.meta.url)) + '/'; + scriptDirectory = nodePath.dirname(url.fileURLToPath(import.meta.url)) + '/'; } #else + // These modules will usually be used on Node.js. Load them eagerly to avoid + // the complexity of lazy-loading. + var fs = require('fs'); + var nodePath = require('path'); scriptDirectory = __dirname + '/'; #endif @@ -308,11 +280,7 @@ if (ENVIRONMENT_IS_SHELL) { return data; }; - readAsync = (f) => { - return new Promise((resolve, reject) => { - setTimeout(() => resolve(readBinary(f))); - }); - }; + readAsync = async (f) => readBinary(f); globalThis.clearTimeout ??= (id) => {}; diff --git a/src/shell_minimal.js b/src/shell_minimal.js index 3c879ffb9383e..4ddf6b3b0303c 100644 --- a/src/shell_minimal.js +++ b/src/shell_minimal.js @@ -41,9 +41,6 @@ var readyPromise = new Promise((resolve, reject) => { readyPromiseResolve = resolve; readyPromiseReject = reject; }); -#if ASSERTIONS -{{{ addReadyPromiseAssertions() }}} -#endif #endif #if ENVIRONMENT_MAY_BE_NODE @@ -128,13 +125,6 @@ function ready() { #endif } -#if POLYFILL -// See https://caniuse.com/mdn-javascript_builtins_object_assign -#if MIN_CHROME_VERSION < 45 || MIN_FIREFOX_VERSION < 34 || MIN_SAFARI_VERSION < 90000 -#include "polyfill/objassign.js" -#endif -#endif - #if PTHREADS // MINIMAL_RUNTIME does not support --proxy-to-worker option, so Worker and Pthread environments // coincide. diff --git a/src/source_map_support.js b/src/source_map_support.js index 713cf017e65be..84e1bfee66fe1 100644 --- a/src/source_map_support.js +++ b/src/source_map_support.js @@ -106,11 +106,14 @@ function getSourceMap() { return JSON.parse(UTF8ArrayToString(buf)); } -function getSourceMapPromise() { - if ((ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) && typeof fetch == 'function') { - return fetch(wasmSourceMapFile, {{{ makeModuleReceiveExpr('fetchSettings', "{ credentials: 'same-origin' }") }}}) - .then((response) => response.json()) - .catch(getSourceMap); +async function getSourceMapAsync() { + if (ENVIRONMENT_IS_WEB || ENVIRONMENT_IS_WORKER) { + try { + var response = await fetch(wasmSourceMapFile, {{{ makeModuleReceiveExpr('fetchSettings', "{ credentials: 'same-origin' }") }}}); + return response.json(); + } catch { + // Fall back to getSourceMap below + } } - return Promise.resolve(getSourceMap()); + return getSourceMap(); } diff --git a/src/web_or_worker_shell_read.js b/src/web_or_worker_shell_read.js index fbc97918f2501..e24d858c4f1df 100644 --- a/src/web_or_worker_shell_read.js +++ b/src/web_or_worker_shell_read.js @@ -16,7 +16,7 @@ } #endif - readAsync = (url) => { + readAsync = async (url) => { #if ENVIRONMENT_MAY_BE_WEBVIEW // Fetch has some additional restrictions over XHR, like it can't be used on a file:// url. // See https://github.com/github/fetch/pull/92#issuecomment-140665932 @@ -41,11 +41,9 @@ #elif ASSERTIONS assert(!isFileURI(url), "readAsync does not work with file:// URLs"); #endif - return fetch(url, {{{ makeModuleReceiveExpr('fetchSettings', "{ credentials: 'same-origin' }") }}}) - .then((response) => { - if (response.ok) { - return response.arrayBuffer(); - } - return Promise.reject(new Error(response.status + ' : ' + response.url)); - }) + var response = await fetch(url, {{{ makeModuleReceiveExpr('fetchSettings', "{ credentials: 'same-origin' }") }}}); + if (response.ok) { + return response.arrayBuffer(); + } + throw new Error(response.status + ' : ' + response.url); }; diff --git a/system/include/compat/avxintrin.h b/system/include/compat/avxintrin.h index 09c9a3e8bf6a0..45171ebbe5e6e 100644 --- a/system/include/compat/avxintrin.h +++ b/system/include/compat/avxintrin.h @@ -40,7 +40,7 @@ typedef struct { __m128i_u v1; } __m256i_u; -union m256_data { +union __m256_data { __m256i int_view; __m256d double_view; __m256 float_view; @@ -1771,42 +1771,42 @@ _mm256_setzero_si256(void) { static __inline__ __m256 __attribute__((__always_inline__, __nodebug__)) _mm256_castpd_ps(__m256d __a) { - m256_data ret; + union __m256_data ret; ret.double_view = __a; return ret.float_view; } static __inline__ __m256i __attribute__((__always_inline__, __nodebug__)) _mm256_castpd_si256(__m256d __a) { - m256_data ret; + union __m256_data ret; ret.double_view = __a; return ret.int_view; } static __inline__ __m256d __attribute__((__always_inline__, __nodebug__)) _mm256_castps_pd(__m256 __a) { - m256_data ret; + union __m256_data ret; ret.float_view = __a; return ret.double_view; } static __inline__ __m256i __attribute__((__always_inline__, __nodebug__)) _mm256_castps_si256(__m256 __a) { - m256_data ret; + union __m256_data ret; ret.float_view = __a; return ret.int_view; } static __inline__ __m256 __attribute__((__always_inline__, __nodebug__)) _mm256_castsi256_ps(__m256i __a) { - m256_data ret; + union __m256_data ret; ret.int_view = __a; return ret.float_view; } static __inline__ __m256d __attribute__((__always_inline__, __nodebug__)) _mm256_castsi256_pd(__m256i __a) { - m256_data ret; + union __m256_data ret; ret.int_view = __a; return ret.double_view; } diff --git a/system/include/emscripten/em_math.h b/system/include/emscripten/em_math.h index 63433bbf31b13..2c44700198b53 100644 --- a/system/include/emscripten/em_math.h +++ b/system/include/emscripten/em_math.h @@ -29,7 +29,7 @@ extern "C" { // Math.ceil -> f32.ceil and f64.ceil (ceil() and ceilf() in math.h) // Math.clz32(x) -> i32.clz and i64.clz (call __builtin_clz() and __builtin_clzll()) // Math.floor -> f32.floor and f64.floor (floor() and floorf() in math.h) -// Math.fround -> f64.promote_f32(f32.demote_f64()) (call double d = (double)(float)someDouble;) +// Math.fround -> f64.promote_f32(f32.demote_f64()) (double d = (double)(float)someDouble;) // Math.imul(x, y) -> i32.mul and i64.mul (directly multiply two signed integers) // Math.min -> f32.min and f64.min (fminf() and fmin() in math.h) // Math.max -> f32.max and f64.max (fmaxf() and fmax() in math.h) diff --git a/system/include/emscripten/heap.h b/system/include/emscripten/heap.h index 0bd56cdbb32db..26041d6be2be2 100644 --- a/system/include/emscripten/heap.h +++ b/system/include/emscripten/heap.h @@ -46,6 +46,7 @@ size_t emscripten_get_heap_max(void); // dlmalloc and emmalloc. void *emscripten_builtin_memalign(size_t alignment, size_t size); void *emscripten_builtin_malloc(size_t size); +void *emscripten_builtin_realloc(void *ptr, size_t size); void *emscripten_builtin_calloc(size_t nmemb, size_t size); void emscripten_builtin_free(void *ptr); diff --git a/system/include/emscripten/webaudio.h b/system/include/emscripten/webaudio.h index 4ec2882cdce53..624a13538865a 100644 --- a/system/include/emscripten/webaudio.h +++ b/system/include/emscripten/webaudio.h @@ -62,7 +62,7 @@ void emscripten_destroy_web_audio_node(EMSCRIPTEN_WEBAUDIO_T objectHandle); // Create Wasm AudioWorklet thread. Call this function once at application startup to establish an AudioWorkletGlobalScope for your app. // After the scope has been initialized, the given callback will fire. // audioContext: The Web Audio context object to initialize the Wasm AudioWorklet thread on. Each AudioContext can have only one AudioWorklet -// thread running, so do not call this function a multiple times on the same AudioContext. +// thread running, so do not call this function multiple times on the same AudioContext. // stackLowestAddress: The base address for the thread's stack. Must be aligned to 16 bytes. Use e.g. memalign(16, 1024) to allocate a 1KB stack for the thread. // stackSize: The size of the thread's stack. Must be a multiple of 16 bytes. // callback: The callback function that will be run when thread creation either succeeds or fails. @@ -76,8 +76,8 @@ typedef int WEBAUDIO_PARAM_AUTOMATION_RATE; typedef struct WebAudioParamDescriptor { float defaultValue; // Default == 0.0 - float minValue; // Default = -3.4028235e38; - float maxValue; // Default = 3.4028235e38; + float minValue; // Default = -3.4028235e38 + float maxValue; // Default = 3.4028235e38 WEBAUDIO_PARAM_AUTOMATION_RATE automationRate; // Either WEBAUDIO_PARAM_A_RATE or WEBAUDIO_PARAM_K_RATE. Default = WEBAUDIO_PARAM_A_RATE } WebAudioParamDescriptor; @@ -96,7 +96,7 @@ typedef void (*EmscriptenWorkletProcessorCreatedCallback)(EMSCRIPTEN_WEBAUDIO_T void emscripten_create_wasm_audio_worklet_processor_async(EMSCRIPTEN_WEBAUDIO_T audioContext, const WebAudioWorkletProcessorCreateOptions *options, EmscriptenWorkletProcessorCreatedCallback callback, void *userData3); // Returns the number of samples processed per channel in an AudioSampleFrame, fixed at 128 in the Web Audio API 1.0 specification, and valid for the lifetime of the audio context. -// For this to change from the default 128, the context would need creating with a yet unexposed WebAudioWorkletProcessorCreateOptions renderSizeHint, part of the 1.1 Web Audio API. +// For this to change from the default 128, the context would need to be created with a yet unexposed WebAudioWorkletProcessorCreateOptions renderSizeHint, part of the 1.1 Web Audio API. int emscripten_audio_context_quantum_size(EMSCRIPTEN_WEBAUDIO_T audioContext); typedef int EMSCRIPTEN_AUDIO_WORKLET_NODE_T; @@ -148,15 +148,15 @@ bool emscripten_current_thread_is_audio_worklet(void); #define EMSCRIPTEN_AUDIO_MAIN_THREAD 0 -/* emscripten_audio_worklet_function_*: Post a pointer to a C/C++ function to be executed either - on the Audio Worklet thread of the given Web Audio context. Notes: +/* emscripten_audio_worklet_function_*: Post a pointer to a C/C++ function to be executed on the Audio Worklet + thread of the given Web Audio context. Notes: - If running inside an Audio Worklet thread, specify ID EMSCRIPTEN_AUDIO_MAIN_THREAD (== 0) to pass a message from the audio worklet to the main thread. - When specifying non-zero ID, the Audio Context denoted by the ID must have been created by the calling thread. - Passing messages between audio thread and main thread with this family of functions is relatively slow and has a really high latency cost compared to direct coordination using atomics and synchronization primitives like - mutexes and synchronization primitives. Additionally these functions will generate garbage on the JS heap. - Therefore avoid using these functions where performance is critical. */ + mutexes. Additionally these functions will generate garbage on the JS heap. Therefore avoid using these + functions where performance is critical. */ void emscripten_audio_worklet_post_function_v(EMSCRIPTEN_WEBAUDIO_T id, void (*funcPtr)(void)); void emscripten_audio_worklet_post_function_vi(EMSCRIPTEN_WEBAUDIO_T id, void (*funcPtr)(int), int arg0); void emscripten_audio_worklet_post_function_vii(EMSCRIPTEN_WEBAUDIO_T id, void (*funcPtr)(int, int), int arg0, int arg1); diff --git a/system/lib/compiler-rt/include/sanitizer/allocator_interface.h b/system/lib/compiler-rt/include/sanitizer/allocator_interface.h index a792e9f0136e6..1696a92681e35 100644 --- a/system/lib/compiler-rt/include/sanitizer/allocator_interface.h +++ b/system/lib/compiler-rt/include/sanitizer/allocator_interface.h @@ -62,13 +62,20 @@ size_t SANITIZER_CDECL __sanitizer_get_free_bytes(void); size_t SANITIZER_CDECL __sanitizer_get_unmapped_bytes(void); /* Malloc hooks that may be optionally provided by user. - __sanitizer_malloc_hook(ptr, size) is called immediately after - allocation of "size" bytes, which returned "ptr". - __sanitizer_free_hook(ptr) is called immediately before - deallocation of "ptr". */ + - __sanitizer_malloc_hook(ptr, size) is called immediately after allocation + of "size" bytes, which returned "ptr". + - __sanitizer_free_hook(ptr) is called immediately before deallocation of + "ptr". + - __sanitizer_ignore_free_hook(ptr) is called immediately before deallocation + of "ptr", and if it returns a non-zero value, the deallocation of "ptr" + will not take place. This allows software to make free a no-op until it + calls free() again in the same pointer at a later time. Hint: read this as + "ignore the free" rather than "ignore the hook". +*/ void SANITIZER_CDECL __sanitizer_malloc_hook(const volatile void *ptr, size_t size); void SANITIZER_CDECL __sanitizer_free_hook(const volatile void *ptr); +int SANITIZER_CDECL __sanitizer_ignore_free_hook(const volatile void *ptr); /* Installs a pair of hooks for malloc/free. Several (currently, 5) hook pairs may be installed, they are executed diff --git a/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h b/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h index 3f3f1e78dfb85..5f262455cb946 100644 --- a/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h +++ b/system/lib/compiler-rt/include/sanitizer/linux_syscall_hooks.h @@ -1856,6 +1856,15 @@ __sanitizer_syscall_pre_impl_sigaltstack((long)ss, (long)oss) #define __sanitizer_syscall_post_sigaltstack(res, ss, oss) \ __sanitizer_syscall_post_impl_sigaltstack(res, (long)ss, (long)oss) +#define __sanitizer_syscall_pre_futex(uaddr, futex_op, val, timeout, uaddr2, \ + val3) \ + __sanitizer_syscall_pre_impl_futex((long)uaddr, (long)futex_op, (long)val, \ + (long)timeout, (long)uaddr2, (long)val3) +#define __sanitizer_syscall_post_futex(res, uaddr, futex_op, val, timeout, \ + uaddr2, val3) \ + __sanitizer_syscall_post_impl_futex(res, (long)uaddr, (long)futex_op, \ + (long)val, (long)timeout, (long)uaddr2, \ + (long)val3) // And now a few syscalls we don't handle yet. #define __sanitizer_syscall_pre_afs_syscall(...) @@ -1875,7 +1884,6 @@ #define __sanitizer_syscall_pre_fchown32(...) #define __sanitizer_syscall_pre_ftime(...) #define __sanitizer_syscall_pre_ftruncate64(...) -#define __sanitizer_syscall_pre_futex(...) #define __sanitizer_syscall_pre_getegid32(...) #define __sanitizer_syscall_pre_geteuid32(...) #define __sanitizer_syscall_pre_getgid32(...) @@ -1954,7 +1962,6 @@ #define __sanitizer_syscall_post_fchown32(res, ...) #define __sanitizer_syscall_post_ftime(res, ...) #define __sanitizer_syscall_post_ftruncate64(res, ...) -#define __sanitizer_syscall_post_futex(res, ...) #define __sanitizer_syscall_post_getegid32(res, ...) #define __sanitizer_syscall_post_geteuid32(res, ...) #define __sanitizer_syscall_post_getgid32(res, ...) @@ -3093,6 +3100,11 @@ void __sanitizer_syscall_post_impl_rt_sigaction(long res, long signum, long act, long oldact, long sz); void __sanitizer_syscall_pre_impl_sigaltstack(long ss, long oss); void __sanitizer_syscall_post_impl_sigaltstack(long res, long ss, long oss); +void __sanitizer_syscall_pre_impl_futex(long uaddr, long futex_op, long val, + long timeout, long uaddr2, long val3); +void __sanitizer_syscall_post_impl_futex(long res, long uaddr, long futex_op, + long val, long timeout, long uaddr2, + long val3); #ifdef __cplusplus } // extern "C" #endif diff --git a/system/lib/compiler-rt/include/sanitizer/nsan_interface.h b/system/lib/compiler-rt/include/sanitizer/nsan_interface.h new file mode 100644 index 0000000000000..057ca0473bb3c --- /dev/null +++ b/system/lib/compiler-rt/include/sanitizer/nsan_interface.h @@ -0,0 +1,75 @@ +//===-- sanitizer/nsan_interface.h ------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Public interface for nsan. +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_NSAN_INTERFACE_H +#define SANITIZER_NSAN_INTERFACE_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/// User-provided default option settings. +/// +/// You can provide your own implementation of this function to return a string +/// containing NSan runtime options (for example, +/// verbosity=1:halt_on_error=0). +/// +/// \returns Default options string. +const char *__nsan_default_options(void); + +// Dumps nsan shadow data for a block of `size_bytes` bytes of application +// memory at location `addr`. +// +// Each line contains application address, shadow types, then values. +// Unknown types are shown as `__`, while known values are shown as +// `f`, `d`, `l` for float, double, and long double respectively. Position is +// shown as a single hex digit. The shadow value itself appears on the line that +// contains the first byte of the value. +// FIXME: Show both shadow and application value. +// +// Example: `__nsan_dump_shadow_mem(addr, 32, 8, 0)` might print: +// +// 0x0add7359: __ f0 f1 f2 f3 __ __ __ (42.000) +// 0x0add7361: __ d1 d2 d3 d4 d5 d6 d7 +// 0x0add7369: d8 f0 f1 f2 f3 __ __ f2 (-1.000) (12.5) +// 0x0add7371: f3 __ __ __ __ __ __ __ +// +// This means that there is: +// - a shadow double for the float at address 0x0add7360, with value 42; +// - a shadow float128 for the double at address 0x0add7362, with value -1; +// - a shadow double for the float at address 0x0add736a, with value 12.5; +// There was also a shadow double for the float at address 0x0add736e, but bytes +// f0 and f1 were overwritten by one or several stores, so that the shadow value +// is no longer valid. +// The argument `reserved` can be any value. Its true value is provided by the +// instrumentation. +void __nsan_dump_shadow_mem(const char *addr, size_t size_bytes, + size_t bytes_per_line, size_t reserved); + +// Explicitly dumps a value. +// FIXME: vector versions ? +void __nsan_dump_float(float value); +void __nsan_dump_double(double value); +void __nsan_dump_longdouble(long double value); + +// Explicitly checks a value. +// FIXME: vector versions ? +void __nsan_check_float(float value); +void __nsan_check_double(double value); +void __nsan_check_longdouble(long double value); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // SANITIZER_NSAN_INTERFACE_H diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp index 22dcf6132707b..9e66f77217ec6 100644 --- a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp @@ -717,7 +717,15 @@ struct Allocator { return; } - RunFreeHooks(ptr); + if (RunFreeHooks(ptr)) { + // Someone used __sanitizer_ignore_free_hook() and decided that they + // didn't want the memory to __sanitizer_ignore_free_hook freed right now. + // When they call free() on this pointer again at a later time, we should + // ignore the alloc-type mismatch and allow them to deallocate the pointer + // through free(), rather than the initial alloc type. + m->alloc_type = FROM_MALLOC; + return; + } // Must mark the chunk as quarantined before any changes to its metadata. // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag. diff --git a/system/lib/compiler-rt/lib/asan/asan_descriptions.cpp b/system/lib/compiler-rt/lib/asan/asan_descriptions.cpp index ef6f3e0a096f8..1c2f20a76343b 100644 --- a/system/lib/compiler-rt/lib/asan/asan_descriptions.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_descriptions.cpp @@ -245,11 +245,11 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr, InternalScopedString str; str.AppendF(" [%zd, %zd)", var.beg, var_end); // Render variable name. - str.AppendF(" '"); + str.Append(" '"); for (uptr i = 0; i < var.name_len; ++i) { str.AppendF("%c", var.name_pos[i]); } - str.AppendF("'"); + str.Append("'"); if (var.line > 0) { str.AppendF(" (line %zd)", var.line); } @@ -260,7 +260,7 @@ static void PrintAccessAndVarIntersection(const StackVarDescr &var, uptr addr, str.AppendF("%s <== Memory access at offset %zd %s this variable%s\n", d.Location(), addr, pos_descr, d.Default()); } else { - str.AppendF("\n"); + str.Append("\n"); } Printf("%s", str.data()); } @@ -292,7 +292,7 @@ static void DescribeAddressRelativeToGlobal(uptr addr, uptr access_size, str.AppendF(" global variable '%s' defined in '", MaybeDemangleGlobalName(g.name)); PrintGlobalLocation(&str, g, /*print_module_name=*/false); - str.AppendF("' (0x%zx) of size %zu\n", g.beg, g.size); + str.AppendF("' (%p) of size %zu\n", (void *)g.beg, g.size); str.Append(d.Default()); PrintGlobalNameIfASCII(&str, g); Printf("%s", str.data()); diff --git a/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp b/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp index 12625e9d75833..dbc4342e83388 100644 --- a/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_fuchsia.cpp @@ -57,8 +57,6 @@ void AsanCheckDynamicRTPrereqs() {} void AsanCheckIncompatibleRT() {} void InitializeAsanInterceptors() {} -void *AsanDoesNotSupportStaticLinkage() { return nullptr; } - void InitializePlatformExceptionHandlers() {} void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { UNIMPLEMENTED(); diff --git a/system/lib/compiler-rt/lib/asan/asan_globals.cpp b/system/lib/compiler-rt/lib/asan/asan_globals.cpp index 083bfeed99945..17289937b4293 100644 --- a/system/lib/compiler-rt/lib/asan/asan_globals.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_globals.cpp @@ -345,8 +345,8 @@ void __asan_unregister_image_globals(uptr *flag) { } void __asan_register_elf_globals(uptr *flag, void *start, void *stop) { - if (*flag) return; - if (!start) return; + if (*flag || start == stop) + return; CHECK_EQ(0, ((uptr)stop - (uptr)start) % sizeof(__asan_global)); __asan_global *globals_start = (__asan_global*)start; __asan_global *globals_stop = (__asan_global*)stop; @@ -355,8 +355,8 @@ void __asan_register_elf_globals(uptr *flag, void *start, void *stop) { } void __asan_unregister_elf_globals(uptr *flag, void *start, void *stop) { - if (!*flag) return; - if (!start) return; + if (!*flag || start == stop) + return; CHECK_EQ(0, ((uptr)stop - (uptr)start) % sizeof(__asan_global)); __asan_global *globals_start = (__asan_global*)start; __asan_global *globals_stop = (__asan_global*)stop; diff --git a/system/lib/compiler-rt/lib/asan/asan_globals_win.cpp b/system/lib/compiler-rt/lib/asan/asan_globals_win.cpp index 19af88ab12b40..9442cc35d5ab7 100644 --- a/system/lib/compiler-rt/lib/asan/asan_globals_win.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_globals_win.cpp @@ -17,10 +17,10 @@ namespace __asan { #pragma section(".ASAN$GA", read, write) #pragma section(".ASAN$GZ", read, write) -extern "C" __declspec(allocate(".ASAN$GA")) - ALIGNED(sizeof(__asan_global)) __asan_global __asan_globals_start = {}; -extern "C" __declspec(allocate(".ASAN$GZ")) - ALIGNED(sizeof(__asan_global)) __asan_global __asan_globals_end = {}; +extern "C" alignas(sizeof(__asan_global)) + __declspec(allocate(".ASAN$GA")) __asan_global __asan_globals_start = {}; +extern "C" alignas(sizeof(__asan_global)) + __declspec(allocate(".ASAN$GZ")) __asan_global __asan_globals_end = {}; #pragma comment(linker, "/merge:.ASAN=.data") static void call_on_globals(void (*hook)(__asan_global *, uptr)) { diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp index 4d6d395616b7a..194c506a3320d 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp @@ -333,7 +333,7 @@ INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret, } # endif -DEFINE_REAL_PTHREAD_FUNCTIONS +DEFINE_INTERNAL_PTHREAD_FUNCTIONS #endif // ASAN_INTERCEPT_PTHREAD_CREATE #if ASAN_INTERCEPT_SWAPCONTEXT @@ -352,8 +352,16 @@ static void ClearShadowMemoryForContextStack(uptr stack, uptr ssize) { PoisonShadow(bottom, ssize, 0); } +// Since Solaris 10/SPARC, ucp->uc_stack.ss_sp refers to the stack base address +// as on other targets. For binary compatibility, the new version uses a +// different external name, so we intercept that. +# if SANITIZER_SOLARIS && defined(__sparc__) +INTERCEPTOR(void, __makecontext_v2, struct ucontext_t *ucp, void (*func)(), + int argc, ...) { +# else INTERCEPTOR(void, makecontext, struct ucontext_t *ucp, void (*func)(), int argc, ...) { +# endif va_list ap; uptr args[64]; // We don't know a better way to forward ... into REAL function. We can @@ -373,7 +381,11 @@ INTERCEPTOR(void, makecontext, struct ucontext_t *ucp, void (*func)(), int argc, ENUMERATE_ARRAY_16(0), ENUMERATE_ARRAY_16(16), ENUMERATE_ARRAY_16(32), \ ENUMERATE_ARRAY_16(48) +# if SANITIZER_SOLARIS && defined(__sparc__) + REAL(__makecontext_v2) +# else REAL(makecontext) +# endif ((struct ucontext_t *)ucp, func, argc, ENUMERATE_ARRAY_64()); # undef ENUMERATE_ARRAY_4 @@ -558,6 +570,17 @@ INTERCEPTOR(char *, strcpy, char *to, const char *from) { return REAL(strcpy)(to, from); } +// Windows doesn't always define the strdup identifier, +// and when it does it's a macro defined to either _strdup +// or _strdup_dbg, _strdup_dbg ends up calling _strdup, so +// we want to intercept that. push/pop_macro are used to avoid problems +// if this file ends up including in the future. +# if SANITIZER_WINDOWS +# pragma push_macro("strdup") +# undef strdup +# define strdup _strdup +# endif + INTERCEPTOR(char*, strdup, const char *s) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, strdup); @@ -575,7 +598,7 @@ INTERCEPTOR(char*, strdup, const char *s) { return reinterpret_cast(new_mem); } -#if ASAN_INTERCEPT___STRDUP +# if ASAN_INTERCEPT___STRDUP INTERCEPTOR(char*, __strdup, const char *s) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, strdup); @@ -724,7 +747,7 @@ INTERCEPTOR(int, atexit, void (*func)()) { extern "C" { extern int _pthread_atfork(void (*prepare)(), void (*parent)(), void (*child)()); -}; +} INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(), void (*child)()) { @@ -738,8 +761,8 @@ INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(), #endif #if ASAN_INTERCEPT_VFORK -DEFINE_REAL(int, vfork) -DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork) +DEFINE_REAL(int, vfork,) +DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork,) #endif // ---------------------- InitializeAsanInterceptors ---------------- {{{1 @@ -758,7 +781,7 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(strncat); ASAN_INTERCEPT_FUNC(strncpy); ASAN_INTERCEPT_FUNC(strdup); -#if ASAN_INTERCEPT___STRDUP +# if ASAN_INTERCEPT___STRDUP ASAN_INTERCEPT_FUNC(__strdup); #endif #if ASAN_INTERCEPT_INDEX && ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX @@ -780,7 +803,12 @@ void InitializeAsanInterceptors() { # if ASAN_INTERCEPT_SWAPCONTEXT ASAN_INTERCEPT_FUNC(swapcontext); + // See the makecontext interceptor above for an explanation. +# if SANITIZER_SOLARIS && defined(__sparc__) + ASAN_INTERCEPT_FUNC(__makecontext_v2); +# else ASAN_INTERCEPT_FUNC(makecontext); +# endif # endif # if ASAN_INTERCEPT__LONGJMP ASAN_INTERCEPT_FUNC(_longjmp); @@ -849,6 +877,10 @@ void InitializeAsanInterceptors() { VReport(1, "AddressSanitizer: libc interceptors initialized\n"); } +# if SANITIZER_WINDOWS +# pragma pop_macro("strdup") +# endif + } // namespace __asan #endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_EMSCRIPTEN diff --git a/system/lib/compiler-rt/lib/asan/asan_internal.h b/system/lib/compiler-rt/lib/asan/asan_internal.h index 2944ebe213b5d..06dfc4b177339 100644 --- a/system/lib/compiler-rt/lib/asan/asan_internal.h +++ b/system/lib/compiler-rt/lib/asan/asan_internal.h @@ -80,7 +80,6 @@ void ReplaceSystemMalloc(); // asan_linux.cpp / asan_mac.cpp / asan_win.cpp uptr FindDynamicShadowStart(); -void *AsanDoesNotSupportStaticLinkage(); void AsanCheckDynamicRTPrereqs(); void AsanCheckIncompatibleRT(); diff --git a/system/lib/compiler-rt/lib/asan/asan_linux.cpp b/system/lib/compiler-rt/lib/asan/asan_linux.cpp index 37d3bad1b1ec6..0b470db86748f 100644 --- a/system/lib/compiler-rt/lib/asan/asan_linux.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_linux.cpp @@ -47,15 +47,12 @@ # if SANITIZER_ANDROID || SANITIZER_FREEBSD || SANITIZER_SOLARIS # include -extern "C" void *_DYNAMIC; # elif SANITIZER_NETBSD # include # include -extern Elf_Dyn _DYNAMIC; # else # include # include -extern ElfW(Dyn) _DYNAMIC[]; # endif typedef enum { @@ -76,11 +73,6 @@ void InitializePlatformInterceptors() {} void InitializePlatformExceptionHandlers() {} bool IsSystemHeapAddress(uptr addr) { return false; } -void *AsanDoesNotSupportStaticLinkage() { - // This will fail to link with -static. - return &_DYNAMIC; -} - # if ASAN_PREMAP_SHADOW uptr FindPremappedShadowStart(uptr shadow_size_bytes) { uptr granularity = GetMmapGranularity(); @@ -101,7 +93,8 @@ uptr FindDynamicShadowStart() { # endif return MapDynamicShadow(shadow_size_bytes, ASAN_SHADOW_SCALE, - /*min_shadow_base_alignment*/ 0, kHighMemEnd); + /*min_shadow_base_alignment*/ 0, kHighMemEnd, + GetMmapGranularity()); } void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { diff --git a/system/lib/compiler-rt/lib/asan/asan_mac.cpp b/system/lib/compiler-rt/lib/asan/asan_mac.cpp index 1b0e9b3fe0060..bfc349223258b 100644 --- a/system/lib/compiler-rt/lib/asan/asan_mac.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_mac.cpp @@ -49,14 +49,10 @@ void InitializePlatformInterceptors() {} void InitializePlatformExceptionHandlers() {} bool IsSystemHeapAddress (uptr addr) { return false; } -// No-op. Mac does not support static linkage anyway. -void *AsanDoesNotSupportStaticLinkage() { - return 0; -} - uptr FindDynamicShadowStart() { return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE, - /*min_shadow_base_alignment*/ 0, kHighMemEnd); + /*min_shadow_base_alignment*/ 0, kHighMemEnd, + GetMmapGranularity()); } // No-op. Mac does not support static linkage anyway. diff --git a/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp b/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp index 40cec4365b964..84f146d99d82d 100644 --- a/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_malloc_linux.cpp @@ -185,11 +185,11 @@ struct MallocDebugL { void* (*valloc)(uptr size); }; -ALIGNED(32) const MallocDebugK asan_malloc_dispatch_k = { +alignas(32) const MallocDebugK asan_malloc_dispatch_k = { WRAP(malloc), WRAP(free), WRAP(calloc), WRAP(realloc), WRAP(memalign), WRAP(malloc_usable_size)}; -ALIGNED(32) const MallocDebugL asan_malloc_dispatch_l = { +alignas(32) const MallocDebugL asan_malloc_dispatch_l = { WRAP(calloc), WRAP(free), WRAP(mallinfo), WRAP(malloc), WRAP(malloc_usable_size), WRAP(memalign), WRAP(posix_memalign), WRAP(pvalloc), WRAP(realloc), diff --git a/system/lib/compiler-rt/lib/asan/asan_mapping.h b/system/lib/compiler-rt/lib/asan/asan_mapping.h index 81173be01e2c4..1d0f8131338ed 100644 --- a/system/lib/compiler-rt/lib/asan/asan_mapping.h +++ b/system/lib/compiler-rt/lib/asan/asan_mapping.h @@ -72,7 +72,10 @@ // || `[0x2000000000, 0x23ffffffff]` || LowShadow || // || `[0x0000000000, 0x1fffffffff]` || LowMem || // -// Default Linux/RISCV64 Sv39 mapping: +// Default Linux/RISCV64 Sv39 mapping with SHADOW_OFFSET == 0xd55550000; +// (the exact location of SHADOW_OFFSET may vary depending the dynamic probing +// by FindDynamicShadowStart). +// // || `[0x1555550000, 0x3fffffffff]` || HighMem || // || `[0x0fffffa000, 0x1555555fff]` || HighShadow || // || `[0x0effffa000, 0x0fffff9fff]` || ShadowGap || @@ -186,7 +189,7 @@ # elif SANITIZER_FREEBSD && defined(__aarch64__) # define ASAN_SHADOW_OFFSET_CONST 0x0000800000000000 # elif SANITIZER_RISCV64 -# define ASAN_SHADOW_OFFSET_CONST 0x0000000d55550000 +# define ASAN_SHADOW_OFFSET_DYNAMIC # elif defined(__aarch64__) # define ASAN_SHADOW_OFFSET_CONST 0x0000001000000000 # elif defined(__powerpc64__) diff --git a/system/lib/compiler-rt/lib/asan/asan_preinit.cpp b/system/lib/compiler-rt/lib/asan/asan_preinit.cpp index b07556ec96f8f..23169383bb74c 100644 --- a/system/lib/compiler-rt/lib/asan/asan_preinit.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_preinit.cpp @@ -15,10 +15,8 @@ using namespace __asan; #if SANITIZER_CAN_USE_PREINIT_ARRAY - // The symbol is called __local_asan_preinit, because it's not intended to be - // exported. - // This code linked into the main executable when -fsanitize=address is in - // the link flags. It can only use exported interface functions. - __attribute__((section(".preinit_array"), used)) - void (*__local_asan_preinit)(void) = __asan_init; +// This section is linked into the main executable when -fsanitize=address is +// specified to perform initialization at a very early stage. +__attribute__((section(".preinit_array"), used)) static auto preinit = + __asan_init; #endif diff --git a/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp b/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp index bed2f62a22511..6e08b8f966507 100644 --- a/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_premap_shadow.cpp @@ -33,7 +33,8 @@ uptr PremapShadowSize() { // PremapShadowSize() bytes on the right of it are mapped r/o. uptr PremapShadow() { return MapDynamicShadow(PremapShadowSize(), /*mmap_alignment_scale*/ 3, - /*min_shadow_base_alignment*/ 0, kHighMemEnd); + /*min_shadow_base_alignment*/ 0, kHighMemEnd, + GetMmapGranularity()); } bool PremapShadowFailed() { diff --git a/system/lib/compiler-rt/lib/asan/asan_report.cpp b/system/lib/compiler-rt/lib/asan/asan_report.cpp index 7603e8131154b..fd590e401f67f 100644 --- a/system/lib/compiler-rt/lib/asan/asan_report.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_report.cpp @@ -24,6 +24,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_interface_internal.h" +#include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_report_decorator.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_symbolizer.h" @@ -32,8 +33,11 @@ namespace __asan { // -------------------- User-specified callbacks ----------------- {{{1 static void (*error_report_callback)(const char*); -static char *error_message_buffer = nullptr; -static uptr error_message_buffer_pos = 0; +using ErrorMessageBuffer = InternalMmapVectorNoCtor; +alignas( + alignof(ErrorMessageBuffer)) static char error_message_buffer_placeholder + [sizeof(ErrorMessageBuffer)]; +static ErrorMessageBuffer *error_message_buffer = nullptr; static Mutex error_message_buf_mutex; static const unsigned kAsanBuggyPcPoolSize = 25; static __sanitizer::atomic_uintptr_t AsanBuggyPcPool[kAsanBuggyPcPoolSize]; @@ -42,17 +46,14 @@ void AppendToErrorMessageBuffer(const char *buffer) { Lock l(&error_message_buf_mutex); if (!error_message_buffer) { error_message_buffer = - (char*)MmapOrDieQuietly(kErrorMessageBufferSize, __func__); - error_message_buffer_pos = 0; + new (error_message_buffer_placeholder) ErrorMessageBuffer(); + error_message_buffer->Initialize(kErrorMessageBufferSize); } - uptr length = internal_strlen(buffer); - RAW_CHECK(kErrorMessageBufferSize >= error_message_buffer_pos); - uptr remaining = kErrorMessageBufferSize - error_message_buffer_pos; - internal_strncpy(error_message_buffer + error_message_buffer_pos, - buffer, remaining); - error_message_buffer[kErrorMessageBufferSize - 1] = '\0'; - // FIXME: reallocate the buffer instead of truncating the message. - error_message_buffer_pos += Min(remaining, length); + uptr error_message_buffer_len = error_message_buffer->size(); + uptr buffer_len = internal_strlen(buffer); + error_message_buffer->resize(error_message_buffer_len + buffer_len); + internal_memcpy(error_message_buffer->data() + error_message_buffer_len, + buffer, buffer_len); } // ---------------------- Helper functions ----------------------- {{{1 @@ -158,14 +159,14 @@ class ScopedInErrorReport { // Copy the message buffer so that we could start logging without holding a // lock that gets acquired during printing. - InternalMmapVector buffer_copy(kErrorMessageBufferSize); + InternalScopedString buffer_copy; { Lock l(&error_message_buf_mutex); - internal_memcpy(buffer_copy.data(), - error_message_buffer, kErrorMessageBufferSize); + error_message_buffer->push_back('\0'); + buffer_copy.Append(error_message_buffer->data()); // Clear error_message_buffer so that if we find other errors // we don't re-log this error. - error_message_buffer_pos = 0; + error_message_buffer->clear(); } LogFullErrorReport(buffer_copy.data()); diff --git a/system/lib/compiler-rt/lib/asan/asan_rtl.cpp b/system/lib/compiler-rt/lib/asan/asan_rtl.cpp index bb149bfbd3f5a..f9aeac33a0c91 100644 --- a/system/lib/compiler-rt/lib/asan/asan_rtl.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_rtl.cpp @@ -386,7 +386,7 @@ void PrintAddressSpaceLayout() { Printf("SHADOW_SCALE: %d\n", (int)ASAN_SHADOW_SCALE); Printf("SHADOW_GRANULARITY: %d\n", (int)ASAN_SHADOW_GRANULARITY); - Printf("SHADOW_OFFSET: 0x%zx\n", (uptr)ASAN_SHADOW_OFFSET); + Printf("SHADOW_OFFSET: %p\n", (void *)ASAN_SHADOW_OFFSET); CHECK(ASAN_SHADOW_SCALE >= 3 && ASAN_SHADOW_SCALE <= 7); if (kMidMemBeg) CHECK(kMidShadowBeg > kLowShadowEnd && @@ -414,6 +414,8 @@ static bool AsanInitInternal() { return false; } + // Make sure we are not statically linked. + __interception::DoesNotSupportStaticLinking(); AsanCheckIncompatibleRT(); AsanCheckDynamicRTPrereqs(); AvoidCVE_2016_2143(); @@ -425,9 +427,6 @@ static bool AsanInitInternal() { InitializeHighMemEnd(); - // Make sure we are not statically linked. - AsanDoesNotSupportStaticLinkage(); - // Install tool-specific callbacks in sanitizer_common. AddDieCallback(AsanDie); SetCheckUnwindCallback(CheckUnwind); diff --git a/system/lib/compiler-rt/lib/asan/asan_suppressions.cpp b/system/lib/compiler-rt/lib/asan/asan_suppressions.cpp index e71d231821866..94289d14d7e78 100644 --- a/system/lib/compiler-rt/lib/asan/asan_suppressions.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_suppressions.cpp @@ -20,7 +20,7 @@ namespace __asan { -ALIGNED(64) static char suppression_placeholder[sizeof(SuppressionContext)]; +alignas(64) static char suppression_placeholder[sizeof(SuppressionContext)]; static SuppressionContext *suppression_ctx = nullptr; static const char kInterceptorName[] = "interceptor_name"; static const char kInterceptorViaFunction[] = "interceptor_via_fun"; @@ -39,8 +39,7 @@ void InitializeSuppressions() { suppression_ctx = new (suppression_placeholder) SuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes)); suppression_ctx->ParseFromFile(flags()->suppressions); - if (&__asan_default_suppressions) - suppression_ctx->Parse(__asan_default_suppressions()); + suppression_ctx->Parse(__asan_default_suppressions()); } bool IsInterceptorSuppressed(const char *interceptor_name) { diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.cpp b/system/lib/compiler-rt/lib/asan/asan_thread.cpp index cc292861c81f2..fc3891759bc88 100644 --- a/system/lib/compiler-rt/lib/asan/asan_thread.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_thread.cpp @@ -44,10 +44,15 @@ static ThreadRegistry *asan_thread_registry; static ThreadArgRetval *thread_data; static Mutex mu_for_thread_context; +// TODO(leonardchan@): It should be possible to make LowLevelAllocator +// threadsafe and consolidate this one into the GlobalLoweLevelAllocator. +// We should be able to do something similar to what's in +// sanitizer_stack_store.cpp. +static LowLevelAllocator allocator_for_thread_context; static ThreadContextBase *GetAsanThreadContext(u32 tid) { Lock lock(&mu_for_thread_context); - return new (GetGlobalLowLevelAllocator()) AsanThreadContext(tid); + return new (allocator_for_thread_context) AsanThreadContext(tid); } static void InitThreads() { @@ -62,10 +67,10 @@ static void InitThreads() { // thread before all TSD destructors will be called for it. // MIPS requires aligned address - static ALIGNED(alignof( - ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)]; - static ALIGNED(alignof( - ThreadArgRetval)) char thread_data_placeholder[sizeof(ThreadArgRetval)]; + alignas(alignof(ThreadRegistry)) static char + thread_registry_placeholder[sizeof(ThreadRegistry)]; + alignas(alignof(ThreadArgRetval)) static char + thread_data_placeholder[sizeof(ThreadArgRetval)]; asan_thread_registry = new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext); diff --git a/system/lib/compiler-rt/lib/asan/asan_win.cpp b/system/lib/compiler-rt/lib/asan/asan_win.cpp index 8507e675684ed..09a13b11cff1f 100644 --- a/system/lib/compiler-rt/lib/asan/asan_win.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_win.cpp @@ -266,11 +266,10 @@ void PlatformTSDDtor(void *tsd) { AsanThread::TSDDtor(tsd); } // }}} // ---------------------- Various stuff ---------------- {{{ -void *AsanDoesNotSupportStaticLinkage() { return 0; } - uptr FindDynamicShadowStart() { return MapDynamicShadow(MemToShadowSize(kHighMemEnd), ASAN_SHADOW_SCALE, - /*min_shadow_base_alignment*/ 0, kHighMemEnd); + /*min_shadow_base_alignment*/ 0, kHighMemEnd, + GetMmapGranularity()); } void AsanCheckDynamicRTPrereqs() {} diff --git a/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp b/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp index 0fa636bec0d00..35871a942a7a1 100644 --- a/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp @@ -80,7 +80,7 @@ INTERCEPT_LIBRARY_FUNCTION(strchr); INTERCEPT_LIBRARY_FUNCTION(strcmp); INTERCEPT_LIBRARY_FUNCTION(strcpy); INTERCEPT_LIBRARY_FUNCTION(strcspn); -INTERCEPT_LIBRARY_FUNCTION(strdup); +INTERCEPT_LIBRARY_FUNCTION(_strdup); INTERCEPT_LIBRARY_FUNCTION(strlen); INTERCEPT_LIBRARY_FUNCTION(strncat); INTERCEPT_LIBRARY_FUNCTION(strncmp); diff --git a/system/lib/compiler-rt/lib/builtins/atomic.c b/system/lib/compiler-rt/lib/builtins/atomic.c index 852bb20f08672..aded25d9baa98 100644 --- a/system/lib/compiler-rt/lib/builtins/atomic.c +++ b/system/lib/compiler-rt/lib/builtins/atomic.c @@ -12,7 +12,7 @@ // // 1) This code must work with C programs that do not link to anything // (including pthreads) and so it should not depend on any pthread -// functions. +// functions. If the user wishes to opt into using pthreads, they may do so. // 2) Atomic operations, rather than explicit mutexes, are most commonly used // on code where contended operations are rate. // @@ -56,7 +56,17 @@ static const long SPINLOCK_MASK = SPINLOCK_COUNT - 1; // defined. Each platform should define the Lock type, and corresponding // lock() and unlock() functions. //////////////////////////////////////////////////////////////////////////////// -#if defined(__FreeBSD__) || defined(__DragonFly__) +#if defined(_LIBATOMIC_USE_PTHREAD) +#include +typedef pthread_mutex_t Lock; +/// Unlock a lock. This is a release operation. +__inline static void unlock(Lock *l) { pthread_mutex_unlock(l); } +/// Locks a lock. +__inline static void lock(Lock *l) { pthread_mutex_lock(l); } +/// locks for atomic operations +static Lock locks[SPINLOCK_COUNT]; + +#elif defined(__FreeBSD__) || defined(__DragonFly__) #include // clang-format off #include diff --git a/system/lib/compiler-rt/lib/builtins/divtc3.c b/system/lib/compiler-rt/lib/builtins/divtc3.c index 099de5802daf0..c393de815337e 100644 --- a/system/lib/compiler-rt/lib/builtins/divtc3.c +++ b/system/lib/compiler-rt/lib/builtins/divtc3.c @@ -13,7 +13,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_F128) +#if defined(CRT_HAS_128BIT) && defined(CRT_HAS_F128) // Returns: the quotient of (a + ib) / (c + id) diff --git a/system/lib/compiler-rt/lib/builtins/extendbfsf2.c b/system/lib/compiler-rt/lib/builtins/extendbfsf2.c new file mode 100644 index 0000000000000..e159d7997f655 --- /dev/null +++ b/system/lib/compiler-rt/lib/builtins/extendbfsf2.c @@ -0,0 +1,13 @@ +//===-- lib/extendbfsf2.c - bfloat -> single conversion -----------*- C -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#define SRC_BFLOAT16 +#define DST_SINGLE +#include "fp_extend_impl.inc" + +COMPILER_RT_ABI float __extendbfsf2(src_t a) { return __extendXfYf2__(a); } diff --git a/system/lib/compiler-rt/lib/builtins/fp_add_impl.inc b/system/lib/compiler-rt/lib/builtins/fp_add_impl.inc index 7133358df9bd2..d20599921e7d8 100644 --- a/system/lib/compiler-rt/lib/builtins/fp_add_impl.inc +++ b/system/lib/compiler-rt/lib/builtins/fp_add_impl.inc @@ -91,7 +91,7 @@ static __inline fp_t __addXf3__(fp_t a, fp_t b) { // Shift the significand of b by the difference in exponents, with a sticky // bottom bit to get rounding correct. - const unsigned int align = aExponent - bExponent; + const unsigned int align = (unsigned int)(aExponent - bExponent); if (align) { if (align < typeWidth) { const bool sticky = (bSignificand << (typeWidth - align)) != 0; diff --git a/system/lib/compiler-rt/lib/builtins/fp_extend.h b/system/lib/compiler-rt/lib/builtins/fp_extend.h index 95ea2a7ac4b2c..22bf2b2514e57 100644 --- a/system/lib/compiler-rt/lib/builtins/fp_extend.h +++ b/system/lib/compiler-rt/lib/builtins/fp_extend.h @@ -37,16 +37,7 @@ static const int srcSigFracBits = 52; // srcBits - srcSigFracBits - 1 static const int srcExpBits = 11; -static inline int src_rep_t_clz_impl(src_rep_t a) { -#if defined __LP64__ - return __builtin_clzl(a); -#else - if (a & REP_C(0xffffffff00000000)) - return clzsi(a >> 32); - else - return 32 + clzsi(a & REP_C(0xffffffff)); -#endif -} +static inline int src_rep_t_clz_impl(src_rep_t a) { return __builtin_clzll(a); } #define src_rep_t_clz src_rep_t_clz_impl #elif defined SRC_80 @@ -81,6 +72,21 @@ static inline int src_rep_t_clz_impl(src_rep_t a) { #define src_rep_t_clz src_rep_t_clz_impl +#elif defined SRC_BFLOAT16 +#ifdef COMPILER_RT_HAS_BFLOAT16 +typedef __bf16 src_t; +#else +typedef uint16_t src_t; +#endif +typedef uint16_t src_rep_t; +#define SRC_REP_C UINT16_C +static const int srcBits = sizeof(src_t) * CHAR_BIT; +static const int srcSigFracBits = 7; +// -1 accounts for the sign bit. +// srcBits - srcSigFracBits - 1 +static const int srcExpBits = 8; +#define src_rep_t_clz __builtin_clz + #else #error Source should be half, single, or double precision! #endif // end source precision diff --git a/system/lib/compiler-rt/lib/builtins/fp_fixint_impl.inc b/system/lib/compiler-rt/lib/builtins/fp_fixint_impl.inc index 3556bad9990b2..2f2f77ce781ae 100644 --- a/system/lib/compiler-rt/lib/builtins/fp_fixint_impl.inc +++ b/system/lib/compiler-rt/lib/builtins/fp_fixint_impl.inc @@ -34,7 +34,7 @@ static __inline fixint_t __fixint(fp_t a) { // If 0 <= exponent < significandBits, right shift to get the result. // Otherwise, shift left. if (exponent < significandBits) - return sign * (significand >> (significandBits - exponent)); + return (fixint_t)(sign * (significand >> (significandBits - exponent))); else - return sign * ((fixuint_t)significand << (exponent - significandBits)); + return (fixint_t)(sign * ((fixuint_t)significand << (exponent - significandBits))); } diff --git a/system/lib/compiler-rt/lib/builtins/fp_lib.h b/system/lib/compiler-rt/lib/builtins/fp_lib.h index c4f0a5b9587f7..b2a89506135be 100644 --- a/system/lib/compiler-rt/lib/builtins/fp_lib.h +++ b/system/lib/compiler-rt/lib/builtins/fp_lib.h @@ -43,8 +43,8 @@ static __inline int rep_clz(rep_t a) { return clzsi(a); } // 32x32 --> 64 bit multiply static __inline void wideMultiply(rep_t a, rep_t b, rep_t *hi, rep_t *lo) { const uint64_t product = (uint64_t)a * b; - *hi = product >> 32; - *lo = product; + *hi = (rep_t)(product >> 32); + *lo = (rep_t)product; } COMPILER_RT_ABI fp_t __addsf3(fp_t a, fp_t b); @@ -58,16 +58,7 @@ typedef double fp_t; #define REP_C UINT64_C #define significandBits 52 -static __inline int rep_clz(rep_t a) { -#if defined __LP64__ - return __builtin_clzl(a); -#else - if (a & REP_C(0xffffffff00000000)) - return clzsi(a >> 32); - else - return 32 + clzsi(a & REP_C(0xffffffff)); -#endif -} +static inline int rep_clz(rep_t a) { return __builtin_clzll(a); } #define loWord(a) (a & 0xffffffffU) #define hiWord(a) (a >> 32) @@ -239,7 +230,7 @@ static __inline int normalize(rep_t *significand) { return 1 - shift; } -static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) { +static __inline void wideLeftShift(rep_t *hi, rep_t *lo, unsigned int count) { *hi = *hi << count | *lo >> (typeWidth - count); *lo = *lo << count; } diff --git a/system/lib/compiler-rt/lib/builtins/int_math.h b/system/lib/compiler-rt/lib/builtins/int_math.h index 74d3e311db5e7..08bfe922ffa13 100644 --- a/system/lib/compiler-rt/lib/builtins/int_math.h +++ b/system/lib/compiler-rt/lib/builtins/int_math.h @@ -65,9 +65,12 @@ #define crt_copysign(x, y) __builtin_copysign((x), (y)) #define crt_copysignf(x, y) __builtin_copysignf((x), (y)) #define crt_copysignl(x, y) __builtin_copysignl((x), (y)) -#if __has_builtin(__builtin_copysignf128) +// We define __has_builtin to always return 0 for GCC versions below 10, +// but __builtin_copysignf128 is available since version 7. +#if __has_builtin(__builtin_copysignf128) || \ + (defined(__GNUC__) && __GNUC__ >= 7) #define crt_copysignf128(x, y) __builtin_copysignf128((x), (y)) -#elif __has_builtin(__builtin_copysignq) || (defined(__GNUC__) && __GNUC__ >= 7) +#elif __has_builtin(__builtin_copysignq) #define crt_copysignf128(x, y) __builtin_copysignq((x), (y)) #endif #endif @@ -80,9 +83,11 @@ #define crt_fabs(x) __builtin_fabs((x)) #define crt_fabsf(x) __builtin_fabsf((x)) #define crt_fabsl(x) __builtin_fabsl((x)) -#if __has_builtin(__builtin_fabsf128) +// We define __has_builtin to always return 0 for GCC versions below 10, +// but __builtin_fabsf128 is available since version 7. +#if __has_builtin(__builtin_fabsf128) || (defined(__GNUC__) && __GNUC__ >= 7) #define crt_fabsf128(x) __builtin_fabsf128((x)) -#elif __has_builtin(__builtin_fabsq) || (defined(__GNUC__) && __GNUC__ >= 7) +#elif __has_builtin(__builtin_fabsq) #define crt_fabsf128(x) __builtin_fabsq((x)) #endif #endif diff --git a/system/lib/compiler-rt/lib/builtins/int_types.h b/system/lib/compiler-rt/lib/builtins/int_types.h index ca97391fc2846..48862f3642175 100644 --- a/system/lib/compiler-rt/lib/builtins/int_types.h +++ b/system/lib/compiler-rt/lib/builtins/int_types.h @@ -107,8 +107,8 @@ typedef union { static __inline ti_int make_ti(di_int h, di_int l) { twords r; - r.s.high = h; - r.s.low = l; + r.s.high = (du_int)h; + r.s.low = (du_int)l; return r.all; } diff --git a/system/lib/compiler-rt/lib/builtins/multc3.c b/system/lib/compiler-rt/lib/builtins/multc3.c index 61a3f45e47279..a89832f0e883e 100644 --- a/system/lib/compiler-rt/lib/builtins/multc3.c +++ b/system/lib/compiler-rt/lib/builtins/multc3.c @@ -15,7 +15,7 @@ #include "int_lib.h" #include "int_math.h" -#if defined(CRT_HAS_F128) +#if defined(CRT_HAS_128BIT) && defined(CRT_HAS_F128) // Returns: the product of a + ib and c + id diff --git a/system/lib/compiler-rt/lib/builtins/os_version_check.c b/system/lib/compiler-rt/lib/builtins/os_version_check.c index 182eabe7a6ae2..01fae834ab219 100644 --- a/system/lib/compiler-rt/lib/builtins/os_version_check.c +++ b/system/lib/compiler-rt/lib/builtins/os_version_check.c @@ -316,8 +316,8 @@ int32_t __isOSVersionAtLeast(int32_t Major, int32_t Minor, int32_t Subminor) { static pthread_once_t once = PTHREAD_ONCE_INIT; pthread_once(&once, readSystemProperties); - return SdkVersion >= Major || - (IsPreRelease && Major == __ANDROID_API_FUTURE__); + // Allow all on pre-release. Note that we still rely on compile-time checks. + return SdkVersion >= Major || IsPreRelease; } #else diff --git a/system/lib/compiler-rt/lib/builtins/trampoline_setup.c b/system/lib/compiler-rt/lib/builtins/trampoline_setup.c index 844eb27944142..830e25e4c0303 100644 --- a/system/lib/compiler-rt/lib/builtins/trampoline_setup.c +++ b/system/lib/compiler-rt/lib/builtins/trampoline_setup.c @@ -41,3 +41,45 @@ COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack, __clear_cache(trampOnStack, &trampOnStack[10]); } #endif // __powerpc__ && !defined(__powerpc64__) + +// The AArch64 compiler generates calls to __trampoline_setup() when creating +// trampoline functions on the stack for use with nested functions. +// This function creates a custom 36-byte trampoline function on the stack +// which loads x18 with a pointer to the outer function's locals +// and then jumps to the target nested function. +// Note: x18 is a reserved platform register on Windows and macOS. + +#if defined(__aarch64__) && defined(__ELF__) +COMPILER_RT_ABI void __trampoline_setup(uint32_t *trampOnStack, + int trampSizeAllocated, + const void *realFunc, void *localsPtr) { + // This should never happen, but if compiler did not allocate + // enough space on stack for the trampoline, abort. + if (trampSizeAllocated < 36) + compilerrt_abort(); + + // create trampoline + // Load realFunc into x17. mov/movk 16 bits at a time. + trampOnStack[0] = + 0xd2800000u | ((((uint64_t)realFunc >> 0) & 0xffffu) << 5) | 0x11; + trampOnStack[1] = + 0xf2a00000u | ((((uint64_t)realFunc >> 16) & 0xffffu) << 5) | 0x11; + trampOnStack[2] = + 0xf2c00000u | ((((uint64_t)realFunc >> 32) & 0xffffu) << 5) | 0x11; + trampOnStack[3] = + 0xf2e00000u | ((((uint64_t)realFunc >> 48) & 0xffffu) << 5) | 0x11; + // Load localsPtr into x18 + trampOnStack[4] = + 0xd2800000u | ((((uint64_t)localsPtr >> 0) & 0xffffu) << 5) | 0x12; + trampOnStack[5] = + 0xf2a00000u | ((((uint64_t)localsPtr >> 16) & 0xffffu) << 5) | 0x12; + trampOnStack[6] = + 0xf2c00000u | ((((uint64_t)localsPtr >> 32) & 0xffffu) << 5) | 0x12; + trampOnStack[7] = + 0xf2e00000u | ((((uint64_t)localsPtr >> 48) & 0xffffu) << 5) | 0x12; + trampOnStack[8] = 0xd61f0220; // br x17 + + // Clear instruction cache. + __clear_cache(trampOnStack, &trampOnStack[9]); +} +#endif // defined(__aarch64__) && !defined(__APPLE__) && !defined(_WIN64) diff --git a/system/lib/compiler-rt/lib/interception/interception.h b/system/lib/compiler-rt/lib/interception/interception.h index 43a03d6eafb2e..ff794a05be391 100644 --- a/system/lib/compiler-rt/lib/interception/interception.h +++ b/system/lib/compiler-rt/lib/interception/interception.h @@ -208,11 +208,11 @@ const interpose_substitution substitution_##func_name[] \ ".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ ASM_TYPE_FUNCTION_STR "\n" \ SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \ - SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \ + C_ASM_STARTPROC "\n" \ C_ASM_TAIL_CALL(SANITIZER_STRINGIFY(TRAMPOLINE(func)), \ "__interceptor_" \ SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func))) "\n" \ - SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \ + C_ASM_ENDPROC "\n" \ ".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ ); @@ -359,6 +359,18 @@ typedef unsigned long long uptr; // NOLINT #else typedef unsigned long uptr; // NOLINT #endif // _WIN64 + +#if defined(__ELF__) && !SANITIZER_FUCHSIA +// The use of interceptors makes many sanitizers unusable for static linking. +// Define a function, if called, will cause a linker error (undefined _DYNAMIC). +// However, -static-pie (which is not common) cannot be detected at link time. +extern uptr kDynamic[] asm("_DYNAMIC"); +inline void DoesNotSupportStaticLinking() { + [[maybe_unused]] volatile auto x = &kDynamic; +} +#else +inline void DoesNotSupportStaticLinking() {} +#endif } // namespace __interception #define INCLUDED_FROM_INTERCEPTION_LIB diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.h b/system/lib/compiler-rt/lib/interception/interception_linux.h index caa42fc4b7ccd..557e18269898f 100644 --- a/system/lib/compiler-rt/lib/interception/interception_linux.h +++ b/system/lib/compiler-rt/lib/interception/interception_linux.h @@ -28,12 +28,14 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, uptr func, uptr trampoline); } // namespace __interception -#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ - ::__interception::InterceptFunction( \ - #func, \ - (::__interception::uptr *)&REAL(func), \ - (::__interception::uptr)&(func), \ - (::__interception::uptr)&TRAMPOLINE(func)) +// Cast func to type of REAL(func) before casting to uptr in case it is an +// overloaded function, which is the case for some glibc functions when +// _FORTIFY_SOURCE is used. This disambiguates which overload to use. +#define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ + ::__interception::InterceptFunction( \ + #func, (::__interception::uptr *)&REAL(func), \ + (::__interception::uptr)(decltype(REAL(func)))&(func), \ + (::__interception::uptr) &TRAMPOLINE(func)) // dlvsym is a GNU extension supported by some other platforms. #if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD @@ -41,7 +43,7 @@ bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, ::__interception::InterceptFunction( \ #func, symver, \ (::__interception::uptr *)&REAL(func), \ - (::__interception::uptr)&(func), \ + (::__interception::uptr)(decltype(REAL(func)))&(func), \ (::__interception::uptr)&TRAMPOLINE(func)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ diff --git a/system/lib/compiler-rt/lib/interception/interception_win.cpp b/system/lib/compiler-rt/lib/interception/interception_win.cpp index 1829358705fe8..a638e66eccee5 100644 --- a/system/lib/compiler-rt/lib/interception/interception_win.cpp +++ b/system/lib/compiler-rt/lib/interception/interception_win.cpp @@ -339,7 +339,7 @@ struct TrampolineMemoryRegion { uptr max_size; }; -UNUSED static const uptr kTrampolineScanLimitRange = 1 << 31; // 2 gig +UNUSED static const uptr kTrampolineScanLimitRange = 1ull << 31; // 2 gig static const int kMaxTrampolineRegion = 1024; static TrampolineMemoryRegion TrampolineRegions[kMaxTrampolineRegion]; @@ -479,6 +479,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { switch (*(u8*)address) { case 0x90: // 90 : nop + case 0xC3: // C3 : ret (for small/empty function interception + case 0xCC: // CC : int 3 i.e. registering weak functions) return 1; case 0x50: // push eax / rax @@ -502,7 +504,6 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { // Cannot overwrite control-instruction. Return 0 to indicate failure. case 0xE9: // E9 XX XX XX XX : jmp