feat: implement build-a-core

Makefile

@@ -4,15 +4,10 @@ OS=$(shell uname -s)
 CORE_MODE=default
 # forces the build of the core - this is the default because we let cargo decide what needs to be rebuilt, but it also runs wasm-opt needlessly if nothing has changed
 CORE_PHONY=0
-# builds the core in docker instead of on the host
-CORE_DOCKER=0
 CORE_PROFILE=debug
 
 ifeq ($(CORE_MODE),default)
 	CORE_PHONY=1
-else ifeq ($(CORE_MODE),docker)
-	CORE_PHONY=1
-	CORE_DOCKER=1
 else ifneq ($(CORE_MODE),lax)
 $(error "CORE_MODE must be one of [default, docker, lax]")
 endif
@@ -72,13 +67,6 @@ git_hooks:
 ##########
 ## CORE ##
 ##########
-ifeq ($(CORE_DOCKER),1)
-${CORE_DIST}: ${CORE_JS_ASSETS_MAP_STD} ${CORE_JS_ASSETS_PROFILE_VALIDATOR} ${CORE_SCHEMA_ASSETS_SECURITY_VALUES}
-	mkdir -p ${CORE_DIST}
-	docker build ./core -o ${CORE_DIST}
-${CORE_WASM}: ${CORE_DIST}
-${CORE_ASYNCIFY_WASM}: ${CORE_DIST}
-else
 deps_core: ${WASI_SDK_FOLDER}
 	rustup target add wasm32-wasi
 	curl https://wasmtime.dev/install.sh -sSf | bash
@@ -108,7 +96,6 @@ ${WASI_SDK_FOLDER}:
 
 test_core: ${WASI_SDK_FOLDER} ${CORE_JS_ASSETS_MAP_STD} ${CORE_JS_ASSETS_PROFILE_VALIDATOR} ${CORE_SCHEMA_ASSETS_SECURITY_VALUES}
 	cd core && cargo test -- -- --nocapture
-endif
 
 build_core: ${CORE_WASM} ${TEST_CORE_WASM} ${CORE_ASYNCIFY_WASM} ${TEST_CORE_ASYNCIFY_WASM}
 build_core_json_schemas:

core/Cargo.toml

@@ -1,12 +1,6 @@
 [workspace]
-members = [
-	"core",
-	"host_to_core_std",
-	"core_to_map_std",
-	"interpreter_js",
-	"json_schemas",
-	"comlink_language"
-]
+resolver = "2"
+members = ["comlink_language", "core", "core_to_map_std", "host_to_core_std", "interpreter_js", "json_schemas"]
 
 [workspace.dependencies]
 base64 = { version = "0.21" }
@@ -15,7 +9,7 @@ hex = { version = "0.4" }
 jsonschema = { version = "0.17", default-features = false } # https://github.com/Stranger6667/jsonschema-rs/issues/222
 regex = { version = "1" }
 serde = { version = "1", features = ["derive"] }
-serde_json = { version = "1"}
+serde_json = { version = "1" }
 serde_urlencoded = { version = "0.7" }
 thiserror = { version = "1" }
 url = { version = "2" }
@@ -24,5 +18,6 @@ tracing = { version = "0.1" }
 [profile.release]
 opt-level = "s"
 lto = true
+codegen-units = 16
 debug = false
 strip = "debuginfo"

scripts/binaryen.Dockerfile

@@ -0,0 +1,21 @@
+FROM debian:bookworm as binaryen-builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN <<EOF
+	set -e
+	apt update
+	apt install -y --no-install-recommends ca-certificates build-essential clang python3 git cmake
+	apt clean
+	rm -rf /var/lib/apt/lists/*
+EOF
+
+WORKDIR /opt/binaryen
+ARG BINARYEN_REF=version_114
+RUN git clone https://github.com/WebAssembly/binaryen.git . && git checkout $BINARYEN_REF && git submodule update --init --depth 10
+
+RUN CC=clang CXX=clang++ cmake -DBUILD_TESTS=OFF . && make -j 6
+# /opt/binaryen/bin and /opt/binaryen/lib
+
+FROM scratch AS exporter
+COPY --from=binaryen-builder /opt/binaryen/bin /bin
+COPY --from=binaryen-builder /opt/binaryen/lib /lib

scripts/build-a-core/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+out/

scripts/build-a-core/README.md

@@ -0,0 +1,100 @@
+# build-a-core
+
+This script allows building very specific configurations of the core using Docker.
+
+## Usage
+
+```
+usage: build-a-core [-c CORES_FILE] build [-h] [--o OUT_DIR] NAMES [NAMES ...]
+
+positional arguments:
+  NAMES                 Names of the core descriptions
+
+options:
+  -h, --help            show this help message and exit
+  --o OUT_DIR, --out-dir OUT_DIR
+                        Path to where to the cores
+```
+
+Defaults:
+* `CORES_FILE` is `<script root>/cores.json`
+* `OUT_DIR` is `<script root>/out`
+
+### Example
+
+Run `python3 scripts/build-a-core build current current-deps-update`. This will run docker and create `scripts/build-a-core/out/{current.wasm, current-deps-update.wasm}`.
+
+## Parameters
+
+The following parameters can be configured. Each configuration is stored in `cores.json` file.
+
+### Dependencies
+
+Dependencies of the build to be downloaded.
+
+| key | values | description |
+|-----|--------|-------------|
+| `toolchain-version` | like `1.71`, `nightly-2023-08-14` | Version of the Rust toolchain used to compile the core. |
+| `core-ref` | git ref | Git ref of the one-sdk repo to check out. |
+| `wasi-sdk-ref` | git ref | Git ref of the wasi-sdk repo to check out. |
+| `binaryen-ref` | git ref | Git ref of the binaryen repo to check out. |
+
+### Cargo profile
+
+The script creates a custom Cargo build profile to build the custom core.
+
+| key | values | description |
+|-----|--------|-------------|
+| `opt-level` | `1`, `2`, `3`, `s`, `z` | Opt level used in the custom cargo profile. |
+| `debug-info` | `none`, `line-directives-only`, `line-tables-only`, `limited`, `full` | Amount of debug information included in the compiled binary. |
+| `strip` | `none`, `debuginfo`, `symbols` | What to strip from the binary. |
+| `debug-assertions` | bool | Whether to enable debug assertions. |
+| `overflow-checks` | bool | Whether to enable overflow checks. |
+| `lto` | `off`, `thin`, `fat` | Link time optimization level. |
+| `panic` | `unwind`, `abort` | Whether to unwind or abort on panics. |
+| `codegen-units` | positive number | Number of parallel codegen units. |
+
+Notes:
+* More info <https://doc.rust-lang.org/cargo/reference/profiles.html>
+* `strip`=`symbols` usually doesn't work with wasm-opt
+* `panic`=`unwind` has no effect on wasm, `abort` is always used
+
+### Build std
+
+Whether to build the Rust standard library as a part of the build.
+
+| key | values | description |
+|-----|--------|-------------|
+| `enabled` | bool | Whether to enable building of Rust std |
+| `crates` | set of `std`, `core`, `alloc`, `proc_macro`, `panic_unwind`, `panic_abort`, `compiler_builtins` | std crates to build, some crates might enable other crates anyway |
+| `features` | optional, set of features | Set of features to enable on the std crate. |
+
+Notes:
+* **Requires** a nightly compiler.
+* `crates` must contain `panic_abort`
+* `features` defined here <https://github.com/rust-lang/rust/blob/master/library/std/Cargo.toml#L51>.
+
+### Wasm opt
+
+Whether to build and run wasm-opt as a post-process operation.
+
+| key | values | description |
+|-----|--------|-------------|
+| `enabled` | bool | Whether to enable building and invoking wasm-opt. |
+| `opt-level` | `1`, `2`, `3`, `4`, `s`, `z` | Opt level passed to wasm-opt. |
+| `converge` | bool | Whether to run wasm-opt until the result is stable. |
+| `asyncify` | bool | Whether to asyncify the core. |
+| `strip-debug` | bool | Whether to run `strip-debug` pass. |
+| `strip-dwarf` | bool | Whether to run `strip-dwarf` pass. |
+| `strip-eh` | bool | Whether to run `strip-eh` pass. |
+| `strip-producers` | bool | Whether to run `strip-producers` pass. |
+| `strip-target-features` | bool | Whether to run `strip-target-features` pass. |
+
+Notes:
+* `strip-eh` requires as least version `114`
+
+## Docker cache
+
+Docker is used mainly for its caching. However, since many separate configurations are built the cache size can grow very fast. Some useful commands to inspect docker cache are:
+* `docker builder du` - show how much space is consumed by downloaded images, caches and the like
+* `docker builder prune` - clean up Docker builder caches

scripts/build-a-core/__main__.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+
+from typing import Optional
+
+import os
+import os.path
+import argparse
+import subprocess
+
+from model import load_cores_json
+from template import dockerfile_template
+
+def _build_cli():
+	parser = argparse.ArgumentParser(
+		prog = "build-a-core",
+		description = "Script to parametrize core builds for testing and experimenting"
+	)
+	parser.add_argument(
+		"-c", "--cores",
+		default = None, type = str,
+		dest = "CORES_FILE",
+		help = "Path to the cores.json file"
+	)
+	subparsers = parser.add_subparsers(required = True)
+
+	cmd_setup = subparsers.add_parser("setup", help = "Setup the local Dockerfile")
+	cmd_setup.add_argument(
+		"-o", "--out-file",
+		default = None, type = str,
+		dest = "OUT_FILE",
+		help = "Path where to output the Dockerfile"
+	)
+	cmd_setup.add_argument("NAME", help = "Name of the core description")
+	cmd_setup.set_defaults(cmd = main_setup)
+
+	cmd_build = subparsers.add_parser("build", help = "Build the core in docker")
+	cmd_build.add_argument(
+		"--o", "--out-dir",
+		default = None, type = str,
+		dest = "OUT_DIR",
+		help = "Path to where to the cores",
+	)
+	cmd_build.add_argument("NAMES", nargs = "+", help = "Names of the core descriptions")
+	cmd_build.set_defaults(cmd = main_build)
+
+	return parser
+
+class Setup:
+	def __init__(self, cores_file: Optional[str]):
+		if cores_file is None:
+			cores_file = os.path.join(os.path.dirname(__file__), "cores.json")
+		self.cores = load_cores_json(os.path.abspath(cores_file))
+	def prepare_dockerfile(self, name: str) -> str:
+		desc = self.cores[name]
+		dockerfile = dockerfile_template(
+			name,
+			desc.dependencies.toolchain_version,
+			desc.dependencies.core_ref,
+			desc.dependencies.wasi_sdk_ref,
+			desc.dependencies.binaryen_ref,
+			desc.cargo_profile.to_toml(),
+			" ".join(desc.build_std.to_command()),
+			" ".join(desc.wasm_opt.to_command())
+		)
+
+		return dockerfile
+
+def _prepare_out_dir(out_dir: Optional[str]) -> str:
+	if out_dir is not None:
+		out_dir = os.path.abspath(out_dir)
+	else:
+		out_dir = os.path.join(os.path.dirname(__file__), "out")
+
+	try:
+		os.mkdir(out_dir)
+	except FileExistsError:
+		pass
+
+	return out_dir
+
+def main_setup(args):
+	out_file = "-"
+	if args.OUT_FILE is not None:
+		out_file = args.OUT_FILE
+
+	s = Setup(args.CORES_FILE)
+	dockerfile = s.prepare_dockerfile(args.NAME)
+	if out_file == "-":
+		print(dockerfile)
+	else:
+		with open(out_file, "w") as file:
+			print(dockerfile, file = file)
+
+def main_build(args):
+	out_dir = _prepare_out_dir(args.OUT_DIR)
+	s = Setup(args.CORES_FILE)
+
+	for name in args.NAMES:
+		dockerfile = s.prepare_dockerfile(name)
+		subprocess.run(
+			["docker", "build", "-", "-o", out_dir],
+			input = dockerfile.encode("utf-8")
+		)
+
+if __name__ == "__main__":
+	args = _build_cli().parse_args()
+	args.cmd(args)