diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b90c35 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +**/__pycache__/ +venv + +tests/core/pyspec/builderspec/bellatrix.py +tests/core/pyspec/builderspec/capella.py diff --git a/.gitmodules b/.gitmodules index 81aceec..eb543ab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "beacon-apis"] path = beacon-apis url = git@github.com:ethereum/beacon-APIs.git +[submodule "consensus-specs"] + path = consensus-specs + url = https://github.com/ethereum/consensus-specs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b618f4e --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +SPEC_DIR = ./specs +TESTS_DIR = ./tests/core + +PY_SPEC_DIR = $(TESTS_DIR)/pyspec +SPEC_MODULE_DIR = $(PY_SPEC_DIR)/builderspec + +SPECS = bellatrix capella + +clean: + rm -rf venv; + @for spec in $(SPECS) ; do \ + rm -rf $(SPEC_MODULE_DIR)/$$spec; \ + done + +# make (i.e. generate) the pyspec +pyspec: + python3 -m venv venv + . venv/bin/activate + pip install marko + pip install git+https://github.com/ethereum/consensus-specs.git + python3 spec.py --specs $(SPECS) + +# test the pyspec +test: pyspec + . venv/bin/activate + pip install pytest + python3 -m pytest $(PY_SPEC_DIR)/builderspec/test diff --git a/consensus-specs b/consensus-specs new file mode 160000 index 0000000..1c424d7 --- /dev/null +++ b/consensus-specs @@ -0,0 +1 @@ +Subproject commit 1c424d76eddbacae3cbffed8276264b46951456b diff --git a/spec.py b/spec.py new file mode 100644 index 0000000..d463dd3 --- /dev/null +++ b/spec.py @@ -0,0 +1,217 @@ +from enum import IntEnum +from typing import Dict, NamedTuple, List, Sequence, Optional, TypeVar, Tuple + +import ast +import os +import string + +from marko.block import Heading, FencedCode, LinkRefDef, BlankLine +from marko.inline import CodeSpan +from marko.ext.gfm import gfm +from marko.ext.gfm.elements import Table +from pathlib import Path + +def is_variable_constant(name: str) -> bool: + if name[0] not in string.ascii_uppercase + '_': + return False + return all(map(lambda c: c in string.ascii_uppercase + '_' + string.digits, name[1:])) + +class VariableDefinition(NamedTuple): + type_name: Optional[str] + value: str + +def parse_variable_def(name: str, typed_value: str) -> VariableDefinition: + typed_value = typed_value.strip() + if '(' not in typed_value: + return VariableDefinition(type_name=None, value=typed_value) + + i = typed_value.index('(') + type_name = typed_value[:i] + return VariableDefinition(type_name=type_name, value=typed_value) + +class SpecObject(NamedTuple): + functions: Dict[str, str] + custom_types: Dict[str, str] + constant_vars: Dict[str, VariableDefinition] + ssz_objects: Dict[str, str] + +def get_source_from_code_block(code: FencedCode) -> str: + return code.children[0].children.strip() + +def get_function_name_from_source(source: str) -> str: + fn = ast.parse(source).body[0] + return fn.name + +def get_class_info_from_source(source: str) -> Tuple[str, Optional[str]]: + class_def = ast.parse(source).body[0] + base = class_def.bases[0] + if isinstance(base, ast.Name): + parent_class = base.id + else: + parent_class = None + return class_def.name, parent_class + +class TableCols(IntEnum): + NAME = 0 + VALUE = 1 + +def get_spec(file_name: Path) -> SpecObject: + functions: Dict[str, str] = {} + constant_vars: Dict[str, VariableDefinition] = {} + ssz_objects: Dict[str, str] = {} + custom_types: Dict[str, str] = {} + + with open(file_name) as source_file: + document = gfm.parse(source_file.read()) + + for child in document.children: + if isinstance(child, FencedCode): + if child.lang != "python": + continue + source = get_source_from_code_block(child) + if source.startswith("def"): + function_name = get_function_name_from_source(source) + function_def = "\n".join(line.rstrip() for line in source.splitlines()) + functions[function_name] = function_def + elif source.startswith("class"): + class_name, parent_class_name = get_class_info_from_source(source) + if parent_class_name: + assert parent_class_name == "Container" + ssz_objects[class_name] = "\n".join(line.rstrip() for line in source.splitlines()) + elif isinstance(child, Table): + for row in child.children: + cells = row.children + assert len(cells) >= 2 + + name = cells[TableCols.NAME].children[0].children + value = cells[TableCols.VALUE].children[0].children + + if isinstance(value, list): + # marko parses `**X**` as a list `[X]` + value = value[0].children + + if not is_variable_constant(name): + # Check whether the (name, value) defines a custom type (i.e. type alias) + if value.startswith(("uint", "Bytes", "ByteList", "Union", "Vector", "List", "ByteVector")): + custom_types[name] = value + continue + + var_def = parse_variable_def(name, value) + constant_vars[name] = var_def + else: + continue + + return SpecObject( + functions=functions, + custom_types=custom_types, + constant_vars=constant_vars, + ssz_objects=ssz_objects, + ) + +def object_to_spec(spec: SpecObject) -> str: + constant_vars = [f"{name} = {var.value}" if var.type_name is None else f"{name}: {var.type_name} = {var.value}" for (name, var) in spec.constant_vars.items()] + spec = ( + "\n\n".join(spec.custom_types.values()) + + ("\n\n" if len(spec.custom_types) > 0 else "") + + "\n".join(constant_vars) + + ("\n\n" if len(spec.constant_vars) > 0 else "") + + "\n\n".join(spec.ssz_objects.values()) + + ("\n\n" if len(spec.ssz_objects) > 0 else "") + + "\n\n".join(spec.functions.values()) + ) + return spec + +def combine_specs(x: SpecObject, y: SpecObject) -> SpecObject: + functions = {**x.functions, **y.functions} + custom_types = {**x.custom_types, **y.custom_types} + constant_vars = {**x.constant_vars, **y.constant_vars} + ssz_objects = {**x.ssz_objects, **y.ssz_objects} + + return SpecObject( + functions=functions, + custom_types=custom_types, + constant_vars=constant_vars, + ssz_objects=ssz_objects, + ) + +def build_spec(files: Sequence[Path]) -> SpecObject: + assert len(files) > 0 + specs = [get_spec(spec) for spec in files] + + assert len(specs) > 0 + total_spec = specs[0] + for spec in specs[1:]: + total_spec = combine_specs(total_spec, spec) + + return total_spec + +def spec_files(spec: str) -> Sequence[Path]: + root = os.path.dirname(__file__) + root = os.path.join(root, "specs", spec) + + spec_files = [] + for root, dirs, files in os.walk(root): + for file in files: + if file.endswith(".md"): + spec_files.append(os.path.join(root, file)) + return spec_files + +BELLATRIX_IMPORTS = [ + """from eth2spec.bellatrix.mainnet import ( + Attestation, + AttesterSlashing, + BLSPubkey, + BLSSignature, + Deposit, + DomainType, + Eth1Data, + ExecutionAddress, + ExecutionPayloadHeader, + MAX_ATTESTATIONS, + MAX_ATTESTER_SLASHINGS, + MAX_DEPOSITS, + MAX_PROPOSER_SLASHINGS, + MAX_VOLUNTARY_EXITS, + ProposerSlashing, + Root, + SignedVoluntaryExit, + Slot, + SyncAggregate, + ValidatorIndex + )""", + "from eth2spec.utils.ssz.ssz_typing import (Bytes32, Container, List, uint64, uint256)" +] + +CAPELLA_IMPORTS = ["from eth2spec.capella.mainnet import SignedBLSToExecutionChange"] + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--specs", dest="specs", nargs="+", type=Path) + args = parser.parse_args() + + specs = args.specs + files = [spec_files(spec) for spec in specs] + spec = SpecObject(functions={}, custom_types={}, constant_vars={}, ssz_objects={}) + for (spec_name, spec_files) in zip(specs, files): + latest_spec = build_spec(spec_files) + spec = combine_specs(spec, latest_spec) + dst = os.path.join("tests", "core", "pyspec", "builderspec") + with open(os.path.join(dst, f"{spec_name}.py"), "w") as f: + spec_src = object_to_spec(spec) + + spec_imports = [] + if str(spec_name) == "bellatrix": + spec_imports = BELLATRIX_IMPORTS + elif str(spec_name) == "capella": + spec_imports = BELLATRIX_IMPORTS + for imp in CAPELLA_IMPORTS: + spec_imports.append(imp) + + spec_imports = "\n".join(spec_imports) + spec_src = "\n\n".join([spec_imports, spec_src]) + + f.write(spec_src) + with open(os.path.join(dst, "__init__.py"), "w") as f: + pass diff --git a/tests/core/pyspec/builderspec/__init__.py b/tests/core/pyspec/builderspec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/pyspec/builderspec/test/__init__.py b/tests/core/pyspec/builderspec/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/pyspec/builderspec/test/bellatrix/__init__.py b/tests/core/pyspec/builderspec/test/bellatrix/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/core/pyspec/builderspec/test/bellatrix/test_bid.py b/tests/core/pyspec/builderspec/test/bellatrix/test_bid.py new file mode 100644 index 0000000..a74e6fa --- /dev/null +++ b/tests/core/pyspec/builderspec/test/bellatrix/test_bid.py @@ -0,0 +1,4 @@ +from builderspec.bellatrix import BuilderBid + +def test_bid(): + bid = BuilderBid()