From 813b10869e7c66edb8839a817e2d3e42731d9a64 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Fri, 21 Oct 2022 21:40:38 +0300 Subject: [PATCH 01/22] Start implementing cxx20 module support for gcc Add variables to enable and controll gcc module support. So far only manual module map specification is supported and proper dependencies on module files are not established. --- SCons/Tool/cxx.py | 31 +++++++++++++++++++++++++++---- SCons/Tool/gcc.py | 2 +- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index 128cdc4f63..bf7f1ed69a 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -50,6 +50,29 @@ def iscplusplus(source): return 1 return 0 +def gen_module_map_file(root, module_map): + module_map_text = '$$root ' + str(root) + '\n' + for module, file in module_map.items(): + module_map_text += str(module) + ' ' + str(file) + '\n' + return module_map_text + +def module_emitter(target, source, env): + import SCons.Defaults + if("CXXMODULEPATH" in env): + if("CXXMAPFILE" not in env): + env["CXXMAPFILE"] = env.Textfile("$CXXMODULEPATH/module.map", gen_module_map_file(env["CXXMODULEPATH"], env.get("CXXMODULEMAP", {}))) + + env.Depends(target, env["CXXMAPFILE"]) + return (target, source, env) + +def module_emitter_static(target, source, env): + import SCons.Defaults + return SCons.Defaults.StaticObjectEmitter(*module_emitter(target, source, env)) + +def module_emitter_shared(target, source, env): + import SCons.Defaults + return SCons.Defaults.SharedObjectEmitter(*module_emitter(target, source, env)) + def generate(env): """ Add Builders and construction variables for Visual Age C++ compilers @@ -62,18 +85,18 @@ def generate(env): for suffix in CXXSuffixes: static_obj.add_action(suffix, SCons.Defaults.CXXAction) shared_obj.add_action(suffix, SCons.Defaults.ShCXXAction) - static_obj.add_emitter(suffix, SCons.Defaults.StaticObjectEmitter) - shared_obj.add_emitter(suffix, SCons.Defaults.SharedObjectEmitter) + static_obj.add_emitter(suffix, module_emitter_static) + shared_obj.add_emitter(suffix, module_emitter_shared) SCons.Tool.cc.add_common_cc_variables(env) if 'CXX' not in env: env['CXX'] = env.Detect(compilers) or compilers[0] env['CXXFLAGS'] = SCons.Util.CLVar('') - env['CXXCOM'] = '$CXX -o $TARGET -c $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES' + env['CXXCOM'] = '$CXX -o $TARGET -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES' env['SHCXX'] = '$CXX' env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS') - env['SHCXXCOM'] = '$SHCXX -o $TARGET -c $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES' + env['SHCXXCOM'] = '$SHCXX -o $TARGET -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES' env['CPPDEFPREFIX'] = '-D' env['CPPDEFSUFFIX'] = '' diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 1a25cb44d3..48943d3e26 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -60,7 +60,7 @@ def generate(env): env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' - + env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=${CXXMAPFILE}' def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) From e9404aa72834c549e4f0de883837846989a33f31 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sat, 22 Oct 2022 03:18:50 +0300 Subject: [PATCH 02/22] Declare CMI files as extra targets if module export statement is found This is done in emitter so won't work for generated files --- SCons/Tool/cxx.py | 12 +++++++++++- SCons/Tool/gcc.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index bf7f1ed69a..f20f109b59 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -29,6 +29,7 @@ # import os.path +import re import SCons.Defaults import SCons.Util @@ -56,13 +57,22 @@ def gen_module_map_file(root, module_map): module_map_text += str(module) + ' ' + str(file) + '\n' return module_map_text +#TODO filter out C++ whitespace and comments +module_decl_re = re.compile("(module;)?(.|\n)*export module (.*);"); + def module_emitter(target, source, env): - import SCons.Defaults if("CXXMODULEPATH" in env): if("CXXMAPFILE" not in env): env["CXXMAPFILE"] = env.Textfile("$CXXMODULEPATH/module.map", gen_module_map_file(env["CXXMODULEPATH"], env.get("CXXMODULEMAP", {}))) env.Depends(target, env["CXXMAPFILE"]) + + export = module_decl_re.match(source[0].get_text_contents()) + if export: + modulename = export[3].strip() + if modulename not in env["CXXMODULEMAP"]: + env["CXXMODULEMAP"][modulename] = modulename + env["CXXMODULESUFFIX"] + target.append(env.File("$CXXMODULEPATH/" + env["CXXMODULEMAP"][modulename])) return (target, source, env) def module_emitter_static(target, source, env): diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 48943d3e26..256d83972e 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -61,6 +61,7 @@ def generate(env): env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=${CXXMAPFILE}' + env['CXXMODULESUFFIX'] = '.gcm' def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) From 3d9b9eff960772219d6ea0916dbb1b0fb873a01c Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Fri, 28 Oct 2022 19:25:35 +0300 Subject: [PATCH 03/22] Add module scanner that does simple regex and chain-calls classic scanner --- SCons/Tool/cxx.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index f20f109b59..ab68c68a15 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -33,6 +33,7 @@ import SCons.Defaults import SCons.Util +import SCons.Scanner compilers = ['CC', 'c++'] @@ -83,6 +84,28 @@ def module_emitter_shared(target, source, env): import SCons.Defaults return SCons.Defaults.SharedObjectEmitter(*module_emitter(target, source, env)) +#TODO filter out C++ whitespace and comments +module_import_re = re.compile(r"\s*(?:export)?\s*import\s*(\S*)\s*;") + +class CxxModuleScanner(SCons.Scanner.Current): + def scan(self, node, env, path): + result = self.c_scanner(node, env, path) + + if not env.get("CXXMODULEPATH"): + return result + + imports = module_import_re.findall(node.get_text_contents()) + for module in imports: + if(module[0] == "<" or module[0] == '"'): + continue + cmi = env["CXXMODULEMAP"].get(module, module+"$CXXMODULESUFFIX") + result.append(env.File("$CXXMODULEPATH/" + cmi)) + return result + def __init__(self, *args, **kwargs): + super().__init__(self.scan, *args, **kwargs) + from SCons.Tool import CScanner + self.c_scanner = CScanner + def generate(env): """ Add Builders and construction variables for Visual Age C++ compilers @@ -91,12 +114,14 @@ def generate(env): import SCons.Tool import SCons.Tool.cc static_obj, shared_obj = SCons.Tool.createObjBuilders(env) + from SCons.Tool import SourceFileScanner for suffix in CXXSuffixes: static_obj.add_action(suffix, SCons.Defaults.CXXAction) shared_obj.add_action(suffix, SCons.Defaults.ShCXXAction) static_obj.add_emitter(suffix, module_emitter_static) shared_obj.add_emitter(suffix, module_emitter_shared) + SourceFileScanner.add_scanner(suffix, CxxModuleScanner()) SCons.Tool.cc.add_common_cc_variables(env) From 2990786733e9fd4ca6ec2f272af48c01bd83032f Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Wed, 2 Nov 2022 23:59:29 +0300 Subject: [PATCH 04/22] Changed mapper type used from file to unix domain socket --- SCons/Tool/cxx.py | 5 +-- SCons/Tool/gcc.py | 92 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index ab68c68a15..a763d5ef76 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -63,10 +63,7 @@ def gen_module_map_file(root, module_map): def module_emitter(target, source, env): if("CXXMODULEPATH" in env): - if("CXXMAPFILE" not in env): - env["CXXMAPFILE"] = env.Textfile("$CXXMODULEPATH/module.map", gen_module_map_file(env["CXXMODULEPATH"], env.get("CXXMODULEMAP", {}))) - - env.Depends(target, env["CXXMAPFILE"]) + env["__CXXMODULEINIT__"](env) export = module_decl_re.match(source[0].get_text_contents()) if export: diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 256d83972e..e03800a33a 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -33,6 +33,8 @@ from . import cc import re +import threading +import asyncio import subprocess import SCons.Util @@ -40,6 +42,93 @@ compilers = ['gcc', 'cc'] +class module_mapper(threading.Thread): + def __init__(self, env, *args, **kw): + super().__init__(*args, **kw) + self.daemon = True + self.loop = asyncio.new_event_loop() + + self.env = env + + self.request_dispatch = {} + self.request_dispatch["HELLO"] = self.hello_response + self.request_dispatch["INCLUDE-TRANSLATE"] = self.include_translate_response + self.request_dispatch["MODULE-REPO"] = self.module_repo_response + self.request_dispatch["MODULE-EXPORT"] = self.module_export_response + self.request_dispatch["MODULE-COMPILED"] = self.module_compiled_response + self.request_dispatch["MODULE-IMPORT"] = self.module_import_response + + def run(self): + self.loop.run_forever() + + def hello_response(self, request, sourcefile): + if(sourcefile != ""): + return ("ERROR", "'Unexpected handshake'") + if len(request) == 4 and request[1] == "1": + return (request[3], "HELLO", "1", "SCONS", "''") + else: + return ("ERROR", "'Invalid handshake'") + + def module_repo_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEPATH"]) + + def include_translate_response(self, request, sourcefile): + return ("BOOL", "TRUE") + + def module_export_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst("$CXXMODULEPATH/" + request[1] + "$CXXMODULESUFFIX"))) + + def module_compiled_response(self, request, sourcefile): + return ("OK") + + def module_import_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst("$CXXMODULEPATH/" + request[1] + "$CXXMODULESUFFIX"))) + + def default_response(self, request, sourcefile): + return ("ERROR", "'Unknown CODY request {}'".format(request[0])) + + async def handle_connect(self, reader, writer): + sourcefile = "" + while True: + try: + request = await reader.readuntil() + except: + return + + request = request.decode("utf-8") + + separator = '' + request = request.rstrip('\n') + if request[-1] == ';': + request = request.rstrip(';') + separator = ' ;' + + request = request.split() + + response = self.request_dispatch.get(request[0], self.default_response)(request, sourcefile) + if(request[0] == "HELLO" and response[0] != "ERROR"): + sourcefile = response[0] + response = response[1:] + response = " ".join(response) + separator + "\n" + + writer.write(response.encode()) + await writer.drain() + + async def listen(self, path): + await asyncio.start_unix_server(self.handle_connect, path=path) + + +def init_mapper(env): + if env.get("__GCCMODULEMAPPER__"): + return + + mapper = module_mapper(env) + mapper.start() + asyncio.run_coroutine_threadsafe(mapper.listen( + env.subst("$CXXMODULEPATH/socket")), mapper.loop) + + env["__GCCMODULEMAPPER__"] = mapper + def generate(env): """Add Builders and construction variables for gcc to an Environment.""" @@ -60,7 +149,8 @@ def generate(env): env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' - env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=${CXXMAPFILE}' + env['__CXXMODULEINIT__'] = init_mapper + env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper==$CXXMODULEPATH/socket?$SOURCE' env['CXXMODULESUFFIX'] = '.gcm' def exists(env): From a89499d3f03357cb36acb59253fdbb63ccbca654 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Thu, 3 Nov 2022 20:23:13 +0300 Subject: [PATCH 05/22] Add support for header imports --- SCons/Tool/cxx.py | 28 ++++++++++++++++++++++++---- SCons/Tool/gcc.py | 35 +++++++++++++++++++++++++++++++---- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index a763d5ef76..c7eeefcc10 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -84,6 +84,7 @@ def module_emitter_shared(target, source, env): #TODO filter out C++ whitespace and comments module_import_re = re.compile(r"\s*(?:export)?\s*import\s*(\S*)\s*;") + class CxxModuleScanner(SCons.Scanner.Current): def scan(self, node, env, path): result = self.c_scanner(node, env, path) @@ -93,11 +94,28 @@ def scan(self, node, env, path): imports = module_import_re.findall(node.get_text_contents()) for module in imports: + is_header_unit = False if(module[0] == "<" or module[0] == '"'): - continue - cmi = env["CXXMODULEMAP"].get(module, module+"$CXXMODULESUFFIX") - result.append(env.File("$CXXMODULEPATH/" + cmi)) + module_id_prefix = "@system-header/" if module[0] == "<" else "@header/" + cmi = module_id_prefix + module[1:-1] + "$CXXMODULESUFFIX" + is_header_unit = True + else: + cmi = env["CXXMODULEMAP"].get( + module, module+"$CXXMODULESUFFIX") + cmi = env.File("$CXXMODULEPATH/" + cmi) + + if(is_header_unit and not cmi.has_builder()): + # TODO: add proper builder for header imports + # as it is this is only good for system headers because sources + # are not scanned + env.Command( + cmi, env.Value(module[1:-1]), "$CXXCOM", + CXXCOMOUTPUTSPEC="$CXXSYSTEMHEADERFLAGS" if module[0] == "<" else "$CXXUSERHEADERFLAGS", + CXXMODULEIDPREFIX=module_id_prefix + ) + result.append(cmi) return result + def __init__(self, *args, **kwargs): super().__init__(self.scan, *args, **kwargs) from SCons.Tool import CScanner @@ -125,7 +143,9 @@ def generate(env): if 'CXX' not in env: env['CXX'] = env.Detect(compilers) or compilers[0] env['CXXFLAGS'] = SCons.Util.CLVar('') - env['CXXCOM'] = '$CXX -o $TARGET -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES' + env['CXXCOMOUTPUTSPEC'] = '-o $TARGET' + env['CXXCOM'] = '$CXX $CXXCOMOUTPUTSPEC -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $CXXFLAGS $CCFLAGS $_CCCOMCOM $SOURCES' + env['CXXMODULEMAP'] = {} env['SHCXX'] = '$CXX' env['SHCXXFLAGS'] = SCons.Util.CLVar('$CXXFLAGS') env['SHCXXCOM'] = '$SHCXX -o $TARGET -c ${ CXXMODULEFLAGS if CXXMODULEPATH else "" } $SHCXXFLAGS $SHCCFLAGS $_CCCOMCOM $SOURCES' diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index e03800a33a..512e5e94dd 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -32,9 +32,11 @@ """ from . import cc +import os import re import threading import asyncio +import atexit import subprocess import SCons.Util @@ -49,6 +51,8 @@ def __init__(self, env, *args, **kw): self.loop = asyncio.new_event_loop() self.env = env + self.load_map() + atexit.register(self.save_map) self.request_dispatch = {} self.request_dispatch["HELLO"] = self.hello_response @@ -58,6 +62,22 @@ def __init__(self, env, *args, **kw): self.request_dispatch["MODULE-COMPILED"] = self.module_compiled_response self.request_dispatch["MODULE-IMPORT"] = self.module_import_response + def save_map(self): + save = open(self.env.subst("$CXXMODULEPATH/module.map"), "w") + save.writelines( + [' '.join(pair) + '\n' for pair in self.env["CXXMODULEMAP"].items()]) + + def load_map(self): + try: + load = open(self.env.subst("$CXXMODULEPATH/module.map"), "r") + except: + return + + saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1)) + for line in load.readlines()]) + saved_map.update(self.env["CXXMODULEMAP"]) + self.env["CXXMODULEMAP"] = saved_map + def run(self): self.loop.run_forever() @@ -76,13 +96,18 @@ def include_translate_response(self, request, sourcefile): return ("BOOL", "TRUE") def module_export_response(self, request, sourcefile): - return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst("$CXXMODULEPATH/" + request[1] + "$CXXMODULESUFFIX"))) + if sourcefile[1] == '@': + cmi = self.env.subst(sourcefile + "$CXXMODULESUFFIX") + self.env["CXXMODULEMAP"][request[1]] = cmi + return ("PATHNAME", cmi) + else: + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) def module_compiled_response(self, request, sourcefile): return ("OK") def module_import_response(self, request, sourcefile): - return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst("$CXXMODULEPATH/" + request[1] + "$CXXMODULESUFFIX"))) + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) def default_response(self, request, sourcefile): return ("ERROR", "'Unknown CODY request {}'".format(request[0])) @@ -117,13 +142,13 @@ async def handle_connect(self, reader, writer): async def listen(self, path): await asyncio.start_unix_server(self.handle_connect, path=path) - def init_mapper(env): if env.get("__GCCMODULEMAPPER__"): return mapper = module_mapper(env) mapper.start() + os.makedirs(env.subst("$CXXMODULEPATH"), exist_ok=True) asyncio.run_coroutine_threadsafe(mapper.listen( env.subst("$CXXMODULEPATH/socket")), mapper.loop) @@ -150,8 +175,10 @@ def generate(env): env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' env['__CXXMODULEINIT__'] = init_mapper - env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper==$CXXMODULEPATH/socket?$SOURCE' + env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper==$CXXMODULEPATH/socket?$CXXMODULEIDPREFIX$SOURCE' env['CXXMODULESUFFIX'] = '.gcm' + env['CXXUSERHEADERFLAGS'] = '-x c++-user-header' + env['CXXSYSTEMHEADERFLAGS'] = '-x c++-system-header' def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) From bf5f82f72a1e2262d9854292de9dc89ad0270d9e Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Fri, 4 Nov 2022 01:05:42 +0300 Subject: [PATCH 06/22] Add $root line to module.map so it could be potentially used to run commands outside scons as a mapper file. --- SCons/Tool/gcc.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 512e5e94dd..7612a69025 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -64,7 +64,9 @@ def __init__(self, env, *args, **kw): def save_map(self): save = open(self.env.subst("$CXXMODULEPATH/module.map"), "w") + save.writelines( + [self.env.subst("$$root $CXXMODULEPATH") + '\n'] + [' '.join(pair) + '\n' for pair in self.env["CXXMODULEMAP"].items()]) def load_map(self): @@ -74,7 +76,7 @@ def load_map(self): return saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1)) - for line in load.readlines()]) + for line in load.readlines() if not line[0] == '$']) saved_map.update(self.env["CXXMODULEMAP"]) self.env["CXXMODULEMAP"] = saved_map From 4d53b8b0f0923885fee77f6f90a9a9940833735e Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Fri, 4 Nov 2022 12:06:10 +0300 Subject: [PATCH 07/22] Make cxx module scanner recursive --- SCons/Tool/cxx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index c7eeefcc10..c32a174ed8 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -117,7 +117,7 @@ def scan(self, node, env, path): return result def __init__(self, *args, **kwargs): - super().__init__(self.scan, *args, **kwargs) + super().__init__(self.scan, recursive = True, *args, **kwargs) from SCons.Tool import CScanner self.c_scanner = CScanner From 0c6a3b8bd02b3ebaac84d1ec925c2332e0c95a1a Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Fri, 4 Nov 2022 12:40:29 +0300 Subject: [PATCH 08/22] Fix cxx module scanner's include search path --- SCons/Tool/cxx.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index c32a174ed8..03c345c9f0 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -117,7 +117,8 @@ def scan(self, node, env, path): return result def __init__(self, *args, **kwargs): - super().__init__(self.scan, recursive = True, *args, **kwargs) + from SCons.Scanner import FindPathDirs + super().__init__(self.scan, recursive = True, path_function = FindPathDirs("CPPPATH"), *args, **kwargs) from SCons.Tool import CScanner self.c_scanner = CScanner From 92263f546ef3b499c27d7e3f1e2dfd06c2e5204d Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sun, 6 Nov 2022 17:34:39 +0300 Subject: [PATCH 09/22] Add unit test --- test/CXX/CXXMODULES-gcc.py | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test/CXX/CXXMODULES-gcc.py diff --git a/test/CXX/CXXMODULES-gcc.py b/test/CXX/CXXMODULES-gcc.py new file mode 100644 index 0000000000..977dbb344a --- /dev/null +++ b/test/CXX/CXXMODULES-gcc.py @@ -0,0 +1,71 @@ +import TestSCons + +test = TestSCons.TestSCons() + +test.write('SConstruct', """\ +env = Environment() + +env.Append(CXXFLAGS = ["-std=c++20"]) +env['CXXMODULEPATH'] = "cxx-scons-modules" +env.Program("scons-module-test", ["itest.cpp", "test.cpp", "main.cpp"]) +""") + +test.write('itest.cpp', """\ +module test; + +int i = 42; +""") + +test.write('test.cpp', r"""\ +module; + +#include +#include "incl.h" + +export module test; +export void test() +{ + std::cout << "hello, world\n"; +} + +export template struct fact; + +export extern int i; + +template +struct fact { + static constexpr unsigned int value = n * fact::value; +}; + +template<> +struct fact<0> { + static constexpr unsigned int value = 1; +}; +""") + +test.write('incl.h', '') + +test.write('main.cpp', """\ +import test; + +import ; + +int main() +{ + std::cout << i << std::endl; + test(); + int i = fact<4>::value; + std::cout << i << std::endl; + return 0; +} +""") + +test.run(arguments = ".") + +test.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: From 79d286163d938db5a8cacdbee28da73f13337d0a Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sun, 6 Nov 2022 19:11:20 +0300 Subject: [PATCH 10/22] Remove redundant semicolon --- SCons/Tool/cxx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index 03c345c9f0..ededca9a92 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -59,7 +59,7 @@ def gen_module_map_file(root, module_map): return module_map_text #TODO filter out C++ whitespace and comments -module_decl_re = re.compile("(module;)?(.|\n)*export module (.*);"); +module_decl_re = re.compile("(module;)?(.|\n)*export module (.*);") def module_emitter(target, source, env): if("CXXMODULEPATH" in env): From 78136f6bdc0e06e4c08d39e8986a1d47cbfe1d38 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Mon, 7 Nov 2022 20:33:05 +0300 Subject: [PATCH 11/22] Moved test cxx module project into fixture --- test/CXX/CXX-modules-fixture/SConstruct | 5 +++ test/CXX/CXX-modules-fixture/incl.h | 0 test/CXX/CXX-modules-fixture/itest.cpp | 3 ++ test/CXX/CXX-modules-fixture/main.cpp | 12 +++++ test/CXX/CXX-modules-fixture/test.cpp | 24 ++++++++++ test/CXX/CXXMODULES-gcc.py | 58 +------------------------ 6 files changed, 45 insertions(+), 57 deletions(-) create mode 100644 test/CXX/CXX-modules-fixture/SConstruct create mode 100644 test/CXX/CXX-modules-fixture/incl.h create mode 100644 test/CXX/CXX-modules-fixture/itest.cpp create mode 100644 test/CXX/CXX-modules-fixture/main.cpp create mode 100644 test/CXX/CXX-modules-fixture/test.cpp diff --git a/test/CXX/CXX-modules-fixture/SConstruct b/test/CXX/CXX-modules-fixture/SConstruct new file mode 100644 index 0000000000..ce4352a95d --- /dev/null +++ b/test/CXX/CXX-modules-fixture/SConstruct @@ -0,0 +1,5 @@ +env = Environment() + +env.Append(CXXFLAGS = ["-std=c++20"]) +env['CXXMODULEPATH'] = "cxx-scons-modules" +env.Program("scons-module-test", ["itest.cpp", "test.cpp", "main.cpp"]) diff --git a/test/CXX/CXX-modules-fixture/incl.h b/test/CXX/CXX-modules-fixture/incl.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/CXX/CXX-modules-fixture/itest.cpp b/test/CXX/CXX-modules-fixture/itest.cpp new file mode 100644 index 0000000000..81feabf28c --- /dev/null +++ b/test/CXX/CXX-modules-fixture/itest.cpp @@ -0,0 +1,3 @@ +module test; + +int i = 42; diff --git a/test/CXX/CXX-modules-fixture/main.cpp b/test/CXX/CXX-modules-fixture/main.cpp new file mode 100644 index 0000000000..5425b02b33 --- /dev/null +++ b/test/CXX/CXX-modules-fixture/main.cpp @@ -0,0 +1,12 @@ +import test; + +import ; + +int main() +{ + std::cout << i << std::endl; + test(); + int i = fact<4>::value; + std::cout << i << std::endl; + return 0; +} diff --git a/test/CXX/CXX-modules-fixture/test.cpp b/test/CXX/CXX-modules-fixture/test.cpp new file mode 100644 index 0000000000..3848c43356 --- /dev/null +++ b/test/CXX/CXX-modules-fixture/test.cpp @@ -0,0 +1,24 @@ +module; + +#include +#include "incl.h" + +export module test; +export void test() +{ + std::cout << "hello, world\n"; +} + +export template struct fact; + +export extern int i; + +template +struct fact { + static constexpr unsigned int value = n * fact::value; +}; + +template<> +struct fact<0> { + static constexpr unsigned int value = 1; +}; diff --git a/test/CXX/CXXMODULES-gcc.py b/test/CXX/CXXMODULES-gcc.py index 977dbb344a..7bc7fa0373 100644 --- a/test/CXX/CXXMODULES-gcc.py +++ b/test/CXX/CXXMODULES-gcc.py @@ -2,63 +2,7 @@ test = TestSCons.TestSCons() -test.write('SConstruct', """\ -env = Environment() - -env.Append(CXXFLAGS = ["-std=c++20"]) -env['CXXMODULEPATH'] = "cxx-scons-modules" -env.Program("scons-module-test", ["itest.cpp", "test.cpp", "main.cpp"]) -""") - -test.write('itest.cpp', """\ -module test; - -int i = 42; -""") - -test.write('test.cpp', r"""\ -module; - -#include -#include "incl.h" - -export module test; -export void test() -{ - std::cout << "hello, world\n"; -} - -export template struct fact; - -export extern int i; - -template -struct fact { - static constexpr unsigned int value = n * fact::value; -}; - -template<> -struct fact<0> { - static constexpr unsigned int value = 1; -}; -""") - -test.write('incl.h', '') - -test.write('main.cpp', """\ -import test; - -import ; - -int main() -{ - std::cout << i << std::endl; - test(); - int i = fact<4>::value; - std::cout << i << std::endl; - return 0; -} -""") +test.dir_fixture("CXX-modules-fixture") test.run(arguments = ".") From 577c5a5c8dd39fb0440ac8f4a59f23cae8528c65 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Thu, 10 Nov 2022 20:52:21 +0300 Subject: [PATCH 12/22] Catch more specific exceptions --- SCons/Tool/gcc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 7612a69025..a468d7710d 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -72,7 +72,7 @@ def save_map(self): def load_map(self): try: load = open(self.env.subst("$CXXMODULEPATH/module.map"), "r") - except: + except FileNotFoundError: return saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1)) @@ -119,7 +119,7 @@ async def handle_connect(self, reader, writer): while True: try: request = await reader.readuntil() - except: + except EOFError: return request = request.decode("utf-8") From 577e35446e10cf3d799720629fc9d4ac6a339694 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sat, 12 Nov 2022 23:10:24 +0300 Subject: [PATCH 13/22] Add dependency on module interface from module implementation unit --- SCons/Tool/cxx.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index ededca9a92..916a3eae67 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -82,7 +82,7 @@ def module_emitter_shared(target, source, env): return SCons.Defaults.SharedObjectEmitter(*module_emitter(target, source, env)) #TODO filter out C++ whitespace and comments -module_import_re = re.compile(r"\s*(?:export)?\s*import\s*(\S*)\s*;") +module_import_re = re.compile(r"\s*(?:(?:export)?\s*import\s*(\S*)\s*;)|(?:(export)?\s*module\s*(\S*)\s*;)") class CxxModuleScanner(SCons.Scanner.Current): @@ -93,7 +93,12 @@ def scan(self, node, env, path): return result imports = module_import_re.findall(node.get_text_contents()) - for module in imports: + for module, export, impl in imports: + if not module: + if not export and impl: # module implementation unit depends on module interface + module = impl + else: + continue is_header_unit = False if(module[0] == "<" or module[0] == '"'): module_id_prefix = "@system-header/" if module[0] == "<" else "@header/" From affe4e3a56fc67d2c6351208bd63b650e1628cc1 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sun, 13 Nov 2022 00:46:03 +0300 Subject: [PATCH 14/22] Add more faithful CODY protocol decode --- SCons/Tool/gcc.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index a468d7710d..d57fb123fe 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -44,6 +44,40 @@ compilers = ['gcc', 'cc'] +def CODY_decode(input): + quoted = False + backslashed = False + result = [] + output = "" + + for c in input: + if quoted: + if backslashed: + output.append(c) + backslashed = False + continue + if c == "'": + quoted = False + continue + if c == "\\": + backslashed = True + continue + output += c + continue + + if c == "'": + quoted = True + continue + if c == ' ' and output: + result.append(output) + output = "" + continue + output += c + if output: + result.append(output) + + return result + class module_mapper(threading.Thread): def __init__(self, env, *args, **kw): super().__init__(*args, **kw) @@ -98,7 +132,7 @@ def include_translate_response(self, request, sourcefile): return ("BOOL", "TRUE") def module_export_response(self, request, sourcefile): - if sourcefile[1] == '@': + if sourcefile[0] == '@': cmi = self.env.subst(sourcefile + "$CXXMODULESUFFIX") self.env["CXXMODULEMAP"][request[1]] = cmi return ("PATHNAME", cmi) @@ -130,7 +164,7 @@ async def handle_connect(self, reader, writer): request = request.rstrip(';') separator = ' ;' - request = request.split() + request = CODY_decode(request) response = self.request_dispatch.get(request[0], self.default_response)(request, sourcefile) if(request[0] == "HELLO" and response[0] != "ERROR"): From 528b7387c31bd9f722e1c8210670693e1e20f41e Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Thu, 17 Nov 2022 16:27:25 +0300 Subject: [PATCH 15/22] Add builder for c++ header units --- SCons/Tool/cxx.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index 916a3eae67..256e7d8f54 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -95,7 +95,7 @@ def scan(self, node, env, path): imports = module_import_re.findall(node.get_text_contents()) for module, export, impl in imports: if not module: - if not export and impl: # module implementation unit depends on module interface + if not export and impl: # module implementation unit depends on module interface module = impl else: continue @@ -110,12 +110,16 @@ def scan(self, node, env, path): cmi = env.File("$CXXMODULEPATH/" + cmi) if(is_header_unit and not cmi.has_builder()): - # TODO: add proper builder for header imports - # as it is this is only good for system headers because sources - # are not scanned - env.Command( - cmi, env.Value(module[1:-1]), "$CXXCOM", + source = self.c_scanner.find_include( + (module[0], module[1:-1]), node.dir, path) + if source[0]: + source = source[0] + else: + source = env.Value(module[1:-1]) + env.CxxHeaderUnit( + cmi, source, CXXCOMOUTPUTSPEC="$CXXSYSTEMHEADERFLAGS" if module[0] == "<" else "$CXXUSERHEADERFLAGS", + CPPPATH = [node.dir, "$CPPPATH"] if module[0] == '"' else "$CPPPATH", CXXMODULEIDPREFIX=module_id_prefix ) result.append(cmi) @@ -144,6 +148,10 @@ def generate(env): shared_obj.add_emitter(suffix, module_emitter_shared) SourceFileScanner.add_scanner(suffix, CxxModuleScanner()) + header_unit = SCons.Builder.Builder(action="$CXXCOM", + source_scanner=CxxModuleScanner()) + env["BUILDERS"]["CxxHeaderUnit"] = header_unit + SCons.Tool.cc.add_common_cc_variables(env) if 'CXX' not in env: From 175ea07c6b83d2257294ec69547893571997e4b3 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Fri, 18 Nov 2022 03:12:42 +0300 Subject: [PATCH 16/22] Make only one header unit builder so as not to trip up unit test --- SCons/Tool/cxx.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/cxx.py b/SCons/Tool/cxx.py index 256e7d8f54..e198ba473d 100644 --- a/SCons/Tool/cxx.py +++ b/SCons/Tool/cxx.py @@ -131,6 +131,11 @@ def __init__(self, *args, **kwargs): from SCons.Tool import CScanner self.c_scanner = CScanner + +header_unit = SCons.Builder.Builder(action="$CXXCOM", + source_scanner=CxxModuleScanner()) + + def generate(env): """ Add Builders and construction variables for Visual Age C++ compilers @@ -148,9 +153,7 @@ def generate(env): shared_obj.add_emitter(suffix, module_emitter_shared) SourceFileScanner.add_scanner(suffix, CxxModuleScanner()) - header_unit = SCons.Builder.Builder(action="$CXXCOM", - source_scanner=CxxModuleScanner()) - env["BUILDERS"]["CxxHeaderUnit"] = header_unit + env['BUILDERS']['CxxHeaderUnit'] = header_unit SCons.Tool.cc.add_common_cc_variables(env) From cfe4ff7958f0e5954e27ad290d106adf76c933e8 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sat, 19 Nov 2022 04:01:37 +0300 Subject: [PATCH 17/22] Explicitly request gcc for the test --- test/CXX/CXX-modules-fixture/SConstruct | 4 ++++ test/CXX/CXXMODULES-gcc.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/CXX/CXX-modules-fixture/SConstruct b/test/CXX/CXX-modules-fixture/SConstruct index ce4352a95d..8cb4bcf976 100644 --- a/test/CXX/CXX-modules-fixture/SConstruct +++ b/test/CXX/CXX-modules-fixture/SConstruct @@ -1,4 +1,8 @@ env = Environment() +try: + env.Tool(ARGUMENTS["toolset"]) +except KeyError: + pass env.Append(CXXFLAGS = ["-std=c++20"]) env['CXXMODULEPATH'] = "cxx-scons-modules" diff --git a/test/CXX/CXXMODULES-gcc.py b/test/CXX/CXXMODULES-gcc.py index 7bc7fa0373..6b0895217b 100644 --- a/test/CXX/CXXMODULES-gcc.py +++ b/test/CXX/CXXMODULES-gcc.py @@ -4,7 +4,7 @@ test.dir_fixture("CXX-modules-fixture") -test.run(arguments = ".") +test.run(arguments = ". toolset=g++") test.pass_test() From b5f8a008e47b02d2ed3272d6e5da9fe234b9c192 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sat, 19 Nov 2022 12:24:30 +0300 Subject: [PATCH 18/22] Install gcc 12 in the github workflow for the test --- .github/workflows/runtest.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/runtest.yml b/.github/workflows/runtest.yml index 02be7c1140..6b7c036b85 100644 --- a/.github/workflows/runtest.yml +++ b/.github/workflows/runtest.yml @@ -40,6 +40,9 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install -r requirements-dev.txt # sudo apt-get update + sudo apt-get install gcc-12 g++-12 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 - name: runtest ${{ matrix.os }} run: | From be547d9e355c36aa5854ab83f6636a0b4215bf9b Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sat, 19 Nov 2022 21:10:39 +0300 Subject: [PATCH 19/22] Added support for using tcp for module mapper on windows --- SCons/Tool/gcc.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index d57fb123fe..3f1b608646 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -32,7 +32,7 @@ """ from . import cc -import os +import os, sys import re import threading import asyncio @@ -176,7 +176,11 @@ async def handle_connect(self, reader, writer): await writer.drain() async def listen(self, path): - await asyncio.start_unix_server(self.handle_connect, path=path) + if self.env["__GCCMAPPERMODE__"] == "domain_socket": + await asyncio.start_unix_server(self.handle_connect, path=path) + else: + srv = await asyncio.start_server(self.handle_connect, host="::1", port = 0) + self.env["__GCCMAPPERPORT__"] = srv.sockets[0].getsockname()[1] def init_mapper(env): if env.get("__GCCMODULEMAPPER__"): @@ -211,7 +215,13 @@ def generate(env): env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' env['__CXXMODULEINIT__'] = init_mapper - env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper==$CXXMODULEPATH/socket?$CXXMODULEIDPREFIX$SOURCE' + if sys.platform != "win32": + env['__GCCMAPPERMODE__'] = "domain_socket" + env['__GCCMAPPERFLAGS__'] = "=$CXXMODULEPATH/socket" + else: + env['__GCCMAPPERMODE__'] = "tcp" + env['__GCCMAPPERFLAGS__'] = "::1:$($__GCCMAPPERPORT__$)" + env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=$__GCCMAPPERFLAGS__?$CXXMODULEIDPREFIX$SOURCE' env['CXXMODULESUFFIX'] = '.gcm' env['CXXUSERHEADERFLAGS'] = '-x c++-user-header' env['CXXSYSTEMHEADERFLAGS'] = '-x c++-system-header' From a184ed5e10ff25b2c03c92162df45b4299042397 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Sun, 20 Nov 2022 21:58:20 +0300 Subject: [PATCH 20/22] Move module mapper code from gcc.py to gxx.py --- SCons/Tool/gcc.py | 165 --------------------------------------------- SCons/Tool/gxx.py | 168 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 165 deletions(-) diff --git a/SCons/Tool/gcc.py b/SCons/Tool/gcc.py index 3f1b608646..6c497ac359 100644 --- a/SCons/Tool/gcc.py +++ b/SCons/Tool/gcc.py @@ -32,11 +32,7 @@ """ from . import cc -import os, sys import re -import threading -import asyncio -import atexit import subprocess import SCons.Util @@ -44,156 +40,6 @@ compilers = ['gcc', 'cc'] -def CODY_decode(input): - quoted = False - backslashed = False - result = [] - output = "" - - for c in input: - if quoted: - if backslashed: - output.append(c) - backslashed = False - continue - if c == "'": - quoted = False - continue - if c == "\\": - backslashed = True - continue - output += c - continue - - if c == "'": - quoted = True - continue - if c == ' ' and output: - result.append(output) - output = "" - continue - output += c - if output: - result.append(output) - - return result - -class module_mapper(threading.Thread): - def __init__(self, env, *args, **kw): - super().__init__(*args, **kw) - self.daemon = True - self.loop = asyncio.new_event_loop() - - self.env = env - self.load_map() - atexit.register(self.save_map) - - self.request_dispatch = {} - self.request_dispatch["HELLO"] = self.hello_response - self.request_dispatch["INCLUDE-TRANSLATE"] = self.include_translate_response - self.request_dispatch["MODULE-REPO"] = self.module_repo_response - self.request_dispatch["MODULE-EXPORT"] = self.module_export_response - self.request_dispatch["MODULE-COMPILED"] = self.module_compiled_response - self.request_dispatch["MODULE-IMPORT"] = self.module_import_response - - def save_map(self): - save = open(self.env.subst("$CXXMODULEPATH/module.map"), "w") - - save.writelines( - [self.env.subst("$$root $CXXMODULEPATH") + '\n'] + - [' '.join(pair) + '\n' for pair in self.env["CXXMODULEMAP"].items()]) - - def load_map(self): - try: - load = open(self.env.subst("$CXXMODULEPATH/module.map"), "r") - except FileNotFoundError: - return - - saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1)) - for line in load.readlines() if not line[0] == '$']) - saved_map.update(self.env["CXXMODULEMAP"]) - self.env["CXXMODULEMAP"] = saved_map - - def run(self): - self.loop.run_forever() - - def hello_response(self, request, sourcefile): - if(sourcefile != ""): - return ("ERROR", "'Unexpected handshake'") - if len(request) == 4 and request[1] == "1": - return (request[3], "HELLO", "1", "SCONS", "''") - else: - return ("ERROR", "'Invalid handshake'") - - def module_repo_response(self, request, sourcefile): - return ("PATHNAME", self.env["CXXMODULEPATH"]) - - def include_translate_response(self, request, sourcefile): - return ("BOOL", "TRUE") - - def module_export_response(self, request, sourcefile): - if sourcefile[0] == '@': - cmi = self.env.subst(sourcefile + "$CXXMODULESUFFIX") - self.env["CXXMODULEMAP"][request[1]] = cmi - return ("PATHNAME", cmi) - else: - return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) - - def module_compiled_response(self, request, sourcefile): - return ("OK") - - def module_import_response(self, request, sourcefile): - return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) - - def default_response(self, request, sourcefile): - return ("ERROR", "'Unknown CODY request {}'".format(request[0])) - - async def handle_connect(self, reader, writer): - sourcefile = "" - while True: - try: - request = await reader.readuntil() - except EOFError: - return - - request = request.decode("utf-8") - - separator = '' - request = request.rstrip('\n') - if request[-1] == ';': - request = request.rstrip(';') - separator = ' ;' - - request = CODY_decode(request) - - response = self.request_dispatch.get(request[0], self.default_response)(request, sourcefile) - if(request[0] == "HELLO" and response[0] != "ERROR"): - sourcefile = response[0] - response = response[1:] - response = " ".join(response) + separator + "\n" - - writer.write(response.encode()) - await writer.drain() - - async def listen(self, path): - if self.env["__GCCMAPPERMODE__"] == "domain_socket": - await asyncio.start_unix_server(self.handle_connect, path=path) - else: - srv = await asyncio.start_server(self.handle_connect, host="::1", port = 0) - self.env["__GCCMAPPERPORT__"] = srv.sockets[0].getsockname()[1] - -def init_mapper(env): - if env.get("__GCCMODULEMAPPER__"): - return - - mapper = module_mapper(env) - mapper.start() - os.makedirs(env.subst("$CXXMODULEPATH"), exist_ok=True) - asyncio.run_coroutine_threadsafe(mapper.listen( - env.subst("$CXXMODULEPATH/socket")), mapper.loop) - - env["__GCCMODULEMAPPER__"] = mapper - def generate(env): """Add Builders and construction variables for gcc to an Environment.""" @@ -214,17 +60,6 @@ def generate(env): env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' - env['__CXXMODULEINIT__'] = init_mapper - if sys.platform != "win32": - env['__GCCMAPPERMODE__'] = "domain_socket" - env['__GCCMAPPERFLAGS__'] = "=$CXXMODULEPATH/socket" - else: - env['__GCCMAPPERMODE__'] = "tcp" - env['__GCCMAPPERFLAGS__'] = "::1:$($__GCCMAPPERPORT__$)" - env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=$__GCCMAPPERFLAGS__?$CXXMODULEIDPREFIX$SOURCE' - env['CXXMODULESUFFIX'] = '.gcm' - env['CXXUSERHEADERFLAGS'] = '-x c++-user-header' - env['CXXSYSTEMHEADERFLAGS'] = '-x c++-system-header' def exists(env): # is executable, and is a GNU compiler (or accepts '--version' at least) diff --git a/SCons/Tool/gxx.py b/SCons/Tool/gxx.py index 1272997be2..30fe931981 100644 --- a/SCons/Tool/gxx.py +++ b/SCons/Tool/gxx.py @@ -32,6 +32,10 @@ """ +import os, sys +import threading +import asyncio +import atexit import SCons.Tool import SCons.Util @@ -41,6 +45,159 @@ compilers = ['g++'] +def CODY_decode(input): + quoted = False + backslashed = False + result = [] + output = "" + + for c in input: + if quoted: + if backslashed: + output.append(c) + backslashed = False + continue + if c == "'": + quoted = False + continue + if c == "\\": + backslashed = True + continue + output += c + continue + + if c == "'": + quoted = True + continue + if c == ' ' and output: + result.append(output) + output = "" + continue + output += c + if output: + result.append(output) + + return result + + +class module_mapper(threading.Thread): + def __init__(self, env, *args, **kw): + super().__init__(*args, **kw) + self.daemon = True + self.loop = asyncio.new_event_loop() + + self.env = env + self.load_map() + atexit.register(self.save_map) + + self.request_dispatch = {} + self.request_dispatch["HELLO"] = self.hello_response + self.request_dispatch["INCLUDE-TRANSLATE"] = self.include_translate_response + self.request_dispatch["MODULE-REPO"] = self.module_repo_response + self.request_dispatch["MODULE-EXPORT"] = self.module_export_response + self.request_dispatch["MODULE-COMPILED"] = self.module_compiled_response + self.request_dispatch["MODULE-IMPORT"] = self.module_import_response + + def save_map(self): + save = open(self.env.subst("$CXXMODULEPATH/module.map"), "w") + + save.writelines( + [self.env.subst("$$root $CXXMODULEPATH") + '\n'] + + [' '.join(pair) + '\n' for pair in self.env["CXXMODULEMAP"].items()]) + + def load_map(self): + try: + load = open(self.env.subst("$CXXMODULEPATH/module.map"), "r") + except FileNotFoundError: + return + + saved_map = dict([tuple(line.rstrip("\n").split(maxsplit=1)) + for line in load.readlines() if not line[0] == '$']) + saved_map.update(self.env["CXXMODULEMAP"]) + self.env["CXXMODULEMAP"] = saved_map + + def run(self): + self.loop.run_forever() + + def hello_response(self, request, sourcefile): + if(sourcefile != ""): + return ("ERROR", "'Unexpected handshake'") + if len(request) == 4 and request[1] == "1": + return (request[3], "HELLO", "1", "SCONS", "''") + else: + return ("ERROR", "'Invalid handshake'") + + def module_repo_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEPATH"]) + + def include_translate_response(self, request, sourcefile): + return ("BOOL", "TRUE") + + def module_export_response(self, request, sourcefile): + if sourcefile[0] == '@': + cmi = self.env.subst(sourcefile + "$CXXMODULESUFFIX") + self.env["CXXMODULEMAP"][request[1]] = cmi + return ("PATHNAME", cmi) + else: + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) + + def module_compiled_response(self, request, sourcefile): + return ("OK") + + def module_import_response(self, request, sourcefile): + return ("PATHNAME", self.env["CXXMODULEMAP"].get(request[1], self.env.subst(request[1] + "$CXXMODULESUFFIX"))) + + def default_response(self, request, sourcefile): + return ("ERROR", "'Unknown CODY request {}'".format(request[0])) + + async def handle_connect(self, reader, writer): + sourcefile = "" + while True: + try: + request = await reader.readuntil() + except EOFError: + return + + request = request.decode("utf-8") + + separator = '' + request = request.rstrip('\n') + if request[-1] == ';': + request = request.rstrip(';') + separator = ' ;' + + request = CODY_decode(request) + + response = self.request_dispatch.get( + request[0], self.default_response)(request, sourcefile) + if(request[0] == "HELLO" and response[0] != "ERROR"): + sourcefile = response[0] + response = response[1:] + response = " ".join(response) + separator + "\n" + + writer.write(response.encode()) + await writer.drain() + + async def listen(self, path): + if self.env["__GCCMAPPERMODE__"] == "domain_socket": + await asyncio.start_unix_server(self.handle_connect, path=path) + else: + srv = await asyncio.start_server(self.handle_connect, host="::1", port=0) + self.env["__GCCMAPPERPORT__"] = srv.sockets[0].getsockname()[1] + + +def init_mapper(env): + if env.get("__GCCMODULEMAPPER__"): + return + + mapper = module_mapper(env) + mapper.start() + os.makedirs(env.subst("$CXXMODULEPATH"), exist_ok=True) + asyncio.run_coroutine_threadsafe(mapper.listen( + env.subst("$CXXMODULEPATH/socket")), mapper.loop) + + env["__GCCMODULEMAPPER__"] = mapper + def generate(env): """Add Builders and construction variables for g++ to an Environment.""" static_obj, shared_obj = SCons.Tool.createObjBuilders(env) @@ -67,6 +224,17 @@ def generate(env): env['CCDEPFLAGS'] = '-MMD -MF ${TARGET}.d' env["NINJA_DEPFILE_PARSE_FORMAT"] = 'gcc' + env['__CXXMODULEINIT__'] = init_mapper + if sys.platform != "win32": + env['__GCCMAPPERMODE__'] = "domain_socket" + env['__GCCMAPPERFLAGS__'] = "=$CXXMODULEPATH/socket" + else: + env['__GCCMAPPERMODE__'] = "tcp" + env['__GCCMAPPERFLAGS__'] = "::1:$($__GCCMAPPERPORT__$)" + env['CXXMODULEFLAGS'] = '-fmodules-ts -fmodule-mapper=$__GCCMAPPERFLAGS__?$CXXMODULEIDPREFIX$SOURCE' + env['CXXMODULESUFFIX'] = '.gcm' + env['CXXUSERHEADERFLAGS'] = '-x c++-user-header' + env['CXXSYSTEMHEADERFLAGS'] = '-x c++-system-header' def exists(env): From fbcf7f2d6139b8e23ff540ec96368b3ec596db54 Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Tue, 22 Nov 2022 16:53:33 +0300 Subject: [PATCH 21/22] Don't use default tools in module fixture --- test/CXX/CXX-modules-fixture/SConstruct | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CXX/CXX-modules-fixture/SConstruct b/test/CXX/CXX-modules-fixture/SConstruct index 8cb4bcf976..30afd5525b 100644 --- a/test/CXX/CXX-modules-fixture/SConstruct +++ b/test/CXX/CXX-modules-fixture/SConstruct @@ -1,4 +1,4 @@ -env = Environment() +env = Environment(tools = ["link"]) try: env.Tool(ARGUMENTS["toolset"]) except KeyError: From e6e825e7c56da38a1fe244f86baef5521e43c00a Mon Sep 17 00:00:00 2001 From: loonycyborg Date: Tue, 22 Nov 2022 23:13:14 +0300 Subject: [PATCH 22/22] Skip gcc module test if g++ isn't found --- test/CXX/CXXMODULES-gcc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/CXX/CXXMODULES-gcc.py b/test/CXX/CXXMODULES-gcc.py index 6b0895217b..f8d7a4c830 100644 --- a/test/CXX/CXXMODULES-gcc.py +++ b/test/CXX/CXXMODULES-gcc.py @@ -2,6 +2,12 @@ test = TestSCons.TestSCons() +from SCons.Tool import gxx +from SCons.Script import DefaultEnvironment + +if not gxx.exists(DefaultEnvironment()): + test.skip_test('g++ not found, skipping test\n') + test.dir_fixture("CXX-modules-fixture") test.run(arguments = ". toolset=g++")