From f01ea4705fb28483918aafafbdf17e31317a617d Mon Sep 17 00:00:00 2001 From: spirsch <95348275+spirsch@users.noreply.github.com> Date: Wed, 11 May 2022 15:39:44 +0200 Subject: [PATCH] Refactoring and minor improvements --- README.md | 51 ++++++++++------- homcc/client/client.py | 40 +++++++------- homcc/client/compilation.py | 27 +++++---- homcc/client/main.py | 36 ++++++++---- homcc/client/parsing.py | 95 ++++++++++++++++---------------- homcc/common/compression.py | 27 +++++++-- homcc/common/messages.py | 4 +- homcc/server/environment.py | 5 +- homcc/server/main.py | 9 ++- homcc/server/parsing.py | 4 +- homcc/server/server.py | 30 ++++++---- tests/client/parsing_test.py | 11 ++-- tests/common/compression_test.py | 38 +++++++------ tests/e2e/e2e_test.py | 2 + tests/server/server_test.py | 2 +- 15 files changed, 222 insertions(+), 159 deletions(-) diff --git a/README.md b/README.md index e1a49a7..a883394 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,16 @@ ## Installation -- Download or [build](#build-debian-packages) the Debian packages -- Install the `homcc` client via: ```sudo apt install ./target/homcc.deb``` -- Install the `homccd` server via: ```sudo apt install ./target/homccd.deb``` +- [Download](https://github.com/celonis/homcc/releases) the latest release or [build](#build-debian-packages) the Debian packages yourself +- Install the `homcc` client via: ```sudo apt install ./homcc.deb``` +- Install the `homccd` server via: ```sudo apt install ./homccd.deb``` -**Note**: Currently, installing both packages leads to an issue with conflicting files. Therefore, to install the second package, use `sudo dpkg -i --force-overwrite ./target/{package.deb}`! +**Note**: Currently, installing both packages leads to an issue with conflicting files. Therefore, to install the second package, use `sudo dpkg -i --force-overwrite ./{package.deb}`! ## Documentation - TODO: - - Brief overview of what `homcc` is and why it exists: distribtued compilation -> faster build times, modern alternative + - Brief overview of what `homcc` is and why it exists: distributed compilation -> faster build times, modern alternative - Differences to `distcc`: thin connection as priority, caching, local pre-processing - Description of client-server interaction - Description of server-side caching @@ -34,7 +34,7 @@ ## Usage and Configuration ### Client: `homcc` -- Follow the client [Installation](#Installation) guide +- Follow the client [Installation](#installation) guide - Find usage description and client defaults: `homcc --help` - Overwrite defaults via a `client.conf` configuration file: - Possible `client.conf` locations: @@ -43,19 +43,21 @@ - `~/.config/homcc/client.conf` - `/etc/homcc/client.conf` - Possible `homcc` configurations: - - `verbose`: enable a verbose mode by specifying `True` which implies detailed and colored logging of debug messages - `compiler`: compiler if none is explicitly specified via the CLI - `timeout`: timeout value in seconds for each remote compilation attempt - `compression`: compression algorithm, choose from `{lzo, lzma}` - `profile`: `schroot` environment profile that will be used on the server side for compilations + - `log_level`: detail level for log messages, choose from `{DEBUG,INFO,WARNING,ERROR,CRITICAL}` + - `verbose`: enable a verbose mode by specifying `True` which implies detailed and colored logging of debug messages, can be combined with `log_level` - Example: ``` # homcc: example client.conf - verbose=True compiler=g++ timeout=180 compression=lzo profile=schroot_environment + log_level=DEBUG + verbose=True ``` - Specify your remote compilation server in a `hosts` file or in the `$HOMCC_HOSTS` environment variable: - Possible `hosts` file locations: @@ -68,14 +70,15 @@ - `HOST`: TCP connection to specified `HOST` with default port `3633` - `HOST:PORT`: TCP connection to specified `HOST` with specified `PORT` - `HOST/LIMIT` format: - - define any of the above `HOST` format with an additional `LIMIT` parameter that specifies the maximum connection limit to the corresponding `HOST` - - it is advised to always specify your `LIMIT`s as they will otherwise default to 2 and only enable minor levels of concurrency + - Define any of the above `HOST` format with an additional `LIMIT` parameter that specifies the maximum connection limit to the corresponding `HOST` + - It is advised to always specify your `LIMIT`s as they will otherwise default to 2 and only enable minor levels of concurrency - `HOST,COMPRESSION` format: - - define any of the above `HOST` or `HOST/LIMIT` format with an additional `COMPRESSION` algorithm information - - choose from: + - Define any of the above `HOST` or `HOST/LIMIT` format with an additional `COMPRESSION` algorithm information + - Choose from: - `lzo`: Lempel-Ziv-Oberhumer compression algorithm - `lzma`: Lempel-Ziv-Markov chain algorithm - - per default, no compression is used as it is usually not necessary for high bandwidth connections + - No compression is used per default, specifying `lzo` is however advised + - **WARNING**: Currently do not include `localhost` in your hosts file! - Example: ``` # homcc: example hosts @@ -83,11 +86,11 @@ 127.0.0.1:3633/21 [::1]:3633/42,lzo ``` -- Use `homcc` in your `conan` profile by specifying: `CCACHE_PREFIX=homcc` +- Use `homcc` by specifying `CCACHE_PREFIX=homcc` in your `conan` profile and only have `CONAN_CPU_COUNT` smaller or equal to the sum of all remote host limits, e.g. `≤ 12+21+42` for the example above! ### Server: `homccd` -- Follow the server [Installation](#Installation) guide +- Follow the server [Installation](#installation) guide - Find usage description and server defaults: `homccd --help` - Overwrite defaults via a `server.conf` configuration file: - Possible `server.conf` locations: @@ -100,14 +103,14 @@ - `port`: TCP port to listen on - `address`: IP address to listen on - `log_level`: detail level for log messages, choose from `{DEBUG,INFO,WARNING,ERROR,CRITICAL}` - - `verbose`: enable a verbose mode by specifying `True` which implies detailed and colored logging of debug messages + - `verbose`: enable a verbose mode by specifying `True` which implies detailed and colored logging of debug messages, can be combined with `log_level` - Example: ``` # homccd: example server.conf limit=64 - log_level=DEBUG port=3633 address=0.0.0.0 + log_level=DEBUG verbose=True ``` - \[Optional]: Setup your `chroot` environments at `/etc/schroot/schroot.conf` or in the
@@ -146,14 +149,22 @@ - Format a specified python file: `black ./path/to/file.py` ### Build Debian packages -- Run `make all` in the repository root to build the `server` and `client` target +- Install required tools:
+ ``` + sudo apt install -y \ + python3 python3-dev python3-pip python3-venv python3-all \ + dh-python debhelper devscripts dput software-properties-common \ + python3-distutils python3-setuptools python3-wheel python3-stdeb \ + liblzo2-dev + ``` +- Run `make homcc`, `make homccd` or `make all` to build the corresponding `client` and `server` package - The generated `.deb` files are then contained in the `./target/` directory ### `schroot` testing setup for Debian systems -- Install necessary tools: `sudo apt install schroot debootstrap` +- Install required tools: `sudo apt install schroot debootstrap` - Create `chroot` environment: - - Download and install selected distribution at your desired location, e.g. `Ubuntu 22.04 Jammy Jellyfish` from [Ubuntu Releases](https://wiki.ubuntu.com/Releases) at `/var/chroot/`:
+ - Download and install selected distribution to your desired location, e.g. `Ubuntu 22.04 Jammy Jellyfish` from [Ubuntu Releases](https://wiki.ubuntu.com/Releases) at `/var/chroot/`:
`sudo debootstrap jammy /var/chroot/jammy http://archive.ubuntu.com/ubuntu` - Configure the environment by creating a corresponding file in the `/etc/schroot/chroot.d/` directory or by appending it to `/etc/schroot/schroot.conf`, e.g. by replacing `USERNAME` in `jammy.conf`:
``` diff --git a/homcc/client/client.py b/homcc/client/client.py index 4733f89..6d93346 100644 --- a/homcc/client/client.py +++ b/homcc/client/client.py @@ -14,8 +14,6 @@ from homcc.client.parsing import ConnectionType, Host, parse_host from homcc.common.arguments import Arguments from homcc.common.messages import ArgumentMessage, DependencyReplyMessage, Message -from homcc.common.compression import Compression, NoCompression - logger = logging.getLogger(__name__) @@ -78,10 +76,10 @@ class TCPClient: """Wrapper class to exchange homcc protocol messages via TCP""" DEFAULT_PORT: int = 3633 - DEFAULT_TIMEOUT: float = 180 + DEFAULT_BUFFER_SIZE_LIMIT: int = 65_536 # default buffer size limit of StreamReader is 64 KiB DEFAULT_OPEN_CONNECTION_TIMEOUT: float = 5 - def __init__(self, host: Host, buffer_limit: Optional[int] = None): + def __init__(self, host: Host): connection_type: ConnectionType = host.type if connection_type != ConnectionType.TCP: @@ -89,11 +87,7 @@ def __init__(self, host: Host, buffer_limit: Optional[int] = None): self.host: str = host.host self.port: int = host.port or self.DEFAULT_PORT - - # default buffer size limit of StreamReader is 64 KiB - self.buffer_limit: int = buffer_limit or 65_536 - - self.compression = Compression.from_name(host.compression) if host.compression is not None else NoCompression() + self.compression = host.compression self._data: bytes = bytes() self._reader: asyncio.StreamReader @@ -101,22 +95,26 @@ def __init__(self, host: Host, buffer_limit: Optional[int] = None): async def __aenter__(self) -> TCPClient: """connect to specified server at host:port""" - logger.debug("Connecting to %s:%i", self.host, self.port) - self._reader, self._writer = await asyncio.wait_for( - asyncio.open_connection(host=self.host, port=self.port, limit=self.buffer_limit), - timeout=self.DEFAULT_OPEN_CONNECTION_TIMEOUT, - ) + logger.debug("Connecting to '%s:%i'.", self.host, self.port) + try: + self._reader, self._writer = await asyncio.wait_for( + asyncio.open_connection(host=self.host, port=self.port, limit=self.DEFAULT_BUFFER_SIZE_LIMIT), + timeout=self.DEFAULT_OPEN_CONNECTION_TIMEOUT, + ) + except asyncio.TimeoutError as error: + logger.warning("Connection establishment to '%s:%s' timed out.", self.host, self.port) + raise error from None return self async def __aexit__(self, *_): """disconnect from server and close client socket""" - logger.debug("Disconnecting from %s:%i", self.host, self.port) + logger.debug("Disconnecting from '%s:%i'.", self.host, self.port) self._writer.close() await self._writer.wait_closed() async def _send(self, message: Message): """send a message to homcc server""" - logger.debug("Sending %s to %s:%i:\n%s", message.message_type, self.host, self.port, message.get_json_str()) + logger.debug("Sending %s to '%s:%i':\n%s", message.message_type, self.host, self.port, message.get_json_str()) self._writer.write(message.to_bytes()) # type: ignore[union-attr] await self._writer.drain() # type: ignore[union-attr] @@ -134,30 +132,30 @@ async def send_dependency_reply_message(self, dependency: str): async def receive(self) -> Message: """receive data from homcc server and convert it to Message""" # read stream into internal buffer - self._data += await self._reader.read(self.buffer_limit) + self._data += await self._reader.read(self.DEFAULT_BUFFER_SIZE_LIMIT) bytes_needed, parsed_message = Message.from_bytes(bytearray(self._data)) # if message is incomplete, continue reading from stream until no more bytes are missing while bytes_needed > 0: - logger.debug("Message is incomplete by %i bytes", bytes_needed) + logger.debug("Message is incomplete by #%i bytes.", bytes_needed) self._data += await self._reader.read(bytes_needed) bytes_needed, parsed_message = Message.from_bytes(bytearray(self._data)) # manage internal buffer consistency if bytes_needed == 0: # reset the internal buffer - logger.debug("Resetting internal buffer") + logger.debug("Resetting internal buffer.") self._data = bytes() elif bytes_needed < 0: # remove the already parsed message - logger.debug("Additional data of %i bytes in buffer", abs(bytes_needed)) + logger.debug("Additional data of #%i bytes in buffer.", abs(bytes_needed)) self._data = self._data[len(self._data) - abs(bytes_needed) :] if not parsed_message: raise ClientParsingError("Received data could not be parsed to a message!") logger.debug( - "Received %s message from %s:%i:\n%s", + "Received %s message from '%s:%i':\n%s", parsed_message.message_type, self.host, self.port, diff --git a/homcc/client/compilation.py b/homcc/client/compilation.py index 2c12a8f..f6b297e 100644 --- a/homcc/client/compilation.py +++ b/homcc/client/compilation.py @@ -15,7 +15,7 @@ TCPClient, ) from homcc.client.errors import HostsExhaustedError, RemoteCompilationError, UnexpectedMessageTypeError -from homcc.client.parsing import ConnectionType, ClientConfig, Host +from homcc.client.parsing import ClientConfig, Host from homcc.common.arguments import Arguments, ArgumentsExecutionResult from homcc.common.hashing import hash_file_with_path from homcc.common.messages import ( @@ -28,27 +28,34 @@ logger = logging.getLogger(__name__) +DEFAULT_COMPILATION_REQUEST_TIMEOUT: float = 180 + async def compile_remotely(arguments: Arguments, hosts: List[str], config: ClientConfig) -> int: """main function to control remote compilation""" - # try to connect to 3 different remote compilation hosts before giving up + # try to connect to 3 different hosts before falling back to local compilation for host in HostSelector(hosts, 3): - timeout: float = config.timeout or 180 - profile: Optional[str] = config.profile - host.compression = host.compression or config.compression - - if host.type == ConnectionType.LOCAL: + # execute compilation requests for localhost directly + if host.type.is_local(): logger.info("Compiling locally:\n%s", arguments) return compile_locally(arguments) + timeout: float = config.timeout or DEFAULT_COMPILATION_REQUEST_TIMEOUT + profile: Optional[str] = config.profile + + # overwrite host compression if none was specified + host.compression = host.compression or config.compression + try: return await asyncio.wait_for(compile_remotely_at(arguments, host, profile), timeout=timeout) - except (asyncio.TimeoutError, ConnectionError) as error: + except ConnectionError as error: logger.warning("%s", error) + except asyncio.TimeoutError: + logger.warning("Compilation request for host '%s' timed out.", host.host) - raise HostsExhaustedError(f"All hosts {hosts} are exhausted!") + raise HostsExhaustedError(f"All hosts {hosts} are exhausted.") async def compile_remotely_at(arguments: Arguments, host: Host, profile: Optional[str]) -> int: @@ -59,7 +66,7 @@ async def compile_remotely_at(arguments: Arguments, host: Host, profile: Optiona async with TCPClient(host) as client: await client.send_argument_message(remote_arguments, os.getcwd(), dependency_dict, profile) - # invert dependency dictionary + # invert dependency dictionary to access dependencies via hash dependency_dict = {file_hash: dependency for dependency, file_hash in dependency_dict.items()} host_response: Message = await client.receive() diff --git a/homcc/client/main.py b/homcc/client/main.py index 68b1508..0e30b6b 100755 --- a/homcc/client/main.py +++ b/homcc/client/main.py @@ -19,6 +19,7 @@ from homcc.client.errors import RecoverableClientError, RemoteCompilationError # pylint: disable=wrong-import-position from homcc.client.parsing import ( # pylint: disable=wrong-import-position ClientConfig, + LogLevel, load_config_file, load_hosts, parse_cli_args, @@ -38,26 +39,37 @@ def main(): # load and parse arguments and configuration information homcc_args_dict, compiler_arguments = parse_cli_args(sys.argv[1:]) - client_config: ClientConfig = parse_config(load_config_file()) + homcc_config: ClientConfig = parse_config(load_config_file()) logging_config: LoggingConfig = LoggingConfig( config=FormatterConfig.COLORED, formatter=Formatter.CLIENT, destination=FormatterDestination.STREAM, ) - # VERBOSE; enables verbose mode - if homcc_args_dict["verbose"] or client_config.verbose: + # LOG_LEVEL and VERBOSITY + log_level: str = homcc_args_dict["log_level"] + + # verbosity implies debug mode + if ( + homcc_args_dict["verbose"] + or homcc_config.verbose + or log_level == "DEBUG" + or homcc_config.log_level == LogLevel.DEBUG + ): logging_config.config |= FormatterConfig.DETAILED logging_config.level = logging.DEBUG + # overwrite verbose debug logging level + if log_level is not None: + logging_config.level = LogLevel[log_level].value + elif homcc_config.log_level is not None: + logging_config.level = int(homcc_config.log_level) + setup_logging(logging_config) # COMPILER; default: "cc" - compiler: Optional[str] = compiler_arguments.compiler - - if not compiler: - compiler = client_config.compiler - compiler_arguments.compiler = compiler + if compiler_arguments.compiler is None: + compiler_arguments.compiler = homcc_config.compiler # SCAN-INCLUDES; and exit if homcc_args_dict["scan_includes"]: @@ -74,15 +86,15 @@ def main(): profile: Optional[str] = homcc_args_dict["profile"] if homcc_args_dict["no_profile"]: - client_config.profile = None + homcc_config.profile = None elif profile: - client_config.profile = profile + homcc_config.profile = profile # TIMEOUT timeout: Optional[float] = homcc_args_dict["timeout"] if timeout: - client_config.timeout = timeout + homcc_config.timeout = timeout if compiler_arguments.is_linking_only(): logger.debug("Linking [%s] to %s", ", ".join(compiler_arguments.object_files), compiler_arguments.output) @@ -91,7 +103,7 @@ def main(): # try to compile remotely if compiler_arguments.is_sendable(): try: - sys.exit(asyncio.run(compile_remotely(compiler_arguments, hosts, client_config))) + sys.exit(asyncio.run(compile_remotely(compiler_arguments, hosts, homcc_config))) # exit on unrecoverable errors except RemoteCompilationError as error: diff --git a/homcc/client/parsing.py b/homcc/client/parsing.py index 09616d8..f0abb97 100644 --- a/homcc/client/parsing.py +++ b/homcc/client/parsing.py @@ -13,6 +13,7 @@ from homcc.common.arguments import Arguments from homcc.common.compression import Compression +from homcc.common.logging import LogLevel from homcc.common.parsing import default_locations, load_config_file_from, parse_config_keys from homcc.client.errors import HostParsingError, NoHostsFoundError @@ -30,6 +31,9 @@ class ConnectionType(str, Enum): TCP = "TCP" SSH = "SSH" + def is_local(self) -> bool: + return self == ConnectionType.LOCAL + class ShowAndExitAction(ABC, Action): """ @@ -101,27 +105,27 @@ class Host: type: ConnectionType host: str + limit: int + compression: Compression port: Optional[int] user: Optional[str] - limit: int - compression: Optional[str] def __init__( self, *, type: ConnectionType, # pylint: disable=redefined-builtin host: str, - port: Optional[str] = None, - user: Optional[str] = None, limit: Optional[str] = None, compression: Optional[str] = None, + port: Optional[str] = None, + user: Optional[str] = None, ): self.type = ConnectionType.LOCAL if host == ConnectionType.LOCAL else type self.host = host - self.port = int(port) if port is not None else None - self.user = user self.limit = int(limit) if limit is not None else 2 # enable minor level of concurrency on default - self.compression = compression + self.compression = Compression.from_name(compression) + self.port = int(port) if port is not None else None # TCP + self.user = user # SSH @dataclass @@ -129,9 +133,10 @@ class ClientConfig: """Class to encapsulate and default client configuration information""" compiler: str - compression: Optional[str] + compression: Compression profile: Optional[str] timeout: Optional[float] + log_level: Optional[LogLevel] verbose: bool def __init__( @@ -141,12 +146,14 @@ def __init__( compression: Optional[str] = None, profile: Optional[str] = None, timeout: Optional[str] = None, + log_level: Optional[str] = None, verbose: Optional[str] = None, ): self.compiler = compiler or Arguments.DEFAULT_COMPILER - self.compression = compression + self.compression = Compression.from_name(compression) self.profile = profile self.timeout = float(timeout) if timeout is not None else None + self.log_level = LogLevel[log_level] if log_level else None # additional parsing step for verbosity self.verbose = verbose is not None and re.match(r"^true$", verbose, re.IGNORECASE) is not None @@ -177,10 +184,18 @@ def parse_cli_args(args: List[str]) -> Tuple[Dict[str, Any], Arguments]: "and exit", ) + parser.add_argument( + "--log-level", + required=False, + type=str, + choices=[level.name for level in LogLevel], + help=f"set detail level for log messages, defaults to {LogLevel.INFO.name}", + ) + parser.add_argument( "--verbose", action="store_true", - help="enables a verbose mode which implies detailed and colored logging of debug messages", + help="enable a verbose mode which implies detailed and colored logging of debug messages", ) indented_newline: str = "\n\t" @@ -261,43 +276,17 @@ def parse_host(host: str) -> Host: host_dict: Dict[str, str] = {} connection_type: ConnectionType - # trim trailing comment - host_comment_match = re.match(r"^(\S+)#(\S+)$", host) # HOST#COMMENT - - if host_comment_match: # HOST#COMMENT + # trim trailing comment: HOST#COMMENT + if (host_comment_match := re.match(r"^(\S+)#(\S+)$", host)) is not None: host, _ = host_comment_match.groups() - # use trailing compression info - host_compression_match = re.match(r"^(\S+),(\S+)$", host) # HOST,COMPRESSION - - if host_compression_match: # HOST,COMPRESSION + # use trailing compression info: HOST,COMPRESSION + if (host_compression_match := re.match(r"^(\S+),(\S+)$", host)) is not None: host, compression = host_compression_match.groups() + host_dict["compression"] = compression - if Compression.from_name(compression): - host_dict["compression"] = compression - else: - logger.error( - 'Compression "%s" is currently not supported! ' - "The remote compilation will be executed without compression enabled!", - compression, - ) - - # categorize host format - user_at_host_match = re.match(r"^(\w+)@([\w.:/]+)$", host) # USER@HOST - at_host_match = re.match(r"^@([\w.:/]+)$", host) # @HOST - host_port_limit_match = re.match(r"^(([\w./]+)|\[(\S+)]):(\d+)(/(\d+))?$", host) # HOST:PORT/LIMIT - host_match = re.match(r"^([\w.:/]+)$", host) # HOST - - if user_at_host_match: # USER@HOST - user, host = user_at_host_match.groups() - connection_type = ConnectionType.SSH - host_dict["user"] = user - - elif at_host_match: # @HOST - host = at_host_match.group(1) - connection_type = ConnectionType.SSH - - elif host_port_limit_match: # HOST:PORT + # HOST:PORT/LIMIT + if (host_port_limit_match := re.match(r"^(([\w./]+)|\[(\S+)]):(\d+)(/(\d+))?$", host)) is not None: _, name_or_ipv4, ipv6, port, _, limit = host_port_limit_match.groups() host = name_or_ipv4 or ipv6 connection_type = ConnectionType.TCP @@ -305,16 +294,26 @@ def parse_host(host: str) -> Host: host_dict["limit"] = limit return Host(type=connection_type, host=host, **host_dict) - elif host_match: # HOST + # USER@HOST + elif (user_at_host_match := re.match(r"^(\w+)@([\w.:/]+)$", host)) is not None: + user, host = user_at_host_match.groups() + connection_type = ConnectionType.SSH + host_dict["user"] = user + + # @HOST + elif (at_host_match := re.match(r"^@([\w.:/]+)$", host)) is not None: + host = at_host_match.group(1) + connection_type = ConnectionType.SSH + + # HOST + elif re.match(r"^([\w.:/]+)$", host) is not None: connection_type = ConnectionType.TCP else: raise HostParsingError(f'Host "{host}" could not be parsed correctly, please provide it in the correct format!') - # extract remaining limit info - host_limit_match = re.match(r"^(\S+)/(\d+)$", host) # HOST/LIMIT - - if host_limit_match: # HOST/LIMIT + # extract remaining limit info: HOST_FORMAT/LIMIT + if (host_limit_match := re.match(r"^(\S+)/(\d+)$", host)) is not None: host, limit = host_limit_match.groups() host_dict["limit"] = limit diff --git a/homcc/common/compression.py b/homcc/common/compression.py index 2a115f7..74b9b80 100644 --- a/homcc/common/compression.py +++ b/homcc/common/compression.py @@ -55,12 +55,21 @@ class Compression(ABC): """Base class for compression algorithms""" @staticmethod - def from_name(name: str) -> Compression: + def from_name(name: Optional[str]) -> Compression: + if name is None: + return NoCompression() + for algorithm in Compression.algorithms(): if algorithm.name() == name: return algorithm() - raise ValueError(f"No compression algorithm with name {name}!") + logger.error( + "No compression algorithm with name '%s'!" + "The remote compilation will be executed without compression enabled!", + name, + ) + + return NoCompression() @abstractmethod def compress(self, data: bytearray) -> bytearray: @@ -79,14 +88,14 @@ def name() -> str: def descriptions() -> List[str]: return [ f"{str(compression.name())}: {compression.__doc__}" - for compression in Compression.algorithms(without_no_compression=True) + for compression in Compression.algorithms(with_no_compression=False) ] @staticmethod - def algorithms(without_no_compression: bool = False) -> List[Type[Compression]]: - algorithms = Compression.__subclasses__() + def algorithms(with_no_compression: bool = True) -> List[Type[Compression]]: + algorithms: List[Type[Compression]] = Compression.__subclasses__() - if without_no_compression: + if not with_no_compression: algorithms.remove(NoCompression) return algorithms @@ -98,6 +107,9 @@ def __eq__(self, other) -> bool: return self.name() == other.name() return False + def __bool__(self): + return True + class NoCompression(Compression): """Class that represents no compression, i.e. the identity function.""" @@ -112,6 +124,9 @@ def decompress(self, data: bytearray) -> bytearray: def name() -> str: return "no_compression" + def __bool__(self): + return False + class LZO(Compression): """Lempel-Ziv-Oberhumer compression algorithm""" diff --git a/homcc/common/messages.py b/homcc/common/messages.py index 4b5dac4..273c6fc 100644 --- a/homcc/common/messages.py +++ b/homcc/common/messages.py @@ -224,8 +224,8 @@ def from_dict(json_dict: dict) -> ArgumentMessage: json_dict["args"], json_dict["cwd"], json_dict["dependencies"], - json_dict.get("profile", None), - Compression.from_name(json_dict.get("compression", str(NoCompression))), + json_dict.get("profile"), + Compression.from_name(json_dict.get("compression")), ) diff --git a/homcc/server/environment.py b/homcc/server/environment.py index f54a83e..9e68cfc 100644 --- a/homcc/server/environment.py +++ b/homcc/server/environment.py @@ -45,7 +45,8 @@ def remove_path(path: Path): remove_path(Path(self.instance_folder)) logger.info("Deleted instance folder '%s'.", self.instance_folder) - def link_dependency_to_cache(self, dependency_file: str, dependency_hash: str, cache: Cache): + @staticmethod + def link_dependency_to_cache(dependency_file: str, dependency_hash: str, cache: Cache): """Links the dependency to a cached dependency with the same hash.""" # first create the folder structure (if needed), else linking won't work dependency_folder = os.path.dirname(dependency_file) @@ -123,7 +124,7 @@ def do_compilation(self, arguments: Arguments) -> CompilationResultMessage: result = self.invoke_compiler(arguments.no_linking()) object_files: List[ObjectFile] = [] - if result.return_code == 0: + if result.return_code == os.EX_OK: for source_file in arguments.source_files: object_file_path: str = self.map_source_file_to_object_file(source_file) object_file_content = Path.read_bytes(Path(object_file_path)) diff --git a/homcc/server/main.py b/homcc/server/main.py index 285d387..ef1cbb0 100755 --- a/homcc/server/main.py +++ b/homcc/server/main.py @@ -42,10 +42,17 @@ def main(): # LOG_LEVEL and VERBOSITY log_level: str = homccd_args_dict["log_level"] - if homccd_args_dict["verbose"] or log_level == "DEBUG" or homccd_config.log_level == LogLevel.DEBUG: + # verbosity implies debug mode + if ( + homccd_args_dict["verbose"] + or homccd_config.verbose + or log_level == "DEBUG" + or homccd_config.log_level == LogLevel.DEBUG + ): logging_config.config |= FormatterConfig.DETAILED logging_config.level = logging.DEBUG + # overwrite verbose debug logging level if log_level is not None: logging_config.level = LogLevel[log_level].value elif homccd_config.log_level is not None: diff --git a/homcc/server/parsing.py b/homcc/server/parsing.py index bf87089..60cc4a1 100644 --- a/homcc/server/parsing.py +++ b/homcc/server/parsing.py @@ -142,14 +142,14 @@ def min_job_limit(value: Union[int, str], minimum: int = 0) -> int: required=False, type=str, choices=[level.name for level in LogLevel], - help="set detail level for log messages", + help=f"set detail level for log messages, defaults to {LogLevel.INFO.name}", ) debug_group.add_argument( "--verbose", required=False, action="store_true", - help="set logging to a detailed DEBUG mode", + help="enable a verbose mode which implies detailed and colored logging of debug messages", ) return vars(parser.parse_args(args)) diff --git a/homcc/server/server.py b/homcc/server/server.py index 486ec6c..86fa77c 100644 --- a/homcc/server/server.py +++ b/homcc/server/server.py @@ -13,7 +13,6 @@ from threading import Lock from typing import Dict, List, Optional, Tuple -from homcc.common.compression import Compression, NoCompression from homcc.common.hashing import hash_file_with_bytes from homcc.common.messages import ( ArgumentMessage, @@ -44,9 +43,7 @@ class TCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): or -1 # fallback error value ) - def __init__( - self, address: Optional[str], port: Optional[int], limit: Optional[int], profiles: Optional[List[str]] - ): + def __init__(self, address: Optional[str], port: Optional[int], limit: Optional[int], profiles: List[str]): address = address or self.DEFAULT_ADDRESS port = port or self.DEFAULT_PORT @@ -63,7 +60,7 @@ def __init__( ) self.profiles_enabled: bool = shutil.which("schroot") is not None - self.profiles: List[str] = profiles or [] + self.profiles: List[str] = profiles self.root_temp_folder: TemporaryDirectory = create_root_temp_folder() @@ -129,8 +126,7 @@ def _handle_message(self, message): def _handle_argument_message(self, message: ArgumentMessage): logger.info("Handling ArgumentMessage...") - profile: Optional[str] = message.get_profile() - if profile is not None: + if (profile := message.get_profile()) is not None: if not self.server.profiles_enabled: logger.info("Refusing client because 'schroot' compilation could not be executed.") self.close_connection( @@ -148,8 +144,7 @@ def _handle_argument_message(self, message: ArgumentMessage): logger.info("Using %s profile.", profile) - compression: Compression = message.get_compression() - if not isinstance(compression, NoCompression): + if compression := message.get_compression(): logger.info("Using %s compression.", compression.name()) self.environment = Environment( @@ -258,7 +253,18 @@ def check_dependencies_exist(self): """Checks if all dependencies exist. If yes, starts compiling. If no, requests missing dependencies.""" if not self._request_next_dependency(): # no further dependencies needed, compile now - result_message = self.environment.do_compilation(self.compiler_arguments) + try: + result_message = self.environment.do_compilation(self.compiler_arguments) + except IOError as error: + logger.error("Error during compilation: %s", error) + + result_message = CompilationResultMessage( + object_files=[], + stdout="", + stderr=f"Invocation of compiler failed:\n{error}", + return_code=os.EX_IOERR, + compression=self.environment.compression, + ) self.request.sendall(result_message.to_bytes()) @@ -319,7 +325,7 @@ def recv_loop(self): recv_bytes += further_recv_bytes def handle(self): - """Handles incoming requests. Returning from this functions means + """Handles incoming requests. Returning from this function means that the connection will be closed from the server side.""" with self.server.current_amount_connections_mutex: self.server.current_amount_connections += 1 @@ -332,7 +338,7 @@ def handle(self): def start_server( - address: Optional[str], port: Optional[int], limit: Optional[int], profiles: Optional[List[str]] = None + address: Optional[str], port: Optional[int], limit: Optional[int], profiles: List[str] ) -> Tuple[TCPServer, threading.Thread]: server: TCPServer = TCPServer(address, port, limit, profiles) diff --git a/tests/client/parsing_test.py b/tests/client/parsing_test.py index 6f91ed8..e98c6d4 100644 --- a/tests/client/parsing_test.py +++ b/tests/client/parsing_test.py @@ -251,17 +251,18 @@ class TestParsingConfig: "", " ", "# HOMCC TEST CONFIG COMMENT", - " # comment with whitespace ", - "COMPILER=g++", - "verbose=TRUE # DEBUG", + " # comment with leading whitespace ", + "COMPILER=g++ # trailing comment", " TIMEOUT = 180 ", "\tCoMpReSsIoN=lzo", "profile=foobar", + "verbose=TRUE", + "log_level=INFO", ] def test_parse_config(self): assert parse_config(self.config) == ClientConfig( - compiler="g++", compression="lzo", verbose="True", timeout="180", profile="foobar" + compiler="g++", compression="lzo", timeout="180", profile="foobar", log_level="INFO", verbose="True" ) def test_parse_loaded_config_file(self, tmp_path: Path): @@ -273,5 +274,5 @@ def test_parse_loaded_config_file(self, tmp_path: Path): config: List[str] = load_config_file(config_file_locations) assert config == self.config assert parse_config(config) == ClientConfig( - compiler="g++", compression="lzo", verbose="True", timeout="180", profile="foobar" + compiler="g++", compression="lzo", timeout="180", profile="foobar", log_level="INFO", verbose="True" ) diff --git a/tests/common/compression_test.py b/tests/common/compression_test.py index 74b55c9..bc32eca 100644 --- a/tests/common/compression_test.py +++ b/tests/common/compression_test.py @@ -2,7 +2,7 @@ from homcc.common.compression import LZO, LZMA, CompressedBytes, NoCompression -test_data = bytearray([0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x6, 0x6, 0x9]) +TEST_DATA: bytearray = bytearray([0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x6, 0x6, 0x9]) class TestCompression: @@ -12,44 +12,48 @@ class TestCompression: def test_lzo(self): lzo = LZO() - compressed_data = lzo.compress(test_data) - assert lzo.decompress(compressed_data) == test_data + compressed_data = lzo.compress(TEST_DATA) + assert lzo + assert lzo.decompress(compressed_data) == TEST_DATA def test_lzma(self): lzma = LZMA() - compressed_data = lzma.compress(test_data) - assert lzma.decompress(compressed_data) == test_data + compressed_data = lzma.compress(TEST_DATA) + assert lzma + assert lzma.decompress(compressed_data) == TEST_DATA def test_no_compression(self): no_compression = NoCompression() - compressed_data = no_compression.compress(test_data) - assert no_compression.decompress(compressed_data) == test_data + compressed_data = no_compression.compress(TEST_DATA) + assert not no_compression + assert no_compression.decompress(compressed_data) == TEST_DATA class TestCompressedBytes: """Tests for the CompressedBytes data structure.""" def test_no_compression(self): - compressed_bytes = CompressedBytes(test_data, NoCompression()) + compressed_bytes = CompressedBytes(TEST_DATA, NoCompression()) - assert compressed_bytes.get_data() == test_data + assert compressed_bytes.get_data() == TEST_DATA assert compressed_bytes.compression == NoCompression() - assert compressed_bytes.to_wire() == test_data - assert len(compressed_bytes) == len(test_data) + assert compressed_bytes.to_wire() == TEST_DATA + assert len(compressed_bytes) == len(TEST_DATA) - assert compressed_bytes.from_wire(test_data, NoCompression()).get_data() == test_data + assert compressed_bytes.from_wire(TEST_DATA, NoCompression()).get_data() == TEST_DATA - def compression_test(self, compression): - compressed_data = compression.compress(test_data) + @staticmethod + def compression_test(compression): + compressed_data = compression.compress(TEST_DATA) - compressed_bytes = CompressedBytes(test_data, compression) + compressed_bytes = CompressedBytes(TEST_DATA, compression) - assert compressed_bytes.get_data() == test_data + assert compressed_bytes.get_data() == TEST_DATA assert compressed_bytes.compression == compression assert compressed_bytes.to_wire() == compressed_data assert len(compressed_bytes) == len(compressed_data) - assert compressed_bytes.from_wire(compressed_data, compression).get_data() == test_data + assert compressed_bytes.from_wire(compressed_data, compression).get_data() == TEST_DATA def test_lzma(self): self.compression_test(LZMA()) diff --git a/tests/e2e/e2e_test.py b/tests/e2e/e2e_test.py index 4890444..80f8a7c 100644 --- a/tests/e2e/e2e_test.py +++ b/tests/e2e/e2e_test.py @@ -25,6 +25,7 @@ def start_server(unused_tcp_port: int) -> subprocess.Popen: "./homcc/server/main.py", f"--listen={TestEndToEnd.ADDRESS}", f"--port={unused_tcp_port}", + "--jobs=1", "--verbose", ] ) @@ -37,6 +38,7 @@ def start_client( return subprocess.run( [ # specify all relevant args explicitly so that config files may not disturb e2e testing "./homcc/client/main.py", + "--log-level=DEBUG", "--verbose", f"--host={TestEndToEnd.ADDRESS}:{unused_tcp_port}{compression_arg}", "--no-profile" if profile is None else f"--profile={profile}", diff --git a/tests/server/server_test.py b/tests/server/server_test.py index 346a384..4c28593 100644 --- a/tests/server/server_test.py +++ b/tests/server/server_test.py @@ -43,7 +43,7 @@ def test_receive_multiple_messages(self, unused_tcp_port): # messages sent by the client with messages that the server deserialized TCPRequestHandler._handle_message = self.patched_handle_message - server, _ = start_server(address="localhost", port=unused_tcp_port, limit=1) + server, _ = start_server(address="localhost", port=unused_tcp_port, limit=1, profiles=[]) with server: arguments = ["-a", "-b", "--help"] cwd = "/home/o.layer/test"