diff --git a/neetbox/config/_config.py b/neetbox/config/_config.py index 0f61d9c4..313724f1 100644 --- a/neetbox/config/_config.py +++ b/neetbox/config/_config.py @@ -4,14 +4,14 @@ # URL: https://gong.host # Date: 20230413 -import collections -from multiprocessing import current_process DEFAULT_GLOBAL_CONFIG = { "daemon": { "enable": True, "allowIpython": False, - "servers": [{"host": "localhost", "port": "20202"},], + "servers": [ + {"host": "localhost", "port": "20202"}, + ], "mode": "detached", "displayName": None, "uploadInterval": 10, diff --git a/neetbox/core/registry.py b/neetbox/core/registry.py index 0075c0dc..fb7e11dc 100644 --- a/neetbox/core/registry.py +++ b/neetbox/core/registry.py @@ -48,7 +48,7 @@ def __new__(cls, name: str) -> "Registry": # not compatible with python below 3.8 def __init__(self, name, *args, **kwargs) -> None: - if not "initialized" in self: + if "initialized" not in self: self["initialized"] = True self.name = name diff --git a/neetbox/daemon/__init__.py b/neetbox/daemon/__init__.py index bfda73a4..a5baaf94 100644 --- a/neetbox/daemon/__init__.py +++ b/neetbox/daemon/__init__.py @@ -5,8 +5,6 @@ # Date: 20230414 import json -import os -import platform import subprocess import time diff --git a/neetbox/daemon/_client_apis.py b/neetbox/daemon/_client_apis.py index 26e00a99..ad7c30c3 100644 --- a/neetbox/daemon/_client_apis.py +++ b/neetbox/daemon/_client_apis.py @@ -5,21 +5,16 @@ # Date: 20230414 +from neetbox.config import get_module_level_config from neetbox.daemon._local_http_client import _local_http_client +from neetbox.logging import logger from neetbox.utils import pkg from neetbox.utils.framing import get_frame_module_traceback -module_name = get_frame_module_traceback().__name__ +module_name = get_frame_module_traceback().__name__ # type: ignore assert pkg.is_installed( "httpx", try_install_if_not=True ), f"{module_name} requires httpx which is not installed" -import json -import time - -import httpx - -from neetbox.config import get_module_level_config -from neetbox.logging import logger logger = logger("NEETBOX DAEMON API") diff --git a/neetbox/daemon/_daemon_client.py b/neetbox/daemon/_daemon_client.py index 41a8cedf..4c0d3666 100644 --- a/neetbox/daemon/_daemon_client.py +++ b/neetbox/daemon/_daemon_client.py @@ -7,6 +7,7 @@ import json import time from threading import Thread +from typing import Union from neetbox.config import get_module_level_config from neetbox.daemon._local_http_client import _local_http_client @@ -15,14 +16,13 @@ __TIME_UNIT_SEC = 0.1 -__upload_thread: Thread = None +__upload_thread: Union[Thread, None] = None def _upload_thread(daemon_config, base_addr, display_name): _ctr = 0 _api_name = "sync" _api_addr = f"{base_addr}/{_api_name}/{display_name}" - global _update_value_dict _disconnect_flag = False _disconnect_retries = 10 while True: @@ -36,18 +36,18 @@ def _upload_thread(daemon_config, base_addr, display_name): _headers = {"Content-Type": "application/json"} try: # upload data - resp = _local_http_client.post(_api_addr, data=_data, headers=_headers) - if resp.is_error: # upload failed + resp = _local_http_client.post(_api_addr, json=_data, headers=_headers) + if resp.is_error: # upload failed raise IOError(f"Failed to upload data to daemon. ({resp.status_code})") except Exception as e: - if _disconnect_flag: # already in retries + if _disconnect_flag: # already in retries _disconnect_retries -= 1 - if not _disconnect_retries: # retry count down exceeded + if not _disconnect_retries: # retry count down exceeded logger.err( "Failed to reconnect to daemon after {10} retries, Trying to launch new daemon..." ) from neetbox.daemon import _try_attach_daemon - + _try_attach_daemon() time.sleep(__TIME_UNIT_SEC) continue @@ -58,7 +58,7 @@ def _upload_thread(daemon_config, base_addr, display_name): else: if not _disconnect_flag: continue - logger.ok(f"Successfully reconnected to daemon.") + logger.ok("Successfully reconnected to daemon.") _disconnect_flag = False _disconnect_retries = 10 diff --git a/neetbox/daemon/_daemon_launcher.py b/neetbox/daemon/_daemon_launcher.py index 4685687a..9e396af5 100644 --- a/neetbox/daemon/_daemon_launcher.py +++ b/neetbox/daemon/_daemon_launcher.py @@ -1,15 +1,14 @@ -print("========= Daemon Launcher =========") - import argparse import json -import os import sys from neetbox.daemon._flask_server import daemon_process + # sys.stdout=open(r'D:\Projects\ML\neetbox\logdir\daemon.log', 'a+') # from neetbox.daemon._local_http_client import _local_http_client +print("========= Daemon Launcher =========") def run(): @@ -26,6 +25,5 @@ def run(): print("_daemon_launcher is starting with __name__ =", __name__, " and args =", sys.argv) - run() print("_daemon_launcher: exiting") diff --git a/neetbox/daemon/_flask_server.py b/neetbox/daemon/_flask_server.py index 15ad0ce4..ed2be203 100644 --- a/neetbox/daemon/_flask_server.py +++ b/neetbox/daemon/_flask_server.py @@ -4,13 +4,6 @@ # URL: https://gong.host # Date: 20230414 -from neetbox.utils import pkg -from neetbox.utils.framing import get_frame_module_traceback - -module_name = get_frame_module_traceback().__name__ -assert pkg.is_installed( - "flask", try_install_if_not=True -), f"{module_name} requires flask which is not installed" import sys import time from threading import Thread @@ -18,6 +11,13 @@ from flask import Flask, abort, json, request from neetbox.config import get_module_level_config +from neetbox.utils import pkg +from neetbox.utils.framing import get_frame_module_traceback + +module_name = get_frame_module_traceback().__name__ # type: ignore +assert pkg.is_installed( + "flask", try_install_if_not=True +), f"{module_name} requires flask which is not installed" _STAT_POOL = {} __DAEMON_SHUTDOWN_IF_NO_UPLOAD_TIMEOUT_SEC = 60 * 60 * 12 # 12 Hours @@ -77,7 +77,9 @@ def __sleep_and_shutdown(secs=3): time.sleep(secs=secs) sys.exit(0) - Thread(target=__sleep_and_shutdown, args=(3)).start() # shutdown after 3 seconds + Thread( + target=__sleep_and_shutdown, args=(3) + ).start() # shutdown after 3 seconds return "ok" def _count_down_thread(): diff --git a/neetbox/daemon/_local_http_client.py b/neetbox/daemon/_local_http_client.py index 4ce734b6..f53ee225 100644 --- a/neetbox/daemon/_local_http_client.py +++ b/neetbox/daemon/_local_http_client.py @@ -1,9 +1,10 @@ -import httpx import logging +import httpx + httpx_logger = logging.getLogger("httpx") httpx_logger.setLevel(logging.ERROR) - + __no_proxy = { "http://": None, "https://": None, @@ -11,7 +12,7 @@ def __load_http_client(): - __local_http_client = httpx.Client(proxies=__no_proxy) + __local_http_client = httpx.Client(proxies=__no_proxy) # type: ignore return __local_http_client diff --git a/neetbox/daemon/daemonable_process.py b/neetbox/daemon/daemonable_process.py index 6dba4ef7..839b0911 100644 --- a/neetbox/daemon/daemonable_process.py +++ b/neetbox/daemon/daemonable_process.py @@ -1,9 +1,8 @@ -import multiprocessing import os import subprocess import sys import time -from typing import * +from typing import List, Literal is_ms_windows = "win32" in sys.platform or "cygwin" in sys.platform @@ -59,7 +58,7 @@ def start(self): creationflags = { "attached": 0, "shared": 0, - "detached": subprocess.CREATE_NO_WINDOW, + "detached": subprocess.CREATE_NO_WINDOW, # type: ignore (only for windows) }[self.__mode] popen = subprocess.Popen( diff --git a/neetbox/integrations/resource.py b/neetbox/integrations/resource.py index 60b72399..fd2bc6f9 100644 --- a/neetbox/integrations/resource.py +++ b/neetbox/integrations/resource.py @@ -12,7 +12,7 @@ from functools import partial from random import random from threading import Event -from typing import Any, Dict, Iterable, List, Union +from typing import Dict, List, Union from urllib.request import urlopen import numpy as np @@ -25,7 +25,6 @@ TextColumn, TimeRemainingColumn, TransferSpeedColumn, - track, ) from neetbox.integrations import engine diff --git a/neetbox/logging/__init__.py b/neetbox/logging/__init__.py index 1c9027e1..45a9f568 100644 --- a/neetbox/logging/__init__.py +++ b/neetbox/logging/__init__.py @@ -3,10 +3,6 @@ _cfg = get_module_level_config() logger.set_log_dir(_cfg["logdir"]) -from neetbox.logging.logger import ( - LogMetadata, - LogSplitStrategies, - SplitStrategyCallable, -) +from neetbox.logging.logger import LogSplitStrategies __all__ = ["logger", "LogSplitStrategies"] diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index d01e25e5..2f650e3a 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -5,11 +5,10 @@ # Date: 20230318 from dataclasses import dataclass -import os -import warnings from random import random from typing import Optional + @dataclass class LogStyle: color: Optional[str] = None @@ -22,7 +21,7 @@ class LogStyle: split_char_cmd = " > " split_char_identity = "/" split_char_txt = " | " - + @classmethod def get_supported_colors(cls): return ["red", "green", "blue", "cyan", "yellow", "magenta"] diff --git a/neetbox/logging/logger.py b/neetbox/logging/logger.py index cdfd3814..f78a4ad0 100644 --- a/neetbox/logging/logger.py +++ b/neetbox/logging/logger.py @@ -401,7 +401,6 @@ def catch( not isclass(exception_type) or not issubclass(exception_type, BaseException) ): return self.catch()(exception_type) - logger = self class Catcher: def __init__(self, from_decorator): @@ -415,8 +414,6 @@ def __exit__(self, type_, value, traceback_): return if not issubclass(type_, exception_type): return False - from_decorator = self._from_decorator - catch_options = [(type_, value, traceback_)] if handler: handler(traceback_) # logger.log( @@ -456,19 +453,18 @@ def catch_wrapper(*args, **kwargs): return catch_wrapper return Catcher(False) - + def mention(self, func): @functools.wraps(func) def with_logging(*args, **kwargs): - self.log(f"Currently running: {func.__name__}",traceback=3) + self.log(f"Currently running: {func.__name__}", traceback=3) return func(*args, **kwargs) + return with_logging def banner(self, text, font: Optional[str] = None): from pyfiglet import Figlet, FigletFont - from neetbox.utils import pkg - builtin_font_list = [ "ansiregular", "ansishadow", @@ -575,7 +571,7 @@ def _bind_file(self, path): return self def file_bend(self) -> bool: - return self.file_writer != None + return self.file_writer is not None DEFAULT_LOGGER = Logger(None) diff --git a/neetbox/pipeline/__init__.py b/neetbox/pipeline/__init__.py index e866d5f0..45a5f704 100644 --- a/neetbox/pipeline/__init__.py +++ b/neetbox/pipeline/__init__.py @@ -4,7 +4,6 @@ # URL: https://gong.host # Date: 20230417 -from neetbox.pipeline._pipe import Pipe from neetbox.pipeline._signal_and_slot import listen, watch __all__ = ["watch", "listen"] diff --git a/neetbox/pipeline/_pipe.py b/neetbox/pipeline/_pipe.py index 876bf173..89f6541d 100644 --- a/neetbox/pipeline/_pipe.py +++ b/neetbox/pipeline/_pipe.py @@ -5,9 +5,6 @@ # Date: 20230413 from neetbox.core import Registry -from neetbox.logging import logger -from neetbox.utils import pkg -from neetbox.utils.mvc import Singleton pipes = Registry("__pipe") diff --git a/neetbox/torch/arch/cnn.py b/neetbox/torch/arch/cnn.py index 1bbd91c6..441fa43a 100644 --- a/neetbox/torch/arch/cnn.py +++ b/neetbox/torch/arch/cnn.py @@ -4,11 +4,7 @@ # URL: https://gong.host # Date: 20230315 -import torch import torch.nn as nn -import torch.nn.functional as F - -from neetbox.utils import pkg def ConvNxN( diff --git a/neetbox/torch/arch/mask_boundary_finder.py b/neetbox/torch/arch/mask_boundary_finder.py index 077be89b..97681301 100644 --- a/neetbox/torch/arch/mask_boundary_finder.py +++ b/neetbox/torch/arch/mask_boundary_finder.py @@ -1,8 +1,6 @@ import torch import torch.nn.functional as F -from neetbox.utils import pkg - class MaskEdgeDetecter(torch.nn.Module): """generate the 'edge bar' for a 0-1 mask Groundtruth of a image diff --git a/neetbox/torch/profile.py b/neetbox/torch/profile.py index b69f1265..45ad7404 100644 --- a/neetbox/torch/profile.py +++ b/neetbox/torch/profile.py @@ -91,7 +91,7 @@ def profile( f"Seems your model has an imbalanced performance peek between CUDA side synchronous test and none-sync one. Consider raising speedtest loop times (currently {speedtest} +2) to have a stable result." ) logger.debug( - f"Note that the CUDA side synchronous speedtest is more reliable since you are using a GPU." + "Note that the CUDA side synchronous speedtest is more reliable since you are using a GPU." ) if profiling: if speedtest: diff --git a/neetbox/utils/__init__.py b/neetbox/utils/__init__.py index d5327bb1..029efd4d 100644 --- a/neetbox/utils/__init__.py +++ b/neetbox/utils/__init__.py @@ -1,3 +1,3 @@ -from neetbox.utils._package import PipPackageHealper as pkg +from neetbox.utils._package import pipPackageHealper as pkg __all__ = ["pkg"] diff --git a/neetbox/utils/_package.py b/neetbox/utils/_package.py index 90d49ad6..1e6fc86b 100644 --- a/neetbox/utils/_package.py +++ b/neetbox/utils/_package.py @@ -18,7 +18,7 @@ def install(self, package, terminate=False): _installed = False while retry: if not retry: - error_str = f"Bad Input" + error_str = "Bad Input" raise ValueError(error_str) print(f"{caller_name} want to install {package} via pip.") choice = input("Make your choice: [y]/n") @@ -55,7 +55,7 @@ def is_installed(self, package: str, try_install_if_not: Union[str, bool] = True importlib.import_module(package) self.installed_packages.append(package) return True - except: + except: # noqa package_name_install = ( package if type(try_install_if_not) is bool else try_install_if_not ) @@ -68,4 +68,4 @@ def is_installed(self, package: str, try_install_if_not: Union[str, bool] = True # singleton -PipPackageHealper = PipPackageHealper() +pipPackageHealper = PipPackageHealper() diff --git a/neetbox/utils/framing.py b/neetbox/utils/framing.py index aab3c11b..f5c55b0c 100644 --- a/neetbox/utils/framing.py +++ b/neetbox/utils/framing.py @@ -5,7 +5,9 @@ # Date: 20230319 import inspect +import types from os import path +from typing import Union def get_frame_traceback(traceback=1): @@ -27,13 +29,10 @@ def get_frame_class_traceback(traceback=1): return None -def get_frame_module_traceback(traceback=1): +def get_frame_module_traceback(traceback=1) -> Union[types.ModuleType, None]: frame = get_frame_traceback(traceback + 1) - try: - module = inspect.getmodule(frame[0]) - return module - except: - return None + module = inspect.getmodule(frame[0]) + return module def get_frame_filepath_traceback(traceback=1): @@ -63,12 +62,9 @@ def parse(self, frame): self.class_obj = self.locals["self"].__class__ if self.class_obj: self.class_name = self.class_obj.__name__ - try: - module = inspect.getmodule(frame[0]) - self.module = module - self.module_name = module.__name__ - except: - pass + module: Union[types.ModuleType, None] = inspect.getmodule(frame[0]) + self.module = module + self.module_name = module.__name__ if module else None self.filepath = path.abspath(frame.filename) self.filename = path.basename(frame.filename) return self diff --git a/pyproject.toml b/pyproject.toml index 9385ae57..779666be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ maintainers = [ "PommesPeter ", "PaperCube ", "PuQing ", - "Lideming " + "Lideming ", ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 235173d6..7f0692d2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,3 @@ -from neetbox.utils import format - - def test_package_installed(): from neetbox.utils import pkg