diff --git a/.gitignore b/.gitignore index 6e10f244..c5c91af2 100644 --- a/.gitignore +++ b/.gitignore @@ -19,10 +19,7 @@ build_and_reinstall_wheel.cmd # log files logs/ log/ -for_debug.txt -.neethistory/ -.neetory .log # others -other/ +neetfiles \ No newline at end of file diff --git a/PKGBUILD b/PKGBUILD index 3cc0f0ba..38aec4ba 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -1,5 +1,5 @@ pkgname=python-neetbox -pkgver=0.4.2 +pkgver=0.4.3 pkgrel=1 pkgdesc="Logging/Debugging/Tracing/Managing/Facilitating long running python projects, especially a replacement of tensorboard for deep learning projects" arch=('any') diff --git a/frontend/src/components/dashboard/project/hardware/gpugraph.tsx b/frontend/src/components/dashboard/project/hardware/gpugraph.tsx index 4751a23e..1143f0e5 100644 --- a/frontend/src/components/dashboard/project/hardware/gpugraph.tsx +++ b/frontend/src/components/dashboard/project/hardware/gpugraph.tsx @@ -2,7 +2,7 @@ import { useCallback } from "react"; import { ECharts } from "../../../echarts"; import { GpuInfo } from "../../../../services/types"; import { TimeDataMapper } from "../../../../utils/timeDataMapper"; -import { getTimeAxisOptions, green2redHue } from "./utils"; +import { getTimeAxisOptions, percent2hue } from "./utils"; import { GraphWrapper } from "./graphWrapper"; import "./gpugraph.css"; @@ -84,7 +84,7 @@ export const GPUGraph = ({ data }: { data: TimeDataMapper }) => { className="gpu-temperature" style={{ backgroundColor: - "hsl(" + green2redHue(lastValue.temperature) + ", 90%, var(--temperature-bg-brightness))", + "hsl(" + percent2hue(lastValue.temperature) + ", 90%, var(--temperature-bg-brightness))", }} > {lastValue.temperature}℃ diff --git a/frontend/src/components/dashboard/project/hardware/utils.ts b/frontend/src/components/dashboard/project/hardware/utils.ts index 7ed81fcf..94addb8e 100644 --- a/frontend/src/components/dashboard/project/hardware/utils.ts +++ b/frontend/src/components/dashboard/project/hardware/utils.ts @@ -12,6 +12,6 @@ export function getTimeAxisOptions(mapper: TimeDataMapper) { }; } -export function green2redHue(value) { - return ((1 - value) * 120).toString(10); +export function percent2hue(value) { + return ((100 - value) * 1.2).toString(10); } diff --git a/frontend/src/components/dashboard/project/runSelect.tsx b/frontend/src/components/dashboard/project/runSelect.tsx index dde50f57..7dd3a815 100644 --- a/frontend/src/components/dashboard/project/runSelect.tsx +++ b/frontend/src/components/dashboard/project/runSelect.tsx @@ -48,7 +48,7 @@ export const RunSelect = memo((props: any) => { <> {isOnlineRun ? ( Online - ) : runId != items[0].runId ? ( + ) : runId != items?.[0].runId ? ( History ) : ( Offline diff --git a/neetbox.toml b/neetbox.toml index 10cc1b7f..349e45cc 100644 --- a/neetbox.toml +++ b/neetbox.toml @@ -1,5 +1,5 @@ name = "neetbox" -version = "0.4.2" +version = "0.4.3" projectId = "3edbc5ed-8e16-44fe-8c25-949a8cb833e6" [logging] diff --git a/neetbox/_protocol.py b/neetbox/_protocol.py index 56d1e3e9..78434917 100644 --- a/neetbox/_protocol.py +++ b/neetbox/_protocol.py @@ -151,5 +151,4 @@ def merge(cls, x: Union["EventMsg", dict], y: Union["EventMsg", dict]): IMAGE_TABLE_NAME = "image" NEETBOX_VERSION = version("neetbox") -HISTORY_FILE_ROOT = ".neethistory" -HISTORY_FILE_TYPE_NAME = "neetory" +NEET_FILE_FOLDER = "neetfiles" \ No newline at end of file diff --git a/neetbox/logging/_writer.py b/neetbox/logging/_writer.py index 7c07e8ce..49d01808 100644 --- a/neetbox/logging/_writer.py +++ b/neetbox/logging/_writer.py @@ -7,16 +7,13 @@ import json import os from abc import ABC, abstractmethod -from collections import defaultdict from dataclasses import dataclass from datetime import datetime from typing import Any, Optional - from rich.console import Console from rich.table import Table from rich.text import Text - -from neetbox.logging.formatting import LogStyle, colored_text, styled_text +from neetbox.logging.formatting import LogStyle from neetbox.utils import formatting from neetbox.utils.framing import TracebackIdentity @@ -89,43 +86,39 @@ def __repr__(self) -> str: # ================== CONSOLE LOG WRITER ===================== -class DefaultDictThatReturnsKeyOnMissing(defaultdict): - def __missing__(self, key): - return key - - -_console_prefix_2_colored_text = DefaultDictThatReturnsKeyOnMissing( - str, - { - "ok": colored_text("[ok]", "green"), - "debug": colored_text("[debug]", "cyan"), - "info": colored_text("[info]", "white"), - "warning": colored_text("[warning]", "yellow"), - "mention": colored_text("[mention]", "yellow"), - "error": colored_text("[error]", "red"), - }, -) +def prefix2RichText(prefix: str): + return { + "ok": Text("[ok]", style="green"), + "debug": Text("[debug]", style="cyan"), + "info": Text("[info]", style="white"), + "warning": Text("[warning]", style="yellow"), + "mention": Text("[mention]", style="yellow"), + "error": Text("[error]", style="red"), + }.get(prefix, Text(prefix)) class __ConsoleLogWriter(LogWriter): _console = Console() def write(self, raw_log: RawLog): - _msg_dict = raw_log.json - _style = raw_log.style - _prefix = _msg_dict["series"] - _prefix = _console_prefix_2_colored_text[_prefix] + " " if _prefix else _prefix + msg_dict = raw_log.json + log_style = raw_log.style + style = " ".join( + [x for x in [log_style.console_color, log_style.text_style] if x is not None] + ) + prefix = msg_dict["series"] + prefix = prefix2RichText(prefix) if prefix else None table = Table(show_header=False, box=None, expand=True) table.add_column(justify="left") table.add_column(justify="right") - rich_msg = str( - _prefix - + styled_text(_msg_dict["whom"], style=_style) - + _style.split_char_cmd * min(len(_msg_dict["whom"]), 1) - + _msg_dict["message"] + whom_text = Text(msg_dict["whom"], style) + split_text = ( + Text(log_style.split_char_cmd, style="dim " + style) if len(msg_dict["whom"]) else "" ) - time_text = Text(_msg_dict["timestamp"], style="dim") - table.add_row(rich_msg, time_text) + message_text = Text(msg_dict["message"], style="default") + time_text = Text(msg_dict["timestamp"], style="default dim") + prefix_and_time = prefix + " " + time_text if prefix else time_text + table.add_row(whom_text + split_text + message_text, prefix_and_time) self.__class__._console.print(table) diff --git a/neetbox/logging/formatting.py b/neetbox/logging/formatting.py index d73f3e7b..522c1840 100644 --- a/neetbox/logging/formatting.py +++ b/neetbox/logging/formatting.py @@ -33,10 +33,6 @@ def get_supported_colors(cls): def get_supported_text_style(cls): return ["bold", "italic", "blink"] - def parse(self, pattern: str): - # todo - pass - def set_color(self, color: str): self.fore = color return self @@ -62,30 +58,4 @@ def randcolor(self): # self.back = colors[(split_index + index_offset) % len(colors)] # self.fore = colors[(split_index - index_offset) % len(colors)] self.console_color = colors[int(random() * len(colors))] - return self - - -DEFAULT_STYLE = LogStyle() - - -def styled_text(text, style: LogStyle): - attributes = [] - if style.console_color: - attributes.append(style.console_color) - if style.text_style: - attributes.append(style.text_style) - render_stack = [] - _prefix = "" - _postfix = "" - while len(attributes): - attr = attributes.pop(-1) - _prefix += f"[{attr}]" - render_stack.append(attr) - while len(render_stack): - _postfix += f"[/{render_stack.pop(-1)}]" - return f"{_prefix}{text}{_postfix}" - - -def colored_text(text: str, color): - text = text.replace("[", r"\[") - return f"[{color}]{text}[/{color}]" + return self \ No newline at end of file diff --git a/neetbox/server/_bridge.py b/neetbox/server/_bridge.py index 9fdb9786..1b2b79f5 100644 --- a/neetbox/server/_bridge.py +++ b/neetbox/server/_bridge.py @@ -5,15 +5,12 @@ # Date: 20231204 from typing import Dict - from websocket_server import WebsocketServer - from neetbox._protocol import * from neetbox.logging import LogStyle, logger +from .db.project import ProjectDB -from .db import history - -logger = logger("NEETBOX", LogStyle(skip_writers=["ws"])) +logger = logger("Bridge", LogStyle(skip_writers=["ws"])) class Bridge: @@ -41,7 +38,7 @@ class Bridge: status: dict cli_ws_dict: dict # { run_id : client} web_ws_list: list # since web do not have run id, use list instead of dict - historyDB: history.DBConnection + historyDB: ProjectDB def __new__(cls, project_id: str, **kwargs) -> None: """Create Bridge of project id, return the old one if already exist @@ -58,7 +55,7 @@ def __new__(cls, project_id: str, **kwargs) -> None: [] ) # frontend ws sids. client data should be able to be shown on multiple frontend flag_auto_load_db = kwargs["auto_load_db"] if "auto_load_db" in kwargs else True - new_bridge.historyDB = history.get_db_of_id(project_id) if flag_auto_load_db else None + new_bridge.historyDB = ProjectDB.get_db_of_id(project_id) if flag_auto_load_db else None cls._id2bridge[project_id] = new_bridge logger.info(f"created new Bridge for project id '{project_id}'") return cls._id2bridge[project_id] @@ -66,7 +63,7 @@ def __new__(cls, project_id: str, **kwargs) -> None: def __del__(self): # on delete logger.info(f"bridge project id {self.project_id} handling on delete...") if 0 == len(self.get_run_ids()): # if there is no active run id - self.historyDB.finialize() + self.historyDB.delete_files() del self.historyDB # delete history db logger.info(f"bridge of project id {self.project_id} deleted.") @@ -90,7 +87,7 @@ def of_id(cls, project_id: str) -> "Bridge": return bridge @classmethod - def from_db(cls, db: history.DBConnection) -> "Bridge": + def from_db(cls, db: ProjectDB) -> "Bridge": project_id = db.fetch_db_project_id() target_bridge = Bridge(project_id, auto_load_db=False) if target_bridge.historyDB is not None: @@ -100,7 +97,7 @@ def from_db(cls, db: history.DBConnection) -> "Bridge": @classmethod def load_histories(cls): - db_list = history.get_db_list() + db_list = ProjectDB.get_db_list() logger.log(f"found {len(db_list)} history db.") for _, history_db in db_list: cls.from_db(history_db) diff --git a/neetbox/server/_daemon_server_launch_script.py b/neetbox/server/_daemon_server_launch_script.py index 58ffc755..2ab58fec 100644 --- a/neetbox/server/_daemon_server_launch_script.py +++ b/neetbox/server/_daemon_server_launch_script.py @@ -2,9 +2,6 @@ import json import sys -from neetbox.server._server import server_process - - def run(argv): if len(argv) <= 1: print("_daemon_: Warning: empty daemon_config") @@ -17,6 +14,7 @@ def run(argv): print(args.config) daemon_config = json.loads(args.config) print("Daemon started with config:", daemon_config) + from ._server import server_process server_process(daemon_config) diff --git a/neetbox/server/_flask_server.py b/neetbox/server/_flask_server.py index 493eb418..824d2e3e 100644 --- a/neetbox/server/_flask_server.py +++ b/neetbox/server/_flask_server.py @@ -9,14 +9,11 @@ import time from threading import Thread from typing import Union - import werkzeug from flask import Response, abort, json, redirect, request, send_from_directory - import neetbox from neetbox._protocol import * -from neetbox.server._bridge import Bridge - +from ._bridge import Bridge from .db import QueryCondition werkzeug_log = logging.getLogger("werkzeug") @@ -28,7 +25,7 @@ def get_flask_server(debug=False): from neetbox.logging import LogStyle from neetbox.logging.logger import Logger, LogLevel - logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) + logger = Logger("FLASK", LogStyle(skip_writers=["ws"])) if debug: logger.set_log_level(LogLevel.DEBUG) diff --git a/neetbox/server/_server.py b/neetbox/server/_server.py index ca405791..9c54fa6a 100644 --- a/neetbox/server/_server.py +++ b/neetbox/server/_server.py @@ -7,9 +7,9 @@ import setproctitle from neetbox._protocol import * -from neetbox.server._bridge import Bridge -from neetbox.server._flask_server import get_flask_server -from neetbox.server._websocket_server import get_web_socket_server +from ._bridge import Bridge +from ._flask_server import get_flask_server +from ._websocket_server import get_web_socket_server def server_process(cfg, debug=False): @@ -17,7 +17,7 @@ def server_process(cfg, debug=False): from neetbox.logging import LogStyle from neetbox.logging.logger import Logger - logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) + logger = Logger("SERVER", LogStyle(skip_writers=["ws"])) # load bridges Bridge.load_histories() # load history files diff --git a/neetbox/server/_websocket_server.py b/neetbox/server/_websocket_server.py index 3f5af2ce..9803aea3 100644 --- a/neetbox/server/_websocket_server.py +++ b/neetbox/server/_websocket_server.py @@ -21,7 +21,7 @@ def get_web_socket_server(config, debug=False): from neetbox.server._bridge import Bridge console = Console() - logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) + logger = Logger("WS SERVER", LogStyle(skip_writers=["ws"])) if debug: logger.set_log_level(LogLevel.DEBUG) diff --git a/neetbox/server/db/__init__.py b/neetbox/server/db/__init__.py index e88dff74..744f04d9 100644 --- a/neetbox/server/db/__init__.py +++ b/neetbox/server/db/__init__.py @@ -4,11 +4,9 @@ # Github: github.com/visualDust # Date: 20231201 -from . import _history as history -from ._condition import DbQueryFetchType, DbQuerySortType, QueryCondition +from .condition import DbQueryFetchType, DbQuerySortType, QueryCondition __all__ = [ - "history", "DbQueryFetchType", "DbQuerySortType", "QueryCondition", diff --git a/neetbox/server/db/_configs.py b/neetbox/server/db/_configs.py new file mode 100644 index 00000000..e69de29b diff --git a/neetbox/server/db/_condition.py b/neetbox/server/db/condition.py similarity index 73% rename from neetbox/server/db/_condition.py rename to neetbox/server/db/condition.py index a2d486a5..c8cc0798 100644 --- a/neetbox/server/db/_condition.py +++ b/neetbox/server/db/condition.py @@ -1,7 +1,7 @@ -from enum import Enum import json -from typing import Dict, Tuple, Union +from enum import Enum from neetbox._protocol import * +from typing import Dict, Tuple, Union class DbQueryFetchType(str, Enum): @@ -99,33 +99,40 @@ def from_json(cls, json_data): order=order, ) - def dumps(self): + def dumpt(self): + query_cond_vars = [] # === id condition === - _id_cond = "" + _id_cond_str = "" if self.id_range[0]: _id_0, _id_1 = self.id_range - _id_cond = ( - f"{ID_COLUMN_NAME}=={_id_0}" - if _id_1 is None - else f"{ID_COLUMN_NAME} BETWEEN {_id_0} AND {_id_1}" - ) + if _id_1 is None: + _id_cond_str = f"{ID_COLUMN_NAME} = ?" + query_cond_vars.append(_id_0) + else: + _id_cond_str = f"{ID_COLUMN_NAME} BETWEEN ? AND ?" + query_cond_vars.append(_id_0) + query_cond_vars.append(_id_1) # === timestamp condition === - _timestamp_cond = "" + _timestamp_cond_str = "" if self.timestamp_range[0]: _ts_0, _ts_1 = self.timestamp_range - _timestamp_cond = ( - f"{TIMESTAMP_COLUMN_NAME}>='{_ts_0}'" - if _ts_1 is None - else f"{TIMESTAMP_COLUMN_NAME} BETWEEN '{_ts_0} AND '{_ts_1}" - ) + if _ts_1 is None: + _timestamp_cond_str = f"{TIMESTAMP_COLUMN_NAME} >= ?" + query_cond_vars.append(_ts_0) + else: + _timestamp_cond_str = f"{TIMESTAMP_COLUMN_NAME} BETWEEN ? AND ?" + query_cond_vars.append(_ts_0) + query_cond_vars.append(_ts_1) # === series condition === - _series_cond = "" + _series_cond_str = "" if self.series: - _series_cond = f"{SERIES_COLUMN_NAME} == '{self.series}'" + _series_cond_str = f"{SERIES_COLUMN_NAME} = ?" + query_cond_vars.append(self.series) # === run-id condition === - _run_id_cond = "" + _run_id_cond_str = "" if self.run_id: - _run_id_cond = f"{RUN_ID_COLUMN_NAME} == {self.run_id}" + _run_id_cond_str = f"{RUN_ID_COLUMN_NAME} = ?" + query_cond_vars.append(self.run_id) # === ORDER BY === _order_cond = f"ORDER BY " if self.order else "" if self.order: @@ -135,21 +142,21 @@ def dumps(self): ) _order_cond = _order_cond[:-2] # remove last ',' # === LIMIT === - _limit_cond = f"LIMIT {self.limit}" if self.limit else "" + _limit_cond_str = f"LIMIT {self.limit}" if self.limit else "" # === concat conditions === - query_conditions = [] - for cond in [_id_cond, _timestamp_cond, _series_cond, _run_id_cond]: + query_condition_strs = [] + for cond in [_id_cond_str, _timestamp_cond_str, _series_cond_str, _run_id_cond_str]: if cond: - query_conditions.append(cond) - query_conditions = " AND ".join(query_conditions) + query_condition_strs.append(cond) + query_condition_strs = " AND ".join(query_condition_strs) # === concat order by and limit === order_and_limit = [] - for cond in [_order_cond, _limit_cond]: + for cond in [_order_cond, _limit_cond_str]: if cond: order_and_limit.append(cond) order_and_limit = " ".join(order_and_limit) # result - if query_conditions: - query_conditions = f"WHERE {query_conditions}" - query_condition_str = f"{query_conditions} {order_and_limit}" - return query_condition_str + if query_condition_strs: + query_condition_strs = f"WHERE {query_condition_strs}" + query_cond_str = f"{query_condition_strs} {order_and_limit}" + return query_cond_str, query_cond_vars \ No newline at end of file diff --git a/neetbox/server/db/_history.py b/neetbox/server/db/project.py similarity index 86% rename from neetbox/server/db/_history.py rename to neetbox/server/db/project.py index 9fdfcc75..595deafd 100644 --- a/neetbox/server/db/_history.py +++ b/neetbox/server/db/project.py @@ -4,24 +4,25 @@ # Github: github.com/visualDust # Date: 20231201 -import collections -import json import os +import json +import atexit import sqlite3 -from datetime import datetime +import collections from typing import Union - +from datetime import datetime from neetbox._protocol import * from neetbox.logging import LogStyle from neetbox.logging.logger import Logger from neetbox.utils import ResourceLoader +from .condition import * -from ._condition import * - -logger = Logger("NEETBOX", LogStyle(skip_writers=["ws"])) +logger = Logger("PROJECT DB", LogStyle(skip_writers=["ws"])) +DB_PROJECT_FILE_ROOT = f"{NEET_FILE_FOLDER}/history" +DB_PROJECT_FILE_TYPE_NAME = "projectdb" -class DBConnection: +class ProjectDB: # static things _path2dbc = {} _id2dbc = {} @@ -32,11 +33,11 @@ class DBConnection: connection: sqlite3.Connection # the db connection _inited_tables: collections.defaultdict - def __new__(cls, project_id: str = None, path: str = None, **kwargs) -> "DBConnection": + def __new__(cls, project_id: str = None, path: str = None, **kwargs) -> "ProjectDB": if path is None and project_id is None: raise RuntimeError(f"please provide at least project id or path when creating db") if path is None: # make path from project id - path = f"{HISTORY_FILE_ROOT}/{project_id}.{HISTORY_FILE_TYPE_NAME}" + path = f"{DB_PROJECT_FILE_ROOT}/{project_id}.{DB_PROJECT_FILE_TYPE_NAME}" if path in cls._path2dbc: return cls._path2dbc[path] if project_id in cls._id2dbc: @@ -66,13 +67,14 @@ def __new__(cls, project_id: str = None, path: str = None, **kwargs) -> "DBConne logger.ok(f"History file(version={_db_file_version}) for project id '{project_id}' loaded.") return new_dbc - def finialize(self): - if self.project_id not in DBConnection._id2dbc: + def delete_files(self): + """delete related files of db""" + if self.project_id not in ProjectDB._id2dbc: logger.err( RuntimeError(f"could not find db to delete with project id {self.project_id}") ) - del DBConnection._id2dbc[self.project_id] - del DBConnection._path2dbc[self.file_path] + del ProjectDB._id2dbc[self.project_id] + del ProjectDB._path2dbc[self.file_path] logger.info(f"deleting history DB for project id {self.project_id}...") if self.connection: try: @@ -106,7 +108,7 @@ def items(cls): def of_project_id(cls, project_id): if project_id in cls._id2dbc: return cls._id2dbc[project_id] - return DBConnection(project_id) + return ProjectDB(project_id) def _execute(self, query, *args, fetch: DbQueryFetchType = DbQueryFetchType.ALL, **kwargs): cur = self.connection.cursor() @@ -295,22 +297,18 @@ def read_json(self, table_name: str, condition: QueryCondition = None): return [] if condition and isinstance(condition.run_id, str): condition.run_id = self.get_id_of_run_id(condition.run_id) # convert run id - condition = condition.dumps() if condition else "" - sql_query = f"SELECT {', '.join((ID_COLUMN_NAME, TIMESTAMP_COLUMN_NAME,SERIES_COLUMN_NAME, JSON_COLUMN_NAME))} FROM {table_name} {condition}" - result, _ = self._query(sql_query, fetch=DbQueryFetchType.ALL) - try: - result = [ - { - ID_COLUMN_NAME: w, - TIMESTAMP_COLUMN_NAME: x, - SERIES_COLUMN_NAME: y, - JSON_COLUMN_NAME: json.loads(z), - } - for w, x, y, z in result - ] - except: - print(result) - raise + cond_str, cond_vars = condition.dumpt() if condition else "" + sql_query = f"SELECT {', '.join((ID_COLUMN_NAME, TIMESTAMP_COLUMN_NAME,SERIES_COLUMN_NAME, JSON_COLUMN_NAME))} FROM {table_name} {cond_str}" + result, _ = self._query(sql_query, *cond_vars, fetch=DbQueryFetchType.ALL) + result = [ + { + ID_COLUMN_NAME: w, + TIMESTAMP_COLUMN_NAME: x, + SERIES_COLUMN_NAME: y, + JSON_COLUMN_NAME: json.loads(z), + } + for w, x, y, z in result + ] return result def set_status(self, run_id: str, series: str, json_data): @@ -336,9 +334,9 @@ def get_status(self, run_id: str = None, series: str = None): condition = QueryCondition(run_id=run_id, series=series) if isinstance(condition.run_id, str): condition.run_id = self.get_id_of_run_id(run_id) - condition = condition.dumps() - sql_query = f"SELECT {', '.join((RUN_ID_COLUMN_NAME, SERIES_COLUMN_NAME, JSON_COLUMN_NAME))} FROM {STATUS_TABLE_NAME} {condition}" - query_result, _ = self._query(sql_query, fetch=DbQueryFetchType.ALL) + cond_str, cond_vars = condition.dumpt() + sql_query = f"SELECT {', '.join((RUN_ID_COLUMN_NAME, SERIES_COLUMN_NAME, JSON_COLUMN_NAME))} FROM {STATUS_TABLE_NAME} {cond_str}" + query_result, _ = self._query(sql_query, *cond_vars, fetch=DbQueryFetchType.ALL) result = {} for id_of_runid, series_name, value in query_result: run_id = self.get_run_id_of_id(id_of_runid) @@ -385,41 +383,56 @@ def read_blob(self, table_name: str, condition: QueryCondition = None, meta_only return [] if condition and isinstance(condition.run_id, str): condition.run_id = self.get_id_of_run_id(condition.run_id) # convert run id - condition = condition.dumps() if condition else "" - sql_query = f"SELECT {', '.join((ID_COLUMN_NAME,TIMESTAMP_COLUMN_NAME, METADATA_COLUMN_NAME, *((BLOB_COLUMN_NAME,) if not meta_only else ())))} FROM {table_name} {condition}" - result, _ = self._query(sql_query, fetch=DbQueryFetchType.ALL) + cond_str, cond_vars = condition.dumpt() if condition else "" + sql_query = f"SELECT {', '.join((ID_COLUMN_NAME,TIMESTAMP_COLUMN_NAME, METADATA_COLUMN_NAME, *((BLOB_COLUMN_NAME,) if not meta_only else ())))} FROM {table_name} {cond_str}" + result, _ = self._query(sql_query, *cond_vars, fetch=DbQueryFetchType.ALL) return result + @classmethod + def load_db_of_path(cls, path): + if not os.path.isfile(path): + raise RuntimeError(f"{path} is not a file") + conn = ProjectDB(path=path) + return conn -# === DEFAULT EXPORT FUNCs === + @classmethod + def get_db_list(cls): + history_file_loader = ResourceLoader( + folder=DB_PROJECT_FILE_ROOT, file_types=[DB_PROJECT_FILE_TYPE_NAME], force_rescan=True + ) + history_file_list = history_file_loader.get_file_list() + for path in history_file_list: + cls.load_db_of_path(path=path) + return ProjectDB._id2dbc.items() + + @classmethod + def get_db_of_id(cls, project_id, rescan: bool = True): + if rescan: + cls.get_db_list() # scan for possible file changes + conn = ProjectDB.of_project_id(project_id=project_id) + return conn -if not os.path.exists(HISTORY_FILE_ROOT): - # create history root dir - os.mkdir(HISTORY_FILE_ROOT) -# check if is dir -if not os.path.isdir(HISTORY_FILE_ROOT): - raise RuntimeError(f"{HISTORY_FILE_ROOT} is not a directory.") +# === SCAN FOR DB FILES === -def load_db_of_path(path): - if not os.path.isfile(path): - raise RuntimeError(f"{path} is not a file") - conn = DBConnection(path=path) - return conn +if not os.path.exists(DB_PROJECT_FILE_ROOT): + # create history root dir + os.makedirs(DB_PROJECT_FILE_ROOT) +# check if is dir +if not os.path.isdir(DB_PROJECT_FILE_ROOT): + raise RuntimeError(f"{DB_PROJECT_FILE_ROOT} is not a directory.") -def get_db_list(): - history_file_loader = ResourceLoader( - folder=HISTORY_FILE_ROOT, file_types=[HISTORY_FILE_TYPE_NAME], force_rescan=True - ) - history_file_list = history_file_loader.get_file_list() - for path in history_file_list: - load_db_of_path(path=path) - return DBConnection._id2dbc.items() +def clear_dbc_on_exit(): + for project_id, dbc in ProjectDB._id2dbc.items(): + logger.info(f"process exiting, cleaning up...") + logger.info(f"trying to close db connection for project id {project_id}") + try: + dbc.connection.close() + except Exception as e: + logger.err(RuntimeError(f"failed to close db connection for project id {project_id}, {e}")) + else: + logger.ok(f"db connection for project id {project_id} closed") -def get_db_of_id(project_id, rescan: bool = True): - if rescan: - get_db_list() # scan for possible file changes - conn = DBConnection.of_project_id(project_id=project_id) - return conn +atexit.register(clear_dbc_on_exit) diff --git a/pyproject.toml b/pyproject.toml index c73d500d..68c51ace 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "neetbox" -version = "0.4.2" +version = "0.4.3" description = "Logging/Debugging/Tracing/Managing/Facilitating long running python projects, especially a replacement of tensorboard for deep learning projects" license = "MIT" authors = ["VisualDust ", "Lideming "] diff --git a/tests/client/neetbox.toml b/tests/client/neetbox.toml index 55643a57..e7a31008 100644 --- a/tests/client/neetbox.toml +++ b/tests/client/neetbox.toml @@ -1,5 +1,5 @@ name = "client" -version = "0.4.2" +version = "0.4.3" projectId = "0aedddf6-c3b1-4e07-8dff-10ba964e4960" [logging]