From 0417258cda0f4ddaa38126e26de059e1a77b6d3a Mon Sep 17 00:00:00 2001 From: Hongbo Miao <3375461+hongbo-miao@users.noreply.github.com> Date: Tue, 31 Dec 2024 19:14:09 -0800 Subject: [PATCH] refactor(python): add timestamp for logger --- aerospace/hm-aerosandbox/src/main.py | 10 +++++-- aerospace/hm-openaerostruct/src/main.py | 6 +++- api-rust/scripts/download_model.py | 18 ++++++----- .../hm-ray/applications/calculate/src/main.py | 12 +++++--- .../process-flight-data/src/main.py | 12 +++++--- ...o_delta_table_adsb_2x_flight_trace_data.py | 7 +++-- ...write_parquet_to_delta_table_motor_data.py | 7 +++-- .../src/utils/get_test_data_loader.py | 4 ++- .../src/utils/get_train_data_loader.py | 4 ++- .../pytorch-mnist/src/utils/save_model.py | 4 ++- .../pytorch-mnist/src/utils/test.py | 4 ++- .../pytorch-mnist/src/utils/train.py | 14 +++++---- computer-vision/hm-imagebind/src/main.py | 7 +++-- computer-vision/open-clip/src/main.py | 21 +++++++------ data-analytics/hm-cudf/src/main.py | 24 ++++++++------- data-analytics/hm-cupy/src/main.py | 18 ++++++----- data-analytics/hm-geopandas/src/main.py | 8 +++-- data-analytics/hm-numba/src/main.py | 24 ++++++++------- data-analytics/hm-pandas/src/main.py | 24 ++++++++------- data-analytics/hm-polars/src/main.py | 24 ++++++++------- .../src/main.py | 8 +++-- .../hm-duckdb/query-duckdb/src/main.py | 10 +++++-- .../hm-duckdb/query-lance/src/main.py | 8 +++-- .../hm-duckdb/query-parquet/src/main.py | 8 +++-- .../hm-duckdb/query-protobuf/src/main.py | 8 +++-- data-storage/hm-lancedb/src/main.py | 12 +++++--- data-storage/hm-protobuf/src/main.py | 8 +++-- data-storage/lance/src/main.py | 16 ++++++---- .../grafana/hm-dashboard/upload_dashboard.py | 10 +++++-- .../iads-config-reader/src/main.py | 18 ++++++----- .../iads-data-reader/src/main.py | 30 +++++++++++-------- embedded/format-can-data/src/main.py | 12 +++++--- embedded/hm-serial/src/main.py | 10 +++++-- embedded/pack-unpack-data/src/main.py | 12 +++++--- .../hm-pyvisa/src/main.py | 12 +++++--- high-performance-computing/hm-jax/src/main.py | 8 +++-- hm-xxhash/src/main.py | 8 +++-- machine-learning/dali/src/main.py | 20 ++++++++----- machine-learning/hm-autogluon/src/main.py | 14 +++++---- machine-learning/hm-docling/src/main.py | 10 +++++-- .../applications/chat-pdf/src/main.py | 8 +++-- .../applications/chat-pdf/src/main.py | 8 +++-- .../experiments/predict-diabetes/src/main.py | 8 +++-- .../hm-supervision/detect-objects/src/main.py | 6 +++- .../analyze-sentiment/src/main.py | 10 +++++-- machine-learning/mineru/src/main.py | 12 +++++--- .../deploy/src/deploy.py | 12 +++++--- .../infer/src/main.py | 10 +++++-- .../src/main.py | 8 +++-- quantum-computing/src/main.py | 14 +++++---- 50 files changed, 388 insertions(+), 202 deletions(-) diff --git a/aerospace/hm-aerosandbox/src/main.py b/aerospace/hm-aerosandbox/src/main.py index fdcd6c14da..6f95253ca2 100644 --- a/aerospace/hm-aerosandbox/src/main.py +++ b/aerospace/hm-aerosandbox/src/main.py @@ -7,6 +7,8 @@ import aerosandbox.tools.pretty_plots as p import matplotlib.pyplot as plt +logger = logging.getLogger(__name__) + def main() -> None: wing_airfoil = asb.Airfoil("sd7037") @@ -110,7 +112,7 @@ def main() -> None: ) aero = vlm.run() for k, v in aero.items(): - logging.info(f"{k.rjust(4)} : {v}") + logger.info(f"{k.rjust(4)} : {v}") vlm.draw(show_kwargs=dict(jupyter_backend="static")) # Operating Point Optimization @@ -126,7 +128,7 @@ def main() -> None: opti.minimize(-L_over_D) sol = opti.solve() best_alpha = sol(alpha) - logging.info(f"Alpha for max L/D: {best_alpha:.3f} deg") + logger.info(f"Alpha for max L/D: {best_alpha:.3f} deg") # Aerodynamic Shape Optimization opti = asb.Opti() @@ -183,5 +185,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/aerospace/hm-openaerostruct/src/main.py b/aerospace/hm-openaerostruct/src/main.py index 56b31b7c85..6d340e78a7 100644 --- a/aerospace/hm-openaerostruct/src/main.py +++ b/aerospace/hm-openaerostruct/src/main.py @@ -7,6 +7,8 @@ from openaerostruct.geometry.utils import generate_mesh from openmdao.api import n2 +logger = logging.getLogger(__name__) + def main() -> None: # Create a dictionary to store options about the mesh @@ -102,5 +104,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/api-rust/scripts/download_model.py b/api-rust/scripts/download_model.py index bc90b2bffc..15184e5303 100644 --- a/api-rust/scripts/download_model.py +++ b/api-rust/scripts/download_model.py @@ -5,6 +5,8 @@ import torch import torchvision.models as models +logger = logging.getLogger(__name__) + def download_labels(): labels_url = ( @@ -13,30 +15,32 @@ def download_labels(): labels_path = Path("models/labels.txt") if not labels_path.exists(): - logging.info("Downloading labels...") + logger.info("Downloading labels...") with httpx.Client() as client: response = client.get(labels_url) labels_path.write_bytes(response.content) - logging.info("Labels downloaded successfully") + logger.info("Labels downloaded successfully") else: - logging.info("Labels file already exists") + logger.info("Labels file already exists") def download_resnet18(): model_path = Path("models/resnet18.ot") if not model_path.exists(): - logging.info("Downloading ResNet18...") + logger.info("Downloading ResNet18...") model = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1) model.eval() traced_model = torch.jit.trace(model, torch.randn(1, 3, 224, 224)) traced_model.save(model_path) - logging.info("Model downloaded and saved successfully") + logger.info("Model downloaded and saved successfully") else: - logging.info("Model file already exists") + logger.info("Model file already exists") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) download_resnet18() download_labels() diff --git a/cloud-computing/hm-ray/applications/calculate/src/main.py b/cloud-computing/hm-ray/applications/calculate/src/main.py index ecc9f7ea19..9dd1438944 100644 --- a/cloud-computing/hm-ray/applications/calculate/src/main.py +++ b/cloud-computing/hm-ray/applications/calculate/src/main.py @@ -3,6 +3,8 @@ import ray +logger = logging.getLogger(__name__) + @ray.remote def square(n: int) -> int: @@ -16,17 +18,19 @@ def sum_list(numbers: list[int]) -> int: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) ray.init() - logging.info(ray.cluster_resources()) + logger.info(ray.cluster_resources()) numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] squared_tasks = [square.remote(n) for n in numbers] squared_results: list[int] = ray.get(squared_tasks) - logging.info(f"{squared_results = }") + logger.info(f"{squared_results = }") sum_task = sum_list.remote(squared_results) total_sum = ray.get(sum_task) - logging.info(f"{total_sum = }") + logger.info(f"{total_sum = }") ray.shutdown() diff --git a/cloud-computing/hm-ray/applications/process-flight-data/src/main.py b/cloud-computing/hm-ray/applications/process-flight-data/src/main.py index 389d75da42..a6d7e93858 100644 --- a/cloud-computing/hm-ray/applications/process-flight-data/src/main.py +++ b/cloud-computing/hm-ray/applications/process-flight-data/src/main.py @@ -7,6 +7,8 @@ import pandas as pd import ray +logger = logging.getLogger(__name__) + @ray.remote def process_flight_data( @@ -36,10 +38,12 @@ def process_flight_data( if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) ray.init() - logging.info(ray.cluster_resources()) + logger.info(ray.cluster_resources()) mlflow_tracking_server_host = config.MLFLOW_TRACKING_SERVER_HOST mlflow_tracking_server_user_name = config.MLFLOW_TRACKING_USERNAME @@ -88,7 +92,7 @@ def process_flight_data( results = ray.get(tasks) for i, df_head in enumerate(results): - logging.info(f"Experiment {i}") - logging.info(f"{df_head = }") + logger.info(f"Experiment {i}") + logger.info(f"{df_head = }") ray.shutdown() diff --git a/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_adsb_2x_flight_trace_data.py b/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_adsb_2x_flight_trace_data.py index af5b0d29f3..942aa79946 100644 --- a/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_adsb_2x_flight_trace_data.py +++ b/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_adsb_2x_flight_trace_data.py @@ -9,7 +9,10 @@ from pyspark.sql import DataFrame from pyspark.sql.functions import col, concat, date_format, from_unixtime, lit, when -logging.basicConfig(level=logging.INFO) +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) raw_parquet_paths = [ "s3://hm-production-bucket/data/raw-parquet/adsb_2x_flight_trace_data/" @@ -142,7 +145,7 @@ def add_coordinate_column( df = s3_node.toDF() if df.isEmpty(): - logging.info("DataFrame is empty.") + logger.info("DataFrame is empty.") job.commit() os._exit(os.EX_OK) diff --git a/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_motor_data.py b/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_motor_data.py index 76ec49c180..474254b5ee 100644 --- a/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_motor_data.py +++ b/cloud-infrastructure/terraform/environments/production/aws/general/files/aws-glue/spark-scripts/src/hm_write_parquet_to_delta_table_motor_data.py @@ -7,7 +7,10 @@ from awsglue.utils import getResolvedOptions from pyspark.context import SparkContext -logging.basicConfig(level=logging.INFO) +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger(__name__) raw_parquet_paths = ["s3://hm-production-bucket/data/parquet/motor/"] delta_table_path = "s3://hm-production-bucket/data/delta-tables/motor_data/" @@ -34,7 +37,7 @@ df = s3_node.toDF() if df.isEmpty(): - logging.info("DataFrame is empty.") + logger.info("DataFrame is empty.") job.commit() os._exit(os.EX_OK) diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_test_data_loader.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_test_data_loader.py index 1d155c6944..6905b87066 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_test_data_loader.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_test_data_loader.py @@ -5,9 +5,11 @@ import torch.utils.data.distributed from torchvision import datasets, transforms +logger = logging.getLogger(__name__) + def get_test_data_loader(test_batch_size, training_dir, **kwargs): - logging.info("Get test data loader") + logger.info("Get test data loader") return torch.utils.data.DataLoader( datasets.MNIST( training_dir, diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_train_data_loader.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_train_data_loader.py index 8468516d9c..46b4d3dbd4 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_train_data_loader.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/get_train_data_loader.py @@ -5,9 +5,11 @@ import torch.utils.data.distributed from torchvision import datasets, transforms +logger = logging.getLogger(__name__) + def get_train_data_loader(batch_size, training_dir, is_distributed, **kwargs): - logging.info("Get train data loader") + logger.info("Get train data loader") dataset = datasets.MNIST( training_dir, train=True, diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/save_model.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/save_model.py index 6fee9bc472..a01a810757 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/save_model.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/save_model.py @@ -5,8 +5,10 @@ import torch.utils.data import torch.utils.data.distributed +logger = logging.getLogger(__name__) + def save_model(model, model_dir): - logging.info("Save the model.") + logger.info("Save the model.") path = os.path.join(model_dir, "model.pth") torch.save(model.cpu().state_dict(), path) diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py index c9483ec7ab..1aa4b874c0 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/test.py @@ -5,6 +5,8 @@ import torch.utils.data import torch.utils.data.distributed +logger = logging.getLogger(__name__) + def test(model, test_loader, device): model.eval() @@ -21,7 +23,7 @@ def test(model, test_loader, device): correct += pred.eq(target.view_as(pred)).sum().item() test_loss /= len(test_loader.dataset) - logging.info( + logger.info( "Test set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( test_loss, correct, diff --git a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py index 0538c53d06..0f5c4bb7bc 100644 --- a/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py +++ b/cloud-platform/aws/amazon-sagemaker/pytorch-mnist/src/utils/train.py @@ -14,13 +14,15 @@ from utils.save_model import save_model from utils.test import test +logger = logging.getLogger(__name__) + def train(args): is_distributed = len(args.hosts) > 1 and args.backend is not None - logging.info("Distributed training:", is_distributed) + logger.info("Distributed training:", is_distributed) use_cuda = args.num_gpus > 0 - logging.info("Number of gpus available:", args.num_gpus) + logger.info("Number of gpus available:", args.num_gpus) kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {} device = torch.device("cuda" if use_cuda else "cpu") @@ -33,7 +35,7 @@ def train(args): dist.init_process_group( backend=args.backend, rank=host_rank, world_size=world_size ) - logging.info( + logger.info( "Initialized the distributed environment: '{}' backend on {} nodes. ".format( args.backend, dist.get_world_size() ) @@ -52,7 +54,7 @@ def train(args): ) test_loader = get_test_data_loader(args.test_batch_size, args.data_dir, **kwargs) - logging.info( + logger.info( "Processes {}/{} ({:.0f}%) of train data".format( len(train_loader.sampler), len(train_loader.dataset), @@ -60,7 +62,7 @@ def train(args): ) ) - logging.info( + logger.info( "Processes {}/{} ({:.0f}%) of test data".format( len(test_loader.sampler), len(test_loader.dataset), @@ -91,7 +93,7 @@ def train(args): average_gradients(model) optimizer.step() if batch_idx % args.log_interval == 0: - logging.info( + logger.info( "Train Epoch: {} [{}/{} ({:.0f}%)] Loss: {:.6f}".format( epoch, batch_idx * len(data), diff --git a/computer-vision/hm-imagebind/src/main.py b/computer-vision/hm-imagebind/src/main.py index 455a2e866b..cc4a31093c 100644 --- a/computer-vision/hm-imagebind/src/main.py +++ b/computer-vision/hm-imagebind/src/main.py @@ -10,6 +10,7 @@ from lancedb.pydantic import LanceModel, Vector from lancedb.table import Table +logger = logging.getLogger(__name__) EMBEDDINGS = get_registry().get("imagebind").create() DATA_DIR = Path("data") @@ -54,7 +55,7 @@ def download_file(client: httpx.Client, url: str, is_audio: bool = True) -> Path if response.status_code == 200: with open(local_file_path, "wb") as file: file.write(response.content) - logging.info(f"Downloaded file: {local_file_path}") + logger.info(f"Downloaded file: {local_file_path}") return local_file_path else: raise RuntimeError(f"Download failed: {response}") @@ -164,5 +165,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/computer-vision/open-clip/src/main.py b/computer-vision/open-clip/src/main.py index 954bf873c9..cb52c6d491 100644 --- a/computer-vision/open-clip/src/main.py +++ b/computer-vision/open-clip/src/main.py @@ -8,7 +8,7 @@ from lancedb.pydantic import LanceModel, Vector from PIL import Image -logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) EMBEDDINGS = ( get_registry() @@ -29,34 +29,37 @@ def main(): # Create or get the table if "pets" in db: - logging.info("Using existing table") + logger.info("Using existing table") table = db["pets"] else: - logging.info("Creating new table") + logger.info("Creating new table") table = db.create_table("pets", schema=Pets, mode="overwrite") # Use a sampling of images from the specified directory image_dir = Path("data/images") - logging.info(f"Loading images from directory: {image_dir}") + logger.info(f"Loading images from directory: {image_dir}") uris = [str(f) for f in image_dir.glob("*.jpg")] uris = sample(uris, 1000) - logging.info(f"Processing {len(uris)} images") + logger.info(f"Processing {len(uris)} images") table.add(pd.DataFrame({"image_uri": uris})) # Query using text query_text = "black cat" - logging.info(f"Performing text search with query: '{query_text}'") + logger.info(f"Performing text search with query: '{query_text}'") search_results = table.search(query_text).limit(3).to_pydantic(Pets) for idx, result in enumerate(search_results): - logging.info(f"Text search result {idx + 1}: {result.image_uri}") + logger.info(f"Text search result {idx + 1}: {result.image_uri}") # Query using an image query_image_path = Path("data/images/samoyed_27.jpg").expanduser() - logging.info(f"Performing image search with query image: {query_image_path}") + logger.info(f"Performing image search with query image: {query_image_path}") query_image = Image.open(query_image_path) search_results = table.search(query_image).limit(3).to_pydantic(Pets) for idx, result in enumerate(search_results): - logging.info(f"Image search result {idx + 1}: {result.image_uri}") + logger.info(f"Image search result {idx + 1}: {result.image_uri}") if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-analytics/hm-cudf/src/main.py b/data-analytics/hm-cudf/src/main.py index b55cb6bad0..cb4fc44229 100644 --- a/data-analytics/hm-cudf/src/main.py +++ b/data-analytics/hm-cudf/src/main.py @@ -6,6 +6,8 @@ import cudf import httpx +logger = logging.getLogger(__name__) + def download_data(transaction_path: Path) -> None: transaction_url = ( @@ -29,18 +31,18 @@ def main() -> None: start_time = time.time() transactions = cudf.read_parquet(transaction_path) load_time = time.time() - start_time - logging.info(f"Data loading time: {load_time:.2f} seconds") + logger.info(f"Data loading time: {load_time:.2f} seconds") # Print DataFrame info - logging.info(transactions.columns) - logging.info(transactions.head()) + logger.info(transactions.columns) + logger.info(transactions.head()) # Time the sum calculation start_time = time.time() total_amount = transactions["AMOUNT"].sum() sum_time = time.time() - start_time - logging.info(f"Total transaction amount: {total_amount}") - logging.info(f"Sum calculation time: {sum_time:.2f} seconds") + logger.info(f"Total transaction amount: {total_amount}") + logger.info(f"Sum calculation time: {sum_time:.2f} seconds") # Time the group by operations start_time = time.time() @@ -56,16 +58,18 @@ def main() -> None: ) ) group_by_time = time.time() - start_time - logging.info("Category statistics:") - logging.info(category_stats) - logging.info(f"Groupby operations time: {group_by_time:.2f} seconds") + logger.info("Category statistics:") + logger.info(category_stats) + logger.info(f"Groupby operations time: {group_by_time:.2f} seconds") # Print total execution time - logging.info( + logger.info( f"Total execution time: {load_time + sum_time + group_by_time:.2f} seconds" ) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-analytics/hm-cupy/src/main.py b/data-analytics/hm-cupy/src/main.py index 46f13d8995..8589843084 100644 --- a/data-analytics/hm-cupy/src/main.py +++ b/data-analytics/hm-cupy/src/main.py @@ -4,6 +4,8 @@ import cupy as cp import numpy as np +logger = logging.getLogger(__name__) + def sum_of_squares_numpy(n: int) -> float: array = np.arange(n) @@ -22,22 +24,24 @@ def main() -> None: start_time = time.time() result_numpy = sum_of_squares_numpy(n) numpy_time = time.time() - start_time - logging.info(f"NumPy result: {result_numpy}") - logging.info(f"NumPy time: {numpy_time} seconds") + logger.info(f"NumPy result: {result_numpy}") + logger.info(f"NumPy time: {numpy_time} seconds") try: # CuPy version start_time = time.time() result_cupy = sum_of_squares_cupy(n) cupy_time = time.time() - start_time - logging.info(f"CuPy result: {result_cupy}") - logging.info(f"CuPy time: {cupy_time} seconds") - logging.info(f"CuPy speedup vs NumPy: {numpy_time / cupy_time}x") + logger.info(f"CuPy result: {result_cupy}") + logger.info(f"CuPy time: {cupy_time} seconds") + logger.info(f"CuPy speedup vs NumPy: {numpy_time / cupy_time}x") except cp.cuda.runtime.CUDARuntimeError: - logging.warning("CUDA GPU is not available") + logger.warning("CUDA GPU is not available") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-analytics/hm-geopandas/src/main.py b/data-analytics/hm-geopandas/src/main.py index 34a06c6b71..bfd8c1999f 100644 --- a/data-analytics/hm-geopandas/src/main.py +++ b/data-analytics/hm-geopandas/src/main.py @@ -5,10 +5,12 @@ import geopandas from matplotlib import pyplot as plt +logger = logging.getLogger(__name__) + def main() -> None: gdf = geopandas.read_file(geodatasets.get_path("geoda.chicago_commpop")) - logging.info(gdf.head()) + logger.info(gdf.head()) gdf.to_parquet(Path("data/chicago_commpop.parquet")) gdf.plot( column="POP2010", @@ -26,5 +28,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-analytics/hm-numba/src/main.py b/data-analytics/hm-numba/src/main.py index c3e3031d30..ebd5832091 100644 --- a/data-analytics/hm-numba/src/main.py +++ b/data-analytics/hm-numba/src/main.py @@ -5,6 +5,8 @@ import numpy as np from numba import cuda +logger = logging.getLogger(__name__) + def sum_of_squares_python(n: int) -> float: total = 0.0 @@ -51,29 +53,31 @@ def main() -> None: start_time = time.time() python_result = sum_of_squares_python(n) python_time = time.time() - start_time - logging.info(f"Python result: {python_result}") - logging.info(f"Python time: {python_time} seconds") + logger.info(f"Python result: {python_result}") + logger.info(f"Python time: {python_time} seconds") # CPU Numba version start_time = time.time() cpu_numba_result = sum_of_squares_cpu_numba(n) cpu_numba_time = time.time() - start_time - logging.info(f"CPU Numba result: {cpu_numba_result}") - logging.info(f"CPU Numba time: {cpu_numba_time} seconds") - logging.info(f"CPU Numba speedup vs Python: {python_time / cpu_numba_time}x") + logger.info(f"CPU Numba result: {cpu_numba_result}") + logger.info(f"CPU Numba time: {cpu_numba_time} seconds") + logger.info(f"CPU Numba speedup vs Python: {python_time / cpu_numba_time}x") # GPU Numba version try: start_time = time.time() gpu_numba_result = sum_of_squares_gpu_numba(n) gpu_numba_time = time.time() - start_time - logging.info(f"GPU Numba result: {gpu_numba_result}") - logging.info(f"GPU Numba time: {gpu_numba_time} seconds") - logging.info(f"GPU Numba speedup vs Python: {python_time / gpu_numba_time}x") + logger.info(f"GPU Numba result: {gpu_numba_result}") + logger.info(f"GPU Numba time: {gpu_numba_time} seconds") + logger.info(f"GPU Numba speedup vs Python: {python_time / gpu_numba_time}x") except cuda.CudaSupportError: - logging.warning("CUDA GPU is not available") + logger.warning("CUDA GPU is not available") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-analytics/hm-pandas/src/main.py b/data-analytics/hm-pandas/src/main.py index d4c939c063..b977a7fbbd 100644 --- a/data-analytics/hm-pandas/src/main.py +++ b/data-analytics/hm-pandas/src/main.py @@ -6,6 +6,8 @@ import httpx import pandas as pd +logger = logging.getLogger(__name__) + def download_data(transaction_path: Path) -> None: transaction_url = ( @@ -29,18 +31,18 @@ def main() -> None: start_time = time.time() transactions = pd.read_parquet(transaction_path) load_time = time.time() - start_time - logging.info(f"Data loading time: {load_time:.2f} seconds") + logger.info(f"Data loading time: {load_time:.2f} seconds") # Print DataFrame info - logging.info(transactions.columns) - logging.info(transactions.head()) + logger.info(transactions.columns) + logger.info(transactions.head()) # Time the sum calculation start_time = time.time() total_amount = transactions["AMOUNT"].sum() sum_time = time.time() - start_time - logging.info(f"Total transaction amount: {total_amount}") - logging.info(f"Sum calculation time: {sum_time:.2f} seconds") + logger.info(f"Total transaction amount: {total_amount}") + logger.info(f"Sum calculation time: {sum_time:.2f} seconds") # Time the group by operations start_time = time.time() @@ -56,16 +58,18 @@ def main() -> None: ) ) group_by_time = time.time() - start_time - logging.info("Category statistics:") - logging.info(category_stats) - logging.info(f"Group by operations time: {group_by_time:.2f} seconds") + logger.info("Category statistics:") + logger.info(category_stats) + logger.info(f"Group by operations time: {group_by_time:.2f} seconds") # Print total execution time - logging.info( + logger.info( f"Total execution time: {load_time + sum_time + group_by_time:.2f} seconds" ) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-analytics/hm-polars/src/main.py b/data-analytics/hm-polars/src/main.py index becdc83ecb..f939bcc015 100644 --- a/data-analytics/hm-polars/src/main.py +++ b/data-analytics/hm-polars/src/main.py @@ -6,6 +6,8 @@ import httpx import polars as pl +logger = logging.getLogger(__name__) + def download_data(transaction_path: Path) -> None: transaction_url = ( @@ -34,18 +36,18 @@ def main() -> None: start_time = time.time() transactions = pl.scan_parquet(transaction_path) load_time = time.time() - start_time - logging.info(f"Data loading time: {load_time:.2f} seconds") + logger.info(f"Data loading time: {load_time:.2f} seconds") # Print DataFrame info - logging.info(transactions.collect_schema().names()) - logging.info(transactions.head()) + logger.info(transactions.collect_schema().names()) + logger.info(transactions.head()) # Time the sum calculation start_time = time.time() total_amount = transactions.select(pl.col("AMOUNT").sum()).collect(engine="gpu") sum_time = time.time() - start_time - logging.info(f"Total transaction amount: {total_amount}") - logging.info(f"Sum calculation time: {sum_time:.2f} seconds") + logger.info(f"Total transaction amount: {total_amount}") + logger.info(f"Sum calculation time: {sum_time:.2f} seconds") # Time the group by operations start_time = time.time() @@ -61,16 +63,18 @@ def main() -> None: .collect(engine=gpu_engine) ) group_by_time = time.time() - start_time - logging.info("Category statistics:") - logging.info(category_stats) - logging.info(f"Groupby operations time: {group_by_time:.2f} seconds") + logger.info("Category statistics:") + logger.info(category_stats) + logger.info(f"Groupby operations time: {group_by_time:.2f} seconds") # Print total execution time - logging.info( + logger.info( f"Total execution time: {load_time + sum_time + group_by_time:.2f} seconds" ) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-storage/delta-lake/read-delta-lake-by-amazon-athena/src/main.py b/data-storage/delta-lake/read-delta-lake-by-amazon-athena/src/main.py index 7e25fd9b1b..4414469585 100644 --- a/data-storage/delta-lake/read-delta-lake-by-amazon-athena/src/main.py +++ b/data-storage/delta-lake/read-delta-lake-by-amazon-athena/src/main.py @@ -2,15 +2,19 @@ import awswrangler as wr +logger = logging.getLogger(__name__) + def main(): df = wr.athena.read_sql_query( "select * from motor limit 10;", database="production_hm_iot_db", ) - logging.info(df) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-storage/hm-duckdb/query-duckdb/src/main.py b/data-storage/hm-duckdb/query-duckdb/src/main.py index 1a29667eaa..790aae3ff7 100644 --- a/data-storage/hm-duckdb/query-duckdb/src/main.py +++ b/data-storage/hm-duckdb/query-duckdb/src/main.py @@ -4,6 +4,8 @@ import duckdb import polars as pl +logger = logging.getLogger(__name__) + def main(duckdb_path: Path) -> None: people = pl.DataFrame( @@ -13,18 +15,20 @@ def main(duckdb_path: Path) -> None: "city": ["New York", "London", "Paris", "Tokyo", "Berlin"], } ) - logging.info(people) + logger.info(people) with duckdb.connect(duckdb_path) as conn: conn.register("people", people) conn.execute("create table if not exists people as select * from people") age = 25 df = conn.execute("select name, age from people where age > ?", [age]).pl() - logging.info(df) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) duckdb_path = Path("data/people.duckdb") main(duckdb_path) diff --git a/data-storage/hm-duckdb/query-lance/src/main.py b/data-storage/hm-duckdb/query-lance/src/main.py index fb34d4c0e5..4547fd35ad 100644 --- a/data-storage/hm-duckdb/query-lance/src/main.py +++ b/data-storage/hm-duckdb/query-lance/src/main.py @@ -4,6 +4,8 @@ import duckdb import lancedb +logger = logging.getLogger(__name__) + def main(database_path: Path) -> None: sample_data = [ @@ -34,11 +36,13 @@ def main(database_path: Path) -> None: order by average_review_score desc """ df = conn.execute(query).pl() - logging.info(df) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) database_path = Path("/tmp/lancedb/products") main(database_path) diff --git a/data-storage/hm-duckdb/query-parquet/src/main.py b/data-storage/hm-duckdb/query-parquet/src/main.py index 039a30b5e3..797233fa4d 100644 --- a/data-storage/hm-duckdb/query-parquet/src/main.py +++ b/data-storage/hm-duckdb/query-parquet/src/main.py @@ -3,6 +3,8 @@ import duckdb +logger = logging.getLogger(__name__) + def main(parquet_path: Path) -> None: with duckdb.connect() as conn: @@ -11,11 +13,13 @@ def main(parquet_path: Path) -> None: from read_parquet('{parquet_path}') """ df = conn.execute(query).pl() - logging.info(df) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) parquet_path = Path("data/file.parquet") main(parquet_path) diff --git a/data-storage/hm-duckdb/query-protobuf/src/main.py b/data-storage/hm-duckdb/query-protobuf/src/main.py index 10ff5f1c91..27c5170800 100644 --- a/data-storage/hm-duckdb/query-protobuf/src/main.py +++ b/data-storage/hm-duckdb/query-protobuf/src/main.py @@ -2,6 +2,8 @@ import duckdb +logger = logging.getLogger(__name__) + def main() -> None: with duckdb.connect(config={"allow_unsigned_extensions": True}) as conn: @@ -17,9 +19,11 @@ def main() -> None: order by timestamp_ns """ df = conn.execute(query).pl() - logging.info(df) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-storage/hm-lancedb/src/main.py b/data-storage/hm-lancedb/src/main.py index 137a70fc12..9f0ac911ee 100644 --- a/data-storage/hm-lancedb/src/main.py +++ b/data-storage/hm-lancedb/src/main.py @@ -5,6 +5,8 @@ from lancedb.embeddings import get_registry from lancedb.pydantic import LanceModel, Vector +logger = logging.getLogger(__name__) + EMBEDDINGS = get_registry().get("sentence-transformers").create(name="all-MiniLM-L6-v2") @@ -34,11 +36,13 @@ def main(): query = "What is the meaning of life?" df = perform_semantic_search(table, query) - logging.info("Question: %s", query) - logging.info("Answer: %s", df["line"][0]) - logging.info(df) + logger.info("Question: %s", query) + logger.info("Answer: %s", df["line"][0]) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-storage/hm-protobuf/src/main.py b/data-storage/hm-protobuf/src/main.py index f2036b4923..5fad4a4f9f 100644 --- a/data-storage/hm-protobuf/src/main.py +++ b/data-storage/hm-protobuf/src/main.py @@ -8,6 +8,8 @@ import polars as pl from protos.production.iot import motor_pb2 +logger = logging.getLogger(__name__) + class ProtobufWriter: def __init__(self, filename: Path) -> None: @@ -94,9 +96,11 @@ def main() -> None: reader: ProtobufReader = ProtobufReader(motor_data_path) df: pl.DataFrame = reader.get_dataframe() reader.close() - logging.info(df) + logger.info(df) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-storage/lance/src/main.py b/data-storage/lance/src/main.py index d39e3bce56..025549ad7e 100644 --- a/data-storage/lance/src/main.py +++ b/data-storage/lance/src/main.py @@ -4,6 +4,8 @@ import numpy as np from lance.vector import vec_to_table +logger = logging.getLogger(__name__) + def main() -> None: # Create sample vectors (minimum 5000 recommended for meaningful indexing) @@ -23,7 +25,7 @@ def main() -> None: # Save to Lance dataset uri = "/tmp/lancedb/vectors.lance" dataset = lance.write_dataset(vector_table, uri, mode="overwrite") - logging.info( + logger.info( "Dataset saved to %s with %d vectors of dimension %d", uri, num_vectors, @@ -50,14 +52,14 @@ def main() -> None: # - Trade-off: more sub-vectors = better compression but potentially lower accuracy num_sub_vectors=16, ) - logging.info("Created vector similarity index") + logger.info("Created vector similarity index") # Read back the dataset dataset = lance.dataset(uri) # Perform vector similarity search query_vector = vectors[1] - logging.info( + logger.info( "Performing similarity search for vector with pattern [1.0]*32 + [2.0]*32 + [3.0]*32 + [4.0]*32" ) @@ -83,14 +85,16 @@ def main() -> None: }, ).to_pandas() - logging.info("Nearest neighbors (distances show similarity, lower = more similar):") + logger.info("Nearest neighbors (distances show similarity, lower = more similar):") for idx, row in results.iterrows(): vector_preview = np.array(row["vector"]) - logging.info( + logger.info( f"Result {idx + 1}/{k}: Distance: {row['_distance']:.4f}, Vector preview: {vector_preview[:8]}..." ) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-visualization/grafana/hm-dashboard/upload_dashboard.py b/data-visualization/grafana/hm-dashboard/upload_dashboard.py index d272e6ea66..1142a56b56 100644 --- a/data-visualization/grafana/hm-dashboard/upload_dashboard.py +++ b/data-visualization/grafana/hm-dashboard/upload_dashboard.py @@ -8,6 +8,8 @@ from grafanalib._gen import DashboardEncoder from grafanalib.core import Dashboard +logger = logging.getLogger(__name__) + def upload_to_grafana(dashboard: Dashboard) -> None: dashboard_json = json.dumps( @@ -27,12 +29,14 @@ def upload_to_grafana(dashboard: Dashboard) -> None: }, verify=True, ) - logging.info(res.status_code) - logging.info(res.content) + logger.info(res.status_code) + logger.info(res.content) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) module = importlib.machinery.SourceFileLoader( "hm_dashboard", pathlib.Path(__file__).parent.joinpath("hm.dashboard.py").resolve().as_posix(), diff --git a/data-visualization/iads/iads-data-manager/iads-config-reader/src/main.py b/data-visualization/iads/iads-data-manager/iads-config-reader/src/main.py index f34c2d80d7..d1499574e6 100644 --- a/data-visualization/iads/iads-data-manager/iads-config-reader/src/main.py +++ b/data-visualization/iads/iads-data-manager/iads-config-reader/src/main.py @@ -5,24 +5,26 @@ import pythoncom import win32com.client +logger = logging.getLogger(__name__) + def show_version_from_file(iads_config: Any, iads_config_path: Path) -> None: try: version = iads_config.VersionFromFile(iads_config_path) - logging.info(f"{version = }") + logger.info(f"{version = }") except Exception as e: - logging.error(f"{iads_config_path = }, {e = }") + logger.exception(f"{iads_config_path = }, {e = }") def execute_query(iads_config: Any, query: str) -> None: try: - logging.info(f"{query = }") + logger.info(f"{query = }") results: list[Any] | None = iads_config.Query(query) if results: for result in results: - logging.info(f"{result = }") + logger.info(f"{result = }") except Exception as e: - logging.error(f"{e = }") + logger.exception(f"{e = }") def process_config(iads_config_path: Path) -> None: @@ -40,7 +42,7 @@ def process_config(iads_config_path: Path) -> None: iads_config.Close(True) except Exception as e: - logging.error(f"{e = }") + logger.exception(f"{e = }") finally: # Clean up COM resources if iads_config: @@ -54,5 +56,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py b/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py index ff2e3fd850..e0acb92806 100644 --- a/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py +++ b/data-visualization/iads/iads-data-manager/iads-data-reader/src/main.py @@ -12,6 +12,8 @@ import pythoncom import win32com.client +logger = logging.getLogger(__name__) + class IadsUtil: IADS_CONFIG_FILE_NAME = "pfConfig" @@ -22,7 +24,7 @@ class IadsUtil: @staticmethod def get_iads_signals(iads_config: Any, query: str) -> set[str]: - logging.info(f"Executing: {query}") + logger.info(f"Executing: {query}") results = iads_config.Query(query) if not results: return set() @@ -31,7 +33,7 @@ def get_iads_signals(iads_config: Any, query: str) -> set[str]: signal_set = {s.rstrip("\x00") for s in results} if len(signal_set) < len(results): duplicated_signal_set = {s for s in signal_set if results.count(s) > 1} - logging.warning(f"Duplicate signals found: {duplicated_signal_set}") + logger.warning(f"Duplicate signals found: {duplicated_signal_set}") return signal_set @staticmethod @@ -98,7 +100,7 @@ def export_to_parquet( irig_end_time, str(parquet_file_path), ] - logging.info(f"Executing command: {' '.join(cmd)}") + logger.info(f"Executing command: {' '.join(cmd)}") result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: return parquet_file_path @@ -128,11 +130,11 @@ def get_iads_dataframe( try: with tempfile.TemporaryDirectory() as tmp_dir: tmp_dir_path = Path(tmp_dir) - logging.info(f"Created temporary directory: {tmp_dir_path}") + logger.info(f"Created temporary directory: {tmp_dir_path}") # Copy IADS config file temp_iads_config_path = IadsUtil.copy_iads_config(iads_config_path) - logging.info( + logger.info( f"Created a copy of IADS config file: {temp_iads_config_path}" ) @@ -147,11 +149,11 @@ def get_iads_dataframe( # Create signal group group_name = "AllSignals" - logging.info( + logger.info( f"Creating signal group with query. group_name = '{group_name}'" ) IadsUtil.create_iads_signal_group(iads_config, signal_set, group_name) - logging.info(f"Signal group '{group_name}' created successfully") + logger.info(f"Signal group '{group_name}' created successfully") # Get IRIG times and year irig_start_time, irig_end_time, year = IadsUtil.get_irig_times( @@ -167,7 +169,7 @@ def get_iads_dataframe( tmp_dir_path, iads_manager_exe_path, ) - logging.info("Export parquet completed successfully") + logger.info("Export parquet completed successfully") # Read the exported parquet file df = pd.read_parquet(parquet_file_path) @@ -180,13 +182,13 @@ def get_iads_dataframe( irig_time_ns, year, timezone ) ) - logging.info( + logger.info( f"Added {IadsUtil.UNIX_TIME_COLUMN_NAME} column with Unix time in nanoseconds ({year = })" ) return df except Exception as e: - logging.error(f"{e = }") + logger.exception(f"{e = }") return None finally: @@ -195,15 +197,17 @@ def get_iads_dataframe( if temp_iads_config_path is not None: temp_iads_config_path.unlink() - logging.info(f"Delete the temporary pfConfig: {temp_iads_config_path}") + logger.info(f"Delete the temporary pfConfig: {temp_iads_config_path}") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) iads_data_path = Path(r"C:\iads_data") iads_manager_exe_path = Path( r"C:\Program Files\IADS\DataManager\IadsDataManager.exe" ) timezone = "America/Los_Angeles" df = IadsUtil.get_iads_dataframe(iads_manager_exe_path, iads_data_path, timezone) - logging.info(f"{df = }") + logger.info(f"{df = }") diff --git a/embedded/format-can-data/src/main.py b/embedded/format-can-data/src/main.py index b51e6b4180..4308c0c88d 100644 --- a/embedded/format-can-data/src/main.py +++ b/embedded/format-can-data/src/main.py @@ -4,21 +4,25 @@ import can import cantools +logger = logging.getLogger(__name__) + def main() -> None: dbc = cantools.database.load_file(Path("src/dbc/engine.dbc")) - logging.info(dbc.messages) + logger.info(dbc.messages) eec1_message_definition = dbc.get_message_by_name("EEC1") - logging.info(eec1_message_definition.signals) + logger.info(eec1_message_definition.signals) data = eec1_message_definition.encode({"EngineSpeed": 200.1}) frame = can.Message(arbitration_id=eec1_message_definition.frame_id, data=data) - logging.info(frame) + logger.info(frame) can_bus = can.interface.Bus("vcan0", bustype="socketcan") can_bus.send(frame) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/embedded/hm-serial/src/main.py b/embedded/hm-serial/src/main.py index 2116efcb0d..9d419ca993 100644 --- a/embedded/hm-serial/src/main.py +++ b/embedded/hm-serial/src/main.py @@ -2,17 +2,21 @@ import serial.tools.list_ports +logger = logging.getLogger(__name__) + def main() -> None: available_ports = serial.tools.list_ports.comports() if not available_ports: - logging.info("No serial ports found.") + logger.info("No serial ports found.") else: for port in available_ports: - logging.info(port.device) + logger.info(port.device) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/embedded/pack-unpack-data/src/main.py b/embedded/pack-unpack-data/src/main.py index 934da641ab..622d9dca5e 100644 --- a/embedded/pack-unpack-data/src/main.py +++ b/embedded/pack-unpack-data/src/main.py @@ -1,22 +1,26 @@ import logging import struct +logger = logging.getLogger(__name__) + def main() -> None: # https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment # https://docs.python.org/3/library/struct.html#format-characters format_string = " None: resource_manager = pyvisa.ResourceManager() - logging.info(resource_manager.list_resources()) - logging.info(resource_manager.list_opened_resources()) + logger.info(resource_manager.list_resources()) + logger.info(resource_manager.list_opened_resources()) instrument = resource_manager.open_resource("ASRL1::INSTR") - logging.info(instrument) + logger.info(instrument) instrument.close() if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/high-performance-computing/hm-jax/src/main.py b/high-performance-computing/hm-jax/src/main.py index 37ff573d33..1d5bcf959f 100644 --- a/high-performance-computing/hm-jax/src/main.py +++ b/high-performance-computing/hm-jax/src/main.py @@ -4,6 +4,8 @@ import matplotlib.pyplot as plt from jax import grad, jit, random +logger = logging.getLogger(__name__) + # The linear model def predict(w: float, b: float, X: jnp.ndarray) -> jnp.ndarray: @@ -59,7 +61,7 @@ def main() -> None: current_loss = loss_fn(w, b, X, y) losses.append(current_loss) if i % 10 == 0: - logging.info( + logger.info( f"Iteration {i}: loss = {current_loss:.4f}, w = {w:.4f}, b = {b:.4f}" ) @@ -78,5 +80,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/hm-xxhash/src/main.py b/hm-xxhash/src/main.py index dc721ea6d4..38b197412a 100644 --- a/hm-xxhash/src/main.py +++ b/hm-xxhash/src/main.py @@ -3,6 +3,8 @@ import xxhash +logger = logging.getLogger(__name__) + def get_file_xxh128(file_path: Path) -> str: hash = xxhash.xxh128() @@ -18,9 +20,11 @@ def get_file_xxh128(file_path: Path) -> str: def main() -> None: file_path = Path("src/main.py") xxh128 = get_file_xxh128(file_path) - logging.info(xxh128) + logger.info(xxh128) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/dali/src/main.py b/machine-learning/dali/src/main.py index 7a593f020e..84050d575b 100644 --- a/machine-learning/dali/src/main.py +++ b/machine-learning/dali/src/main.py @@ -8,6 +8,8 @@ from nvidia.dali import pipeline_def from nvidia.dali.plugin.pytorch import DALIGenericIterator +logger = logging.getLogger(__name__) + def download_sample_images(data_path: Path) -> None: # Create main directory if it doesn not exist @@ -28,10 +30,10 @@ def download_sample_images(data_path: Path) -> None: filename = f"image_{i}.jpg" filepath = class_dir / filename if not filepath.exists(): - logging.info(f"Downloading {url} to {filepath}") + logger.info(f"Downloading {url} to {filepath}") urllib.request.urlretrieve(url, str(filepath)) except Exception as e: - logging.error(f"Error downloading {url}: {e}") + logger.exception(f"Error downloading {url}: {e}") @pipeline_def(batch_size=2, num_threads=2, device_id=None) @@ -69,7 +71,7 @@ def main() -> None: # Get total number of samples num_samples = get_num_samples(data_path) if num_samples == 0: - logging.error("No images available in the directory.") + logger.exception("No images available in the directory.") return pipe = image_pipeline( @@ -84,18 +86,20 @@ def main() -> None: auto_reset=True, ) - logging.info("Pipeline created successfully!") - logging.info(f"Ready to process images from {data_path}") + logger.info("Pipeline created successfully!") + logger.info(f"Ready to process images from {data_path}") try: for i, data in enumerate(dali_iter): images: torch.Tensor = data[0]["data"] labels: torch.Tensor = data[0]["label"] - logging.info(f"Batch {i}: Image shape: {images.shape}, Labels: {labels}") + logger.info(f"Batch {i}: Image shape: {images.shape}, Labels: {labels}") except StopIteration: - logging.info("Finished processing all images.") + logger.info("Finished processing all images.") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/hm-autogluon/src/main.py b/machine-learning/hm-autogluon/src/main.py index 7aff3677c2..80937476ab 100644 --- a/machine-learning/hm-autogluon/src/main.py +++ b/machine-learning/hm-autogluon/src/main.py @@ -3,6 +3,8 @@ import pandas as pd from autogluon.tabular import TabularDataset, TabularPredictor +logger = logging.getLogger(__name__) + def main(dataset_url: str, label: str) -> None: train_data = TabularDataset(dataset_url) @@ -11,18 +13,20 @@ def main(dataset_url: str, label: str) -> None: test_data = TabularDataset(dataset_url) y_pred = predictor.predict(test_data.drop(columns=[label])) - logging.info(f"{y_pred = }") + logger.info(f"{y_pred = }") performance = predictor.evaluate(test_data) - logging.info(f"{performance = }") + logger.info(f"{performance = }") leaderboard = predictor.leaderboard(test_data) - logging.info(f"{type(leaderboard) = }") - logging.info(f"{leaderboard = }") + logger.info(f"{type(leaderboard) = }") + logger.info(f"{leaderboard = }") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) # pd.set_option("display.max_rows", None) pd.set_option("display.max_columns", None) dataset_url = ( diff --git a/machine-learning/hm-docling/src/main.py b/machine-learning/hm-docling/src/main.py index 3c0c9822bf..cfcdf402d1 100644 --- a/machine-learning/hm-docling/src/main.py +++ b/machine-learning/hm-docling/src/main.py @@ -9,6 +9,8 @@ ) from docling.document_converter import DocumentConverter, PdfFormatOption +logger = logging.getLogger(__name__) + def main() -> None: data_dir = Path("data") @@ -38,11 +40,13 @@ def main() -> None: # Write markdown to file markdown_path = pdf_path.with_suffix(".md") markdown_path.write_text(markdown_content, encoding="utf-8") - logging.info(f"Converted {pdf_path.name}") + logger.info(f"Converted {pdf_path.name}") except Exception as e: - logging.info(f"Error processing {pdf_path.name}: {e}") + logger.info(f"Error processing {pdf_path.name}: {e}") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/hm-langchain/applications/chat-pdf/src/main.py b/machine-learning/hm-langchain/applications/chat-pdf/src/main.py index ce08101af7..04430626ac 100644 --- a/machine-learning/hm-langchain/applications/chat-pdf/src/main.py +++ b/machine-learning/hm-langchain/applications/chat-pdf/src/main.py @@ -7,6 +7,8 @@ from langchain_community.vectorstores import Chroma from langchain_text_splitters import RecursiveCharacterTextSplitter +logger = logging.getLogger(__name__) + def main(model_path: str, pdf_path: str, question: str) -> None: loader = PyPDFLoader(file_path=pdf_path) @@ -20,11 +22,13 @@ def main(model_path: str, pdf_path: str, question: str) -> None: llm=llm, chain_type="stuff", retriever=vectorstore.as_retriever() ) answer = qa.run(question) - logging.info(answer) + logger.info(answer) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) # https://gpt4all.io/index.html external_model_path = "data/ggml-model-gpt4all-falcon-q4_0.bin" diff --git a/machine-learning/hm-llama-index/applications/chat-pdf/src/main.py b/machine-learning/hm-llama-index/applications/chat-pdf/src/main.py index dacd99e064..f58ad9e65e 100644 --- a/machine-learning/hm-llama-index/applications/chat-pdf/src/main.py +++ b/machine-learning/hm-llama-index/applications/chat-pdf/src/main.py @@ -6,6 +6,8 @@ from llama_index.embeddings.huggingface import HuggingFaceEmbedding from llama_index.llms.litellm import LiteLLM +logger = logging.getLogger(__name__) + def chat_with_pdf(pdf_path: Path, question: str) -> str: documents = SimpleDirectoryReader(input_files=[pdf_path]).load_data() @@ -31,9 +33,11 @@ def main() -> None: pdf_path = Path("data/file.pdf") question = "Could you please summarize this PDF?" answer = chat_with_pdf(pdf_path, question) - logging.info(answer) + logger.info(answer) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/hm-mlflow/experiments/predict-diabetes/src/main.py b/machine-learning/hm-mlflow/experiments/predict-diabetes/src/main.py index 7a6bcf2a44..76b957a8f8 100644 --- a/machine-learning/hm-mlflow/experiments/predict-diabetes/src/main.py +++ b/machine-learning/hm-mlflow/experiments/predict-diabetes/src/main.py @@ -6,6 +6,8 @@ from sklearn.ensemble import RandomForestRegressor from sklearn.model_selection import train_test_split +logger = logging.getLogger(__name__) + def main(): mlflow.set_tracking_uri(config.MLFLOW_TRACKING_SERVER_URL) @@ -22,9 +24,11 @@ def main(): random_forest_regressor.fit(x_train, y_train) predictions = random_forest_regressor.predict(x_test) - logging.info(f"{predictions = }") + logger.info(f"{predictions = }") if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/hm-supervision/detect-objects/src/main.py b/machine-learning/hm-supervision/detect-objects/src/main.py index 21b0b2e4af..543015c296 100644 --- a/machine-learning/hm-supervision/detect-objects/src/main.py +++ b/machine-learning/hm-supervision/detect-objects/src/main.py @@ -6,6 +6,8 @@ import supervision as sv from ultralytics import YOLO +logger = logging.getLogger(__name__) + class_colors = {} @@ -49,7 +51,9 @@ def main(model_path: Path, image_path: Path) -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) data_dir_path = Path("data") external_model_path = data_dir_path / Path("yolov8x.pt") external_image_path = data_dir_path / Path("image.jpg") diff --git a/machine-learning/hugging-face/analyze-sentiment/src/main.py b/machine-learning/hugging-face/analyze-sentiment/src/main.py index 2e4d3cef05..eca125a8d4 100644 --- a/machine-learning/hugging-face/analyze-sentiment/src/main.py +++ b/machine-learning/hugging-face/analyze-sentiment/src/main.py @@ -2,13 +2,17 @@ from transformers import pipeline +logger = logging.getLogger(__name__) + def main() -> None: classifier = pipeline("sentiment-analysis") - logging.info(classifier("I've been waiting for this course my whole life.")) - logging.info(classifier("I've been waiting for so long for this package to ship.")) + logger.info(classifier("I've been waiting for this course my whole life.")) + logger.info(classifier("I've been waiting for so long for this package to ship.")) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/mineru/src/main.py b/machine-learning/mineru/src/main.py index 6f5f2dfdf6..c362b607cb 100644 --- a/machine-learning/mineru/src/main.py +++ b/machine-learning/mineru/src/main.py @@ -6,11 +6,13 @@ from magic_pdf.data.dataset import PymuDocDataset from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze +logger = logging.getLogger(__name__) + def process_pdf(pdf_file_path: Path, output_dir_path: Path) -> None: pdf_file_stem = pdf_file_path.stem output_dir_path.mkdir(parents=True, exist_ok=True) - logging.info(f"Processing PDF: {pdf_file_path}") + logger.info(f"Processing PDF: {pdf_file_path}") image_writer = FileBasedDataWriter(str(output_dir_path)) markdown_writer = FileBasedDataWriter(str(output_dir_path)) @@ -23,7 +25,7 @@ def process_pdf(pdf_file_path: Path, output_dir_path: Path) -> None: # Process PDF based on type pdf_parse_method = dataset.classify() - logging.info(f"Processing PDF using {pdf_parse_method} mode") + logger.info(f"Processing PDF using {pdf_parse_method} mode") if pdf_parse_method == SupportedPdfParseMethod.OCR: infer_result = dataset.apply(doc_analyze, ocr=True) @@ -47,7 +49,7 @@ def process_pdf(pdf_file_path: Path, output_dir_path: Path) -> None: markdown_writer, content_list_output_name, image_dir_name ) - logging.info("Processing completed successfully") + logger.info("Processing completed successfully") def main() -> None: @@ -61,5 +63,7 @@ def main() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/machine-learning/triton/amazon-sagemaker-triton-resnet-50/deploy/src/deploy.py b/machine-learning/triton/amazon-sagemaker-triton-resnet-50/deploy/src/deploy.py index a85032c22d..ee28c8f4d9 100644 --- a/machine-learning/triton/amazon-sagemaker-triton-resnet-50/deploy/src/deploy.py +++ b/machine-learning/triton/amazon-sagemaker-triton-resnet-50/deploy/src/deploy.py @@ -5,6 +5,8 @@ import sagemaker from botocore.client import BaseClient +logger = logging.getLogger(__name__) + def check_endpoint_status( sagemaker_client: BaseClient, sagemaker_endpoint_name: str @@ -74,7 +76,7 @@ def deploy() -> None: "Mode": "MultiModel", }, ) - logging.info(f'Model Arn: {res["ModelArn"]}') + logger.info(f'Model Arn: {res["ModelArn"]}') # Create an endpoint config res = sagemaker_client.create_endpoint_config( @@ -89,14 +91,14 @@ def deploy() -> None: } ], ) - logging.info(f'Endpoint Config Arn: {res["EndpointConfigArn"]}') + logger.info(f'Endpoint Config Arn: {res["EndpointConfigArn"]}') # Create an endpoint res = sagemaker_client.create_endpoint( EndpointName=sagemaker_endpoint_name, EndpointConfigName=sagemaker_endpoint_config_name, ) - logging.info(f'Endpoint Arn: {res["EndpointArn"]}') + logger.info(f'Endpoint Arn: {res["EndpointArn"]}') check_endpoint_status(sagemaker_client, sagemaker_endpoint_name) @@ -138,5 +140,7 @@ def deploy() -> None: if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) deploy() diff --git a/machine-learning/triton/amazon-sagemaker-triton-resnet-50/infer/src/main.py b/machine-learning/triton/amazon-sagemaker-triton-resnet-50/infer/src/main.py index 0a70898e66..a2fe47a1bb 100644 --- a/machine-learning/triton/amazon-sagemaker-triton-resnet-50/infer/src/main.py +++ b/machine-learning/triton/amazon-sagemaker-triton-resnet-50/infer/src/main.py @@ -6,6 +6,8 @@ from utils.predict_by_pytorch_model import predict_by_pytorch_model from utils.predict_by_tensorrt_model import predict_by_tensorrt_model +logger = logging.getLogger(__name__) + def main() -> None: image_path = Path("data/dog.jpg") @@ -21,7 +23,7 @@ def main() -> None: result = predict_by_tensorrt_model( sagemaker_runtime_client, endpoint_name, request_body, header_length ) - logging.info(result.as_numpy(output_name)) + logger.info(result.as_numpy(output_name)) # PyTorch input_name = "INPUT__0" @@ -32,9 +34,11 @@ def main() -> None: result = predict_by_pytorch_model( sagemaker_runtime_client, endpoint_name, request_body, header_length ) - logging.info(result.as_numpy(output_name)) + logger.info(result.as_numpy(output_name)) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/matlab/call-matlab-function-in-python/src/main.py b/matlab/call-matlab-function-in-python/src/main.py index ded978b6a3..7cf00284ea 100644 --- a/matlab/call-matlab-function-in-python/src/main.py +++ b/matlab/call-matlab-function-in-python/src/main.py @@ -2,13 +2,17 @@ import matlab.engine +logger = logging.getLogger(__name__) + def main() -> None: matlab_engine = matlab.engine.start_matlab() greatest_common_divisor = matlab_engine.gcd(100.0, 80.0) - logging.info(greatest_common_divisor) + logger.info(greatest_common_divisor) if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main() diff --git a/quantum-computing/src/main.py b/quantum-computing/src/main.py index 466a62d32b..088bddb817 100644 --- a/quantum-computing/src/main.py +++ b/quantum-computing/src/main.py @@ -7,6 +7,8 @@ from qiskit_ibm_runtime import EstimatorV2 as Estimator from qiskit_ibm_runtime.fake_provider import FakeAlmadenV2 +logger = logging.getLogger(__name__) + def main(): # Create a new circuit with two qubits @@ -35,18 +37,18 @@ def main(): job = estimator.run([(isa_circuit, mapped_observables)]) result = job.result() - logging.info(f"{result = }") + logger.info(f"{result = }") # Submitted one Pub, so this contains one inner result (and some metadata of its own), # which had five observables, so contains information on all five. pub_result = job.result()[0] - logging.info(f"{pub_result = }") + logger.info(f"{pub_result = }") values = pub_result.data.evs - logging.info(f"{values = }") + logger.info(f"{values = }") errors = pub_result.data.stds - logging.info(f"{errors = }") + logger.info(f"{errors = }") plt.plot(observables_labels, values, "-o") plt.xlabel("Observables") @@ -55,5 +57,7 @@ def main(): if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) + logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" + ) main()