feat: develop the bouding box cropper processing (#136)

.github/workflows/engine_tests.yaml

@@ -17,6 +17,10 @@ permissions:
   contents: read
   pull-requests: write
 
+env:
+  PICSELLIA_TEST_TOKEN: ${{ secrets.PICSELLIA_TEST_TOKEN }}
+  PICSELLIA_TEST_HOST: ${{ vars.PICSELLIA_TEST_HOST }}
+
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -34,7 +38,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install flake8 pytest pytest-cov
+        pip install flake8
+        pip install picsellia
         pip install -r requirements-dev.txt
 
     - name: Lint with flake8

.pre-commit-config.yaml

@@ -15,6 +15,8 @@ repos:
     - id: check-toml
     - id: check-yaml
     - id: end-of-file-fixer
+    - id: file-contents-sorter
+      files: requirements.in
     - id: fix-byte-order-marker
     - id: mixed-line-ending
       args: ["--fix=lf"]
@@ -44,5 +46,6 @@ repos:
     - id: mypy
       args:
         - --check-untyped-defs
+        - --ignore-missing-imports
       exclude: ^tests/
       additional_dependencies: ['types-PyYAML']

base/cpu/Dockerfile

@@ -10,11 +10,10 @@ RUN if [ "$PYTHON_VERSION" = "3.10" ]; then \
         python${PYTHON_VERSION} -m ensurepip --upgrade && python${PYTHON_VERSION} -m pip install -U setuptools; \
     fi
 
-RUN pip install uv
+RUN python${PYTHON_VERSION} -m pip install uv
 
 COPY ./base /experiment
 
-RUN ln -s /experiment/run.sh /usr/bin/run
-
-RUN chmod +x /experiment/run.sh
-RUN chown -R 42420:42420 /experiment
+RUN ln -s /experiment/run.sh /usr/bin/run && \
+    chmod +x /experiment/run.sh && \
+    chown -R 42420:42420 /experiment

base/cuda/Dockerfile

@@ -15,14 +15,13 @@ RUN apt-get update && apt-get upgrade -y && \
         python${PYTHON_VERSION}-full python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-distutils && \
     rm -rf /var/lib/apt/lists/*
 
-
-RUN if [ "$PYTHON_VERSION" = "3.10" ]; then \
-        python${PYTHON_VERSION} -m ensurepip --upgrade && python${PYTHON_VERSION} -m pip install -U setuptools; \
-    elif [ "$PYTHON_VERSION" = "3.8" ]; then \
-        python${PYTHON_VERSION} -m pip install --upgrade pip && pip3 install -U setuptools; \
+RUN if [ "$PYTHON_VERSION" = "3.8" ]; then \
+        python${PYTHON_VERSION} -m pip install --upgrade pip && python${PYTHON_VERSION} -m pip install -U setuptools; \
+    else \
+        python${PYTHON_VERSION} -m ensurepip --upgrade && python${PYTHON_VERSION} -m pip install --upgrade pip && python${PYTHON_VERSION} -m pip install -U setuptools; \
     fi
 
-RUN pip install uv
+RUN python${PYTHON_VERSION} -m pip install uv
 
 ENV PATH="/usr/local/cuda/bin:${PATH}" \
     LD_LIBRARY_PATH="/usr/local/cuda/lib:/usr/local/cuda/lib64:${LD_LIBRARY_PATH}"

base/run.sh

@@ -1,20 +1,7 @@
 #!/bin/bash
 
 usage() {
-  echo "Usage: $0 <training_script>.py"
-  exit 1
-}
-
-# Depending on the model to train, choose to either use python3.8 or python3.10
-get_python_command() {
-  for version in 3.12 3.11 3.10 3.9 3.8; do
-    if command -v python$version &> /dev/null; then
-      echo "python$version"
-      return 0
-    fi
-  done
-
-  echo "Error: No supported Python version (3.8 to 3.12) is available on the system."
+  echo "Usage: $0 <python_version> <training_script>.py"
   exit 1
 }
 
@@ -26,38 +13,46 @@ monitor_log_handler() {
   while :; do
     if ! kill -0 "$log_handler_pid" 2>/dev/null; then
       kill -9 "$training_script_pid"
-      echo -e "\e[31mError:\e[0m The log handler has terminated unexpectedly. The training script has been stopped to prevent sata loss. Please check the log file or the environment variables provided.\e[0m"
+      echo -e "\e[31mError:\e[0m The log handler has terminated unexpectedly. The training script has been stopped to prevent data loss. Please check the log file or the environment variables provided.\e[0m"
       exit 1
     fi
     sleep 1
   done
 }
 
 # Validate input arguments
-if [ $# -ne 1 ]; then
-  echo "Error: Exactly one argument is required."
+if [ $# -ne 2 ]; then
+  echo "Error: Exactly two arguments are required."
+  usage
+fi
+
+python_version=$1
+script_file=$2
+
+if [[ "$python_version" != python3.* ]]; then
+  echo "Error: The first argument must be a valid Python version (e.g., python3.8)."
   usage
 fi
 
-if [ "${1: -3}" != ".py" ]; then
-  echo "Error: The argument must be a Python file (.py)."
+if [ "${script_file: -3}" != ".py" ]; then
+  echo "Error: The second argument must be a Python file (.py)."
   usage
 fi
 
-if [ ! -f "$1" ]; then
-  echo "Error: The file $1 does not exist."
+if [ ! -f "$script_file" ]; then
+  echo "Error: The file $script_file does not exist."
   usage
 fi
 
 log_file_path="/experiment/training.log"
-python_cmd=$(get_python_command)
+python_cmd=$python_version
 
 # 1. Start the log handler script in the background
 $python_cmd logs/handler.py --log_file_path "$log_file_path" &
 log_handler_pid=$!
 
 # 2. Start the training script in the background and redirect output to log file
-$python_cmd "$1" > "$log_file_path" 2>&1 &
+$python_cmd "$script_file" > "$log_file_path" 2>&1 &
 training_script_pid=$!
 
 # 3. Start the monitor in the background

src/decorators/pipeline_decorator.py

@@ -1,14 +1,14 @@
 import ast
 import inspect
-from typing import Optional, Union, Callable, TypeVar, Any, List, Tuple, Dict
+from typing import Optional, Union, Callable, TypeVar, Any, List, Tuple, Dict, overload
 
 from tabulate import tabulate  # type: ignore
 
 from src.enums import PipelineState, StepState
 from src.logger import LoggerManager
 from src.models.steps.step_metadata import StepMetadata
 
-F = TypeVar("F", bound=Callable[..., None])
+F = TypeVar("F", bound=Callable[..., Any])
 
 
 class Pipeline:
@@ -407,6 +407,22 @@ def register_step_metadata(step_metadata: StepMetadata) -> None:
         Pipeline.STEPS_REGISTRY[step_name] = step_metadata
 
 
+@overload
+def pipeline(_func: F) -> Pipeline:  # pragma: no cover
+    ...
+
+
+@overload
+def pipeline(
+    *,
+    context: Optional[Any] = None,
+    name: Optional[str] = None,
+    log_folder_path: Optional[str] = None,
+    remove_logs_on_completion: bool = True,
+) -> Callable[[F], Pipeline]:  # pragma: no cover
+    ...
+
+
 def pipeline(
     _func: Optional["F"] = None,
     context: Optional[Any] = None,

src/decorators/step_decorator.py

@@ -1,13 +1,13 @@
 import logging
 import time
 import uuid
-from typing import Callable, Union, TypeVar, Optional, Any
+from typing import Callable, Union, TypeVar, Optional, Any, overload
 
 from src import Pipeline
 from src.enums import StepState, PipelineState
 from src.models.steps.step_metadata import StepMetadata
 
-F = TypeVar("F", bound=Callable[..., None])
+F = TypeVar("F", bound=Callable[..., Any])
 
 
 class Step:
@@ -173,6 +173,18 @@ def _prepare_step_logger(self, pipeline: Pipeline) -> logging.Logger:
         )
 
 
+@overload
+def step(_func: F) -> Step:  # pragma: no cover
+    ...
+
+
+@overload
+def step(
+    *, name: Optional[str] = None, continue_on_failure: bool = False
+) -> Callable[[F], Step]:  # pragma: no cover
+    ...
+
+
 def step(
     _func: Optional["F"] = None,
     name: Optional[str] = None,