From 0c9acb27e030cb997832740f1177a692a634a38e Mon Sep 17 00:00:00 2001 From: Ivo Facoco Date: Mon, 9 Dec 2024 11:26:54 +0000 Subject: [PATCH 1/3] Allow numpy 2.0 and pydantic >2.9.0 --- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ded9ee6..51186e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -160,7 +160,7 @@ repos: description: Automatically upgrade syntax for newer versions of the language args: [--py36-plus] - repo: https://github.com/jendrikseipp/vulture - rev: v2.13 + rev: v2.14 hooks: - id: vulture name: vulture - finds unused code in Python programs @@ -244,13 +244,13 @@ repos: language: system files: requirements/*.txt - repo: https://github.com/PyCQA/bandit - rev: 1.7.10 + rev: 1.8.0 hooks: - id: bandit args: ["-c", "pyproject.toml"] additional_dependencies: [".[toml]"] - repo: https://github.com/PyCQA/bandit - rev: 1.7.10 + rev: 1.8.0 hooks: - id: bandit name: bandit - find common security issues in Python code. diff --git a/pyproject.toml b/pyproject.toml index 8f64293..73711bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,10 +36,10 @@ exclude = ["tests/*", "notebooks/*", "docs/*", "src/**/main.py", python = ">=3.9, <3.13" loguru = {version = ">=0.7.2, <0.8.0"} matplotlib = {version = ">=3.4.3, <4.0.0"} -numpy = {version = ">=1.22.0, <2.0.0"} +numpy = {version = ">=1.22.0, <2.5.0"} piq = {version = ">=0.8.0, <1.0.0"} pot = {version = ">=0.9.4, <0.10.0"} -pydantic = {version = ">=2.8.2, <2.9.0"} +pydantic = {version = ">=2.8.2, <3.0.0"} python-dotenv = {version = ">=1.0.0, <2.0.0"} torch = {version = ">=2.1.0, <2.5.0"} gudhi = {version = ">=3.9.0, <=4.0.0"} @@ -98,7 +98,7 @@ optional = true black = "^23.11.0" flake8 = "^6.1.0" isort = "^5.12.0" -mypy = "^1.7.1" +mypy = "^1.13.0" pre-commit = "^3.5.0" pytest = "^7.4.3" pytest-cov = "^4.1.0" From 6323fa8e01d4de3737e8688d9dd20c48d84945a9 Mon Sep 17 00:00:00 2001 From: Ivo Facoco Date: Tue, 10 Dec 2024 09:53:22 +0000 Subject: [PATCH 2/3] fix: rename internal extract_features_dataloader to _extract_features_dataloader in image extractor fix: simplified StandardTransform to work with tensors feat: add simple implementation for dataloader feature extraction in image docs: rewrote google style docs to numpydoc style --- src/pymdma/common/definitions.py | 2 +- src/pymdma/image/input_layer.py | 4 +- src/pymdma/image/models/extractor.py | 100 ++++++++++++++++++++++----- src/pymdma/image/models/imagenet.py | 4 +- tests/conftest.py | 14 ++++ tests/test_image_import.py | 39 +++++++++++ 6 files changed, 142 insertions(+), 21 deletions(-) diff --git a/src/pymdma/common/definitions.py b/src/pymdma/common/definitions.py index fa53c04..eb31468 100644 --- a/src/pymdma/common/definitions.py +++ b/src/pymdma/common/definitions.py @@ -79,5 +79,5 @@ def __init__(self, name: str) -> None: self.name = name @abstractmethod - def extract_features_dataloader(self, dataloader): + def _extract_features_dataloader(self, dataloader): pass diff --git a/src/pymdma/image/input_layer.py b/src/pymdma/image/input_layer.py index 4f63ec9..6e79376 100644 --- a/src/pymdma/image/input_layer.py +++ b/src/pymdma/image/input_layer.py @@ -208,11 +208,11 @@ def get_embeddings( extractor = ExtractorFactory.model_from_name(model_name) if extractor is None else extractor # extractor = model_instance if model_instance is not None else FeatureExtractor(model_name, device=self.device) - reference_feats, _labels, _reference_ids = extractor.extract_features_dataloader( + reference_feats, _labels, _reference_ids = extractor._extract_features_dataloader( self.reference_loader, device=self.device, ) - synthetic_feats, _labels, synthetic_ids = extractor.extract_features_dataloader( + synthetic_feats, _labels, synthetic_ids = extractor._extract_features_dataloader( self.target_loader, device=self.device, ) diff --git a/src/pymdma/image/models/extractor.py b/src/pymdma/image/models/extractor.py index f356fe6..a1edcfd 100644 --- a/src/pymdma/image/models/extractor.py +++ b/src/pymdma/image/models/extractor.py @@ -6,6 +6,7 @@ from loguru import logger from PIL import Image from torch.utils.data import DataLoader +from torchvision.transforms import transforms from tqdm import tqdm from pymdma.common.definitions import EmbedderInterface @@ -26,13 +27,15 @@ def __init__( self.interp = interpolation self.preprocess_transform = preprocess_transform + self._to_tensor = transforms.PILToTensor() + def __call__(self, image: Image.Image) -> torch.Tensor: image = self.preprocess_transform(image) if self.preprocess_transform is not None else image image = image.resize(self.img_size, self.interp) # bring image to the range [0, 1] and normalize to [-1, 1] - image = np.array(image).astype(np.float32) / 255.0 + image = self._to_tensor(image).float() / 255.0 image = image * 2.0 - 1.0 - return torch.from_numpy(image).permute(2, 0, 1).float() + return image class BaseExtractor(torch.nn.Module, EmbedderInterface): @@ -55,14 +58,20 @@ def extract_features_from_files( device: str = "cpu", preprocess_transform: Optional[Callable] = None, ) -> np.ndarray: - """Extract features from a list of image files. - - Args: - files (List[Path]): list of paths to image files - batch_size (int): batch size for feature extraction. Defaults to 50. - - Returns: - np.ndarray: array of features + """Extract features from a list of image files. Converts images to + tensors and normalizes them to the [-1, 1] range. + + Parameters + ---------- + files : List[Path] + list of paths to image files + batch_size : int, optional + batch size for feature extraction, by default 50 + + Returns + ------- + np.ndarray + array of features with shape (n_samples, n_features) """ if batch_size > len(files): # print("Warning: batch size is bigger than the data size. " "Setting batch size to data size") @@ -90,19 +99,78 @@ def extract_features_from_files( return np.concatenate(act_array, axis=0) @torch.no_grad() - def extract_features_dataloader( + def extract_features_from_dataloader( + self, + dataloader: DataLoader, + normalize: bool = False, + device: str = "cpu", + ): + """Extract features from a DataLoader. + + Parameters + ---------- + dataloader : DataLoader + PyTorch DataLoader + normalize : bool, optional + Wether to normalize the images to 0.5 mean and 0.5 std across channels. + Assumes that the images are in the [0, 1] range, by default False + device : str, optional + device to use, by default "cpu" + + Notes + ----- + Has the following assumptions: + - Dataloader outputs a tuple with the first element being the image batch tensor. + - Image tensors are of shape (batch_size, channels, height, width) + - Tensors are in the [0, 1] range in which case you should set `normalize` to True. + - Tensors have been normalized to the [-1, 1] range (0.5 mean and 0.5 std across channels) or to another range (e.g. ImageNet normalization). + + It is recommended to disable the `shuffle` option in the DataLoader for consistency. + + Depending on the model, you might need to resize the images to the model's input size. + + Returns + ------- + np.ndarray + array of features with shape (n_samples, n_features) + """ + self.extractor = self.extractor.to(device, dtype=torch.float32) + + # validation dry run + sample_batch = next(iter(dataloader)) + assert isinstance( + sample_batch[0], torch.Tensor + ), f"First element of the tuple must be a torch.Tensor. Got {type(sample_batch[0])}." + + if sample_batch[0].shape[2:] != self.input_size: + logger.warning( + f"Model default input size {self.input_size} does not match the size of the images in the dataloader {sample_batch[0][2:]}. Might lead to execution errors." + ) + + act_array = [] + for batch in tqdm(dataloader, total=len(dataloader)): + images = batch[0] + images = images.to(device, dtype=torch.float32) + if normalize: + images = images * 2.0 - 1.0 + act_array.append(self(images).detach().cpu().numpy()) + return np.concatenate(act_array, axis=0) + + @torch.no_grad() + def _extract_features_dataloader( self, dataloader: DataLoader, device: str = "cpu", preprocess_transform: Optional[Callable] = None, ) -> Tuple[np.ndarray, np.ndarray]: - """Use selected model to extract features from all images in - dataloader. + """Internal method to extract features from a DataLoader. - Args: - dataloader (DataLoader): image dataloader + Parameters + ---------- + dataloader : DataLoader + image dataloader Returns: - Tuple[np.ndarray, np.ndarray]: array of features and array of image labels + Tuple[np.ndarray, np.ndarray, np.ndarray]: extracted features, labels, and image ids """ logger.info("Extracting image features.") act_array = [] diff --git a/src/pymdma/image/models/imagenet.py b/src/pymdma/image/models/imagenet.py index 3751187..dcc5d8a 100644 --- a/src/pymdma/image/models/imagenet.py +++ b/src/pymdma/image/models/imagenet.py @@ -63,9 +63,9 @@ def forward(self, x): class DinoExtractor(BaseExtractor): - def __init__(self, model_name) -> None: + def __init__(self, model_name, input_size: tuple[int, int] = (224, 224)) -> None: super().__init__( - input_size=(224, 224), + input_size=input_size, interpolation=Image.Resampling.BICUBIC, ) diff --git a/tests/conftest.py b/tests/conftest.py index 21338a4..db053bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ import pytest from fastapi.testclient import TestClient from PIL import Image +from torchvision.transforms import transforms from pymdma.api.run_api import app from pymdma.config import data_dir @@ -65,6 +66,19 @@ def get_extractor(name): return get_extractor +@pytest.fixture(scope="module") +def image_transforms(): + def get_transforms(input_size: Tuple[int], interpolation: int = Image.BILINEAR): + return transforms.Compose( + [ + transforms.Resize(input_size, interpolation=interpolation), + transforms.ToTensor(), + ] + ) + + return get_transforms + + # ################################################################################################### # ################################## Time-Series Fixtures ########################################### # ################################################################################################### diff --git a/tests/test_image_import.py b/tests/test_image_import.py index 9f5d84d..4e9a8f3 100644 --- a/tests/test_image_import.py +++ b/tests/test_image_import.py @@ -1,7 +1,9 @@ import matplotlib.pyplot as plt import numpy as np import pytest +from torch.utils.data import DataLoader +from pymdma.image.data.simple_dataset import SimpleDataset from pymdma.image.measures.input_val.annotation import coco as ann from pymdma.image.measures.input_val.data import no_reference as no_ref_quality from pymdma.image.measures.input_val.data import reference as ref_quality @@ -111,3 +113,40 @@ def test_extractor_models(image_feature_extractor, synth_image_filenames, extrac dataset_level, instance_level = result.value assert dataset_level > 0.90, "Dataset level is below threshold" assert all([inst == 1 for inst in instance_level]), "Same image instance should be precise" + + +@pytest.mark.parametrize( + "extractor_name, input_size", + [ + ("inception_fid", (299, 299)), + ("vgg16", (224, 224)), + ("dino_vits8", (224, 224)), + ("vit_b_16", (224, 224)), + ], +) +def test_extractor_methods( + image_feature_extractor, synth_image_filenames, image_transforms, extractor_name, input_size +): + extractor = image_feature_extractor(extractor_name) + + dataset = SimpleDataset(synth_image_filenames, image_transforms(input_size, extractor.interpolation)) + dataloader = DataLoader(dataset, batch_size=2, shuffle=False) + + features_files = extractor.extract_features_from_files(synth_image_filenames) + features_dataloader = extractor.extract_features_from_dataloader(dataloader, normalize=True) + + assert features_files.shape[0] == len(synth_image_filenames), "Feature length does not match input length" + assert features_dataloader.shape[0] == len(synth_image_filenames), "Feature length does not match input length" + + assert ( + features_dataloader.mean() == features_files.mean() + ), "Feature extraction from files and dataloader should be the same" + assert ( + features_dataloader.std() == features_files.std() + ), "Feature extraction from files and dataloader should be the same" + + prec = ImprovedPrecision() + result = prec.compute(features_files, features_dataloader) + dataset_level, instance_level = result.value + assert dataset_level > 0.98, "Dataset level is below threshold" + assert all([inst == 1 for inst in instance_level]), "Same image instance should be precise" From dd96e8ee8c84d97b19cb198cc251d3f30738eb8e Mon Sep 17 00:00:00 2001 From: Ivo Facoco Date: Tue, 10 Dec 2024 10:09:38 +0000 Subject: [PATCH 3/3] Minor patch 0.1.6 - Image extractor and minor version --- CHANGELOG.md | 19 +++++++++++++++++-- VERSION | 2 +- pyproject.toml | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bb30dd..3928997 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file. This format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.6] - 2024-12-10 +Minor patch release with new image features extraction method and documentation updates. + +### Added + - `extract_features_from_dataloader` method to the image `BaseExtractor` class + +### Changed + - Allowed for `numpy >= 2.0.0` in the requirements + - Allowed `pydantic>2.9.0` in the requirements + - Moved internal method in the image `BaseExtractor` to underscore method + +### Fixed + - Simplified the `StandardTransform` in images to work directly with tensors + + ## [0.1.5] - 2024-11-29 Introduce new time-series metrics and documentation updates. @@ -20,12 +35,12 @@ Introduce new time-series metrics and documentation updates. ### Fixed - PyPI security issues due to direct external `pydom` dependency - + ## [0.1.4] - 2024-11-21 Taxonomy rework and documentation updates. -### Added +### Added - readthedocs slug in the README file - References to tabular metrics diff --git a/VERSION b/VERSION index 9faa1b7..c946ee6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.5 +0.1.6 diff --git a/pyproject.toml b/pyproject.toml index 73711bb..0e43ff1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ # https://github.com/microsoft/vscode-python/blob/master/CHANGELOG.md#enhancements-1 [tool.poetry] name = "pymdma" -version = "0.1.5" +version = "0.1.6" description = "Multimodal Data Metrics for Auditing real and synthetic data" authors = ["Fraunhofer AICOS "] maintainers = [