diff --git a/docs/source/installation.md b/docs/source/installation.md index 31c99a794c1..cba17548025 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -77,10 +77,20 @@ pip install monai MONAI supports the extras syntax such as `pip install 'monai[nibabel]'`. The options are ```text -clearml, cucim, cupy, einops, fire, gdown, h5py, huggingface_hub, hyena, ignite, imagecodecs, itk, jsonschema, lmdb, lpips, matplotlib, metrics_reloaded, mlflow, nibabel, nni, nvimgcodec, onnx, openslide, optuna, pandas, pillow, polygraphy, psutil, pyamg, pybind11, pydicom, pynrrd, pynvml, pyyaml, requests, segment_anything, scipy, skimage, tensorboard, tensorboardX, tifffile, torchio, torchvision, tqdm, transformers, zarr +clearml, cucim, cupy, einops, fire, gdown, h5py, huggingface_hub, hyena, ignite, imagecodecs, itk, jsonschema, lmdb, matplotlib, metrics_reloaded, mlflow, nibabel, nni, nvimgcodec, onnx, openslide, optuna, pandas, pillow, polygraphy, psutil, pyamg, pybind11, pydicom, pynrrd, pynvml, pyyaml, requests, segment_anything, scipy, skimage, tensorboard, tensorboardX, tifffile, torchio, torchvision, tqdm, transformers, zarr ``` -which correspond to the packages: `clearml`, `cucim` (`cucim-cu12` or `cucim-cu13`), `cupy-cuda13x`, `einops`, `fire`, `gdown`, `h5py`, `huggingface_hub`, `nvsubquadratic`, `omegaconf`, `pytorch-ignite`, `imagecodecs`, `itk`, `jsonschema`, `lmdb`, `lpips`, `matplotlib`, `MetricsReloaded`, `mlflow`, `nibabel`, `nni`, `filelock`, `nvidia-nvimgcodec-cu13`, `onnx`, `onnxruntime`, `onnx_graphsurgeon`, `onnxscript`, `openslide-python`, `openslide-bin`, `optuna`, `pandas`, `pillow`, `polygraphy`, `psutil`, `pyamg`, `pybind11`, `pydicom`, `pynrrd`, `nvidia-ml-py`, `pyyaml`, `requests`, `segment_anything`, `scipy`, `scikit-image`, `tensorboard`, `tensorboardX`, `tifffile`, `torchio`, `torchvision`, `tqdm`, `transformers`, `zarr`. +which correspond to the packages: `clearml`, `cucim` (`cucim-cu12` or `cucim-cu13`), `cupy-cuda13x`, `einops`, `fire`, `gdown`, `h5py`, `huggingface_hub`, `nvsubquadratic`, `omegaconf`, `pytorch-ignite`, `imagecodecs`, `itk`, `jsonschema`, `lmdb`, `matplotlib`, `MetricsReloaded`, `mlflow`, `nibabel`, `nni`, `filelock`, `nvidia-nvimgcodec-cu13`, `onnx`, `onnxruntime`, `onnx_graphsurgeon`, `onnxscript`, `openslide-python`, `openslide-bin`, `optuna`, `pandas`, `pillow`, `polygraphy`, `psutil`, `pyamg`, `pybind11`, `pydicom`, `pynrrd`, `nvidia-ml-py`, `pyyaml`, `requests`, `segment_anything`, `scipy`, `scikit-image`, `tensorboard`, `tensorboardX`, `tifffile`, `torchio`, `torchvision`, `tqdm`, `transformers`, `zarr`. + +The AlexNet, VGG, and SqueezeNet options in `PerceptualLoss` require `torchvision` +(`pip install "monai[torchvision]"`). The external `lpips` package is no longer required. +ImageNet feature weights are downloaded by Torchvision. LPIPS calibration weights are +downloaded from the official [MONAI Hugging Face repository](https://huggingface.co/MONAI/checkpoints), +verified with SHA-256, and cached under the Torch Hub `checkpoints` directory on first use. +`PerceptualLoss(cache_dir=...)` selects the Torch Hub cache location for both sets of weights. +With `pretrained=False`, calibration weights are not downloaded; ImageNet feature weights +are still required. +The existing `monai[lpips]` extra is retained as an alias for `monai[torchvision]`. Almost all of these can be installed together with the `all` option. For development on MONAI, this should be accompanied by `testing` which will install the testing static checking packages. Cupy and `nvimgcodec` are omitted from `all` since the choice between Cuda 12 and 13 versions of the libraries can't be resolved when installing and must be manually installed. diff --git a/docs/source/networks.rst b/docs/source/networks.rst index 14cc756d7ad..226a8140055 100644 --- a/docs/source/networks.rst +++ b/docs/source/networks.rst @@ -772,6 +772,25 @@ Nets .. autoclass:: TorchVisionFCModel :members: +`LPIPS` +~~~~~~~ +.. autoclass:: LPIPS + :members: + +`LPIPSFeatures` +~~~~~~~~~~~~~~~ +.. autoclass:: LPIPSFeatures + :members: + +`NetLinLayer` +~~~~~~~~~~~~~ +.. autoclass:: NetLinLayer + +`ScalingLayer` +~~~~~~~~~~~~~~ +.. autoclass:: ScalingLayer + :members: + `MILModel` ~~~~~~~~~~ .. autoclass:: MILModel diff --git a/monai/losses/perceptual.py b/monai/losses/perceptual.py index 6c0608f605a..e9e0bab9b20 100644 --- a/monai/losses/perceptual.py +++ b/monai/losses/perceptual.py @@ -16,6 +16,7 @@ import torch import torch.nn as nn +from monai.networks.nets import LPIPS from monai.utils import optional_import from monai.utils.enums import StrEnum @@ -24,7 +25,6 @@ ("medicalnet_resnet10_23datasets", "medicalnet_resnet50_23datasets", "radimagenet_resnet50") ) -LPIPS, _ = optional_import("lpips", name="LPIPS") torchvision, _ = optional_import("torchvision") @@ -67,8 +67,9 @@ class PerceptualLoss(nn.Module): is_fake_3d: if True use 2.5D approach for a 3D perceptual loss. fake_3d_ratio: ratio of how many slices per axis are used in the 2.5D approach. cache_dir: path to cache directory to save the pretrained network weights. - pretrained: whether to load pretrained weights. This argument only works when using networks from - LIPIS or Torchvision. Defaults to ``True``. + pretrained: whether to load pretrained weights. For "alex", "vgg", and "squeeze", this controls + the LPIPS linear calibration weights; the feature network always uses ImageNet weights from + Torchvision. For "resnet50", this controls the feature network weights. Defaults to ``True``. pretrained_path: if `pretrained` is `True`, users can specify a weights file to be loaded via using this argument. This argument only works when ``"network_type"`` is "resnet50". Defaults to `None`. @@ -151,8 +152,7 @@ def __init__( pretrained_state_dict_key=pretrained_state_dict_key, ) else: - # VGG, AlexNet and SqueezeNet are independently handled by LPIPS. - self.perceptual_function = LPIPS(pretrained=pretrained, net=network_type, verbose=False) + self.perceptual_function = LPIPS(pretrained=pretrained, net=network_type) self.is_fake_3d = is_fake_3d self.fake_3d_ratio = fake_3d_ratio diff --git a/monai/networks/nets/__init__.py b/monai/networks/nets/__init__.py index 085254f1be1..d3d197c8290 100644 --- a/monai/networks/nets/__init__.py +++ b/monai/networks/nets/__init__.py @@ -97,6 +97,7 @@ from .navit import NaViT from .netadapter import NetAdapter from .patchgan_discriminator import MultiScalePatchDiscriminator, PatchDiscriminator +from .perceptual import LPIPS, LPIPSFeatures, NetLinLayer, ScalingLayer from .quicknat import Quicknat from .regressor import Regressor from .regunet import GlobalNet, LocalNet, RegUNet diff --git a/monai/networks/nets/perceptual.py b/monai/networks/nets/perceptual.py new file mode 100644 index 00000000000..aedf8d3fc2d --- /dev/null +++ b/monai/networks/nets/perceptual.py @@ -0,0 +1,221 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Adapted from richzhang/PerceptualSimilarity, commit +# 082bb24f84c091ea94de2867d34c4544f68e0963 (lpips/lpips.py and pretrained_networks.py). +# Copyright (c) 2018, Richard Zhang, Phillip Isola, Alexei A. Efros, +# Eli Shechtman, Oliver Wang. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import annotations + +from collections import OrderedDict +from pathlib import Path + +import torch +from torch import nn + +from monai.apps.utils import download_url +from monai.utils import optional_import + +torchvision, _ = optional_import("torchvision") + +__all__ = ["LPIPS", "LPIPSFeatures", "NetLinLayer", "ScalingLayer"] + +# Resolve the default at construction time so torch.hub.set_dir() remains effective. +LPIPS_CHECKPOINTS_DIR: Path | None = None +LPIPS_WEIGHTS_URL = "https://huggingface.co/MONAI/checkpoints/resolve/main" +LPIPS_NETWORKS = { + "alex": ( + "alexnet", + "AlexNet_Weights", + (2, 5, 8, 10, 12), + (64, 192, 384, 256, 256), + "df73285e35b22355a2df87cdb6b70b343713b667eddbda73e1977e0c860835c0", + ), + "vgg": ( + "vgg16", + "VGG16_Weights", + (4, 9, 16, 23, 30), + (64, 128, 256, 512, 512), + "a78928a0af1e5f0fcb1f3b9e8f8c3a2a5a3de244d830ad5c1feddc79b8432868", + ), + "squeeze": ( + "squeezenet1_1", + "SqueezeNet1_1_Weights", + (2, 5, 8, 10, 11, 12, 13), + (64, 128, 256, 384, 384, 512, 512), + "4a5350f23600cb79923ce65bb07cbf57dca461329894153e05a1346bd531cf76", + ), +} + + +class ScalingLayer(nn.Module): + """Apply the fixed RGB shift and scale used by LPIPS v0.1.""" + + def __init__(self) -> None: + super().__init__() + self.register_buffer("shift", torch.tensor([-0.030, -0.088, -0.188])[None, :, None, None]) + self.register_buffer("scale", torch.tensor([0.458, 0.448, 0.450])[None, :, None, None]) + + def forward(self, input: torch.Tensor) -> torch.Tensor: + """Scale an image batch, broadcasting grayscale inputs to RGB. + + Args: + input: Image tensor of shape ``(B, C, H, W)``, with one or three channels. + + Returns: + Scaled image tensor of shape ``(B, 3, H, W)``. + """ + return (input - self.shift) / self.scale + + +class LPIPSFeatures(nn.Sequential): + """Extract frozen Torchvision features using legacy LPIPS slice names. + + Args: + features: Ordered feature layers from a Torchvision backbone. + endpoints: Exclusive layer indices at which each successive slice ends. + """ + + def __init__(self, features: nn.Sequential, endpoints: tuple[int, ...]) -> None: + slices: OrderedDict[str, nn.Module] = OrderedDict() + start = 0 + for index, end in enumerate(endpoints, 1): + # Keep the original names so existing PerceptualLoss checkpoints can be loaded. + block = nn.Sequential() + for layer in range(start, end): + block.add_module(str(layer), features[layer]) + slices[f"slice{index}"] = block + start = end + super().__init__(slices) + self.requires_grad_(False) + + def forward(self, input: torch.Tensor) -> tuple[torch.Tensor, ...]: + """Collect the output of each feature slice. + + Args: + input: Scaled RGB image tensor of shape ``(B, 3, H, W)``. + + Returns: + Feature tensors in slice order, each with shape ``(B, C_i, H_i, W_i)``. + """ + outputs = [] + for layer in self: + # Each slice consumes the preceding features, not the original RGB input. + input = layer(input) + outputs.append(input) + return tuple(outputs) + + +class NetLinLayer(nn.Sequential): + """Calibrate feature differences with Dropout and a bias-free 1x1 convolution. + + Args: + channels: Number of input feature channels. + """ + + def __init__(self, channels: int) -> None: + # The named child retains the model.1.weight keys in upstream calibration files + # and saved PerceptualLoss checkpoints while Sequential supplies forward(). + super().__init__(OrderedDict(model=nn.Sequential(nn.Dropout(), nn.Conv2d(channels, 1, 1, bias=False)))) + + +class LPIPS(nn.Module): + """LPIPS v0.1 network used by :class:`monai.losses.PerceptualLoss`. + + ImageNet feature weights come from Torchvision. ``pretrained`` controls only the + learned linear calibration, matching the original LPIPS constructor. The v0.1 + calibration weights are downloaded from the official ``MONAI/checkpoints`` repository + on Hugging Face, verified with SHA-256, and cached in the Torch Hub checkpoints directory. + + Args: + pretrained: Whether to load pretrained linear calibration weights. The feature + backbone always uses ImageNet weights, including when this is ``False``. + net: Feature backbone name: ``"alex"``, ``"vgg"``, or ``"squeeze"``. + + Raises: + ValueError: If ``net`` is not a supported feature backbone name. + monai.apps.HashCheckError: If the calibration checkpoint fails integrity verification. + RuntimeError: If downloading the calibration checkpoint fails. + OSError: If the calibration checkpoint cannot be downloaded or read. + """ + + def __init__(self, pretrained: bool = True, net: str = "alex") -> None: + super().__init__() + if net not in LPIPS_NETWORKS: + raise ValueError(f"Unsupported LPIPS network: {net}.") + factory, weights, endpoints, channels, hash_val = LPIPS_NETWORKS[net] + self.scaling_layer = ScalingLayer() + backbone = getattr(torchvision.models, factory)(weights=getattr(torchvision.models, weights).IMAGENET1K_V1) + self.net = LPIPSFeatures(backbone.features, endpoints) + # Legacy checkpoints contain both lin{index} and lins.{index} keys, so register + # each calibration layer under both names while sharing the same parameters. + layers = [] + for index, channel_count in enumerate(channels): + layer = NetLinLayer(channel_count) + self.add_module(f"lin{index}", layer) + layers.append(layer) + self.lins = nn.ModuleList(layers) + if pretrained: + directory = ( + LPIPS_CHECKPOINTS_DIR if LPIPS_CHECKPOINTS_DIR is not None else Path(torch.hub.get_dir(), "checkpoints") + ) + path = directory / f"lpips_v0.1_{net}.pth" + download_url( + f"{LPIPS_WEIGHTS_URL}/{net}.pth", filepath=path, hash_val=hash_val, hash_type="sha256", progress=False + ) + self.load_state_dict(torch.load(path, map_location="cpu", weights_only=True), strict=False) + self.eval() + + def forward(self, input: torch.Tensor, target: torch.Tensor) -> torch.Tensor: + """Compute one LPIPS distance for each image pair. + + Args: + input: Image tensor of shape ``(B, C, H, W)``, with one or three channels. + target: Reference image tensor with the same shape as ``input``. + + Returns: + Per-image distances of shape ``(B, 1, 1, 1)``, before the outer + ``PerceptualLoss`` reduction. + """ + features_input = self.net(self.scaling_layer(input)) + features_target = self.net(self.scaling_layer(target)) + results = [] + for layer, source, reference in zip(self.lins, features_input, features_target): + # LPIPS places epsilon outside the square root. Keep its original normalization + # rather than the different normalization used by the other perceptual networks. + source = source / (torch.sqrt(torch.sum(source**2, dim=1, keepdim=True)) + 1e-10) + reference = reference / (torch.sqrt(torch.sum(reference**2, dim=1, keepdim=True)) + 1e-10) + results.append(layer((source - reference) ** 2).mean([2, 3], keepdim=True)) + result = results[0] + for value in results[1:]: + result = result + value + return result diff --git a/pyproject.toml b/pyproject.toml index b466ae7533d..01d5aee18b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,7 +69,6 @@ all = [ "itk>=5.2", "jsonschema", "lmdb", - "lpips==0.1.4", "matplotlib>=3.6.3", "MetricsReloaded @ git+https://github.com/Project-MONAI/MetricsReloaded@monai-support", "mlflow>=3.15.2", @@ -124,7 +123,7 @@ imagecodecs = ["imagecodecs; platform_system == 'Linux' or platform_system == 'D itk = ["itk>=5.2"] jsonschema = ["jsonschema"] lmdb = ["lmdb"] -lpips = ["lpips==0.1.4"] +lpips = ["torchvision"] # compatibility extra; LPIPS is implemented within MONAI matplotlib = ["matplotlib>=3.6.3"] # per #5882 this requirement must be on one line to get removed by sed in weekly-preview.yml metrics_reloaded = ["MetricsReloaded @ git+https://github.com/Project-MONAI/MetricsReloaded@monai-support"] diff --git a/tests/losses/test_perceptual_lpips.py b/tests/losses/test_perceptual_lpips.py new file mode 100644 index 00000000000..660764ff2cf --- /dev/null +++ b/tests/losses/test_perceptual_lpips.py @@ -0,0 +1,102 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import unittest +from pathlib import Path +from unittest.mock import patch + +import torch +from parameterized import parameterized + +from monai.losses import PerceptualLoss +from monai.networks.nets import LPIPS +from monai.utils import optional_import +from tests.test_utils import dict_product, skip_if_quick + +torchvision, has_torchvision = optional_import("torchvision") + + +TEST_NETWORKS = [("alex", "alexnet"), ("vgg", "vgg16"), ("squeeze", "squeezenet1_1")] +TEST_INPUTS = [(params,) for params in dict_product(network=TEST_NETWORKS, channels=[1, 3])] + + +class TestPerceptualLPIPSValidation(unittest.TestCase): + def test_unsupported_network(self): + with self.assertRaisesRegex(ValueError, "Unsupported LPIPS network"): + LPIPS(net="unsupported") + + +@unittest.skipUnless(has_torchvision, "Requires torchvision") +@skip_if_quick +class TestPerceptualLPIPS(unittest.TestCase): + def setUp(self): + # Use real feature networks, replacing only external ImageNet downloads. + torch.manual_seed(9284) + for _, factory_name in TEST_NETWORKS: + factory = getattr(torchvision.models, factory_name) + patcher = patch.object( + torchvision.models, factory_name, side_effect=lambda factory=factory, **kwargs: factory(weights=None) + ) + self.addCleanup(patcher.stop) + patcher.start() + + @parameterized.expand(TEST_INPUTS) + def test_distance(self, params): + loss = PerceptualLoss(spatial_dims=2, network_type=params["network"][0], pretrained=False) + source = torch.rand(2, params["channels"], 64, 64) + target = torch.rand_like(source) + distance = loss.perceptual_function(source, target) + self.assertEqual(distance.shape, (2, 1, 1, 1)) + self.assertTrue(torch.isfinite(distance).all()) + self.assertEqual(loss(source, source).item(), 0.0) + torch.testing.assert_close(distance, loss.perceptual_function(target, source)) + torch.testing.assert_close(loss(source, target), distance.mean()) + + @parameterized.expand(TEST_INPUTS) + def test_input_gradient(self, params): + loss = PerceptualLoss(spatial_dims=2, network_type=params["network"][0], pretrained=False) + source = torch.rand(2, params["channels"], 64, 64, requires_grad=True) + loss(source, torch.rand_like(source)).backward() + self.assertTrue(torch.isfinite(source.grad).all()) + self.assertGreater(source.grad.abs().sum().item(), 0) + + @parameterized.expand(TEST_NETWORKS) + def test_load_legacy_state_dict(self, net, factory_name): + # Independent key/shape manifest captured from unmodified LPIPS v0.1. + manifest_path = Path(__file__).parents[1] / "testing_data" / "lpips_v0_1_state_dict.json" + manifest = json.loads(manifest_path.read_text())["networks"][net] + loss = PerceptualLoss(spatial_dims=2, network_type=net, pretrained=False) + legacy_state = {f"perceptual_function.{key}": torch.full(shape, 0.125) for key, shape in manifest.items()} + loss.load_state_dict(legacy_state, strict=True) + for key, value in loss.state_dict().items(): + torch.testing.assert_close(value, legacy_state[key], rtol=0, atol=0) + + def test_fake_3d_matches_three_axes(self): + loss = PerceptualLoss(spatial_dims=3, network_type="squeeze", fake_3d_ratio=1.0, pretrained=False) + source = torch.rand(1, 1, 32, 32, 32, requires_grad=True) + target = torch.rand_like(source) + expected = sum( + loss.perceptual_function( + source.movedim(axis, 1).reshape(-1, 1, 32, 32), target.movedim(axis, 1).reshape(-1, 1, 32, 32) + ).mean() + for axis in (2, 3, 4) + ) + actual = loss(source, target) + torch.testing.assert_close(actual, expected) + actual.backward() + self.assertTrue(torch.isfinite(source.grad).all()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/min_tests.py b/tests/min_tests.py index 25a42fe4b5d..a7ec441341a 100644 --- a/tests/min_tests.py +++ b/tests/min_tests.py @@ -214,6 +214,8 @@ def run_testsuit(): "test_bundle_workflow", "test_zarr_avg_merger", "test_perceptual_loss", + "test_perceptual", + "test_perceptual_lpips", "test_ultrasound_confidence_map_transform", "test_vista3d_utils", "test_vista3d_transforms", diff --git a/tests/networks/nets/test_perceptual.py b/tests/networks/nets/test_perceptual.py new file mode 100644 index 00000000000..508fb5d1fe2 --- /dev/null +++ b/tests/networks/nets/test_perceptual.py @@ -0,0 +1,121 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import torch +from parameterized import parameterized +from torch import nn + +from monai.losses import PerceptualLoss +from monai.networks.nets import LPIPS, LPIPSFeatures, NetLinLayer, ScalingLayer +from monai.utils import optional_import +from tests.test_utils import dict_product, skip_if_quick + +torchvision, has_torchvision = optional_import("torchvision") + + +TEST_NETWORKS = [("alex", "alexnet"), ("vgg", "vgg16"), ("squeeze", "squeezenet1_1")] +TEST_PRETRAINED = [(params,) for params in dict_product(network=TEST_NETWORKS, pretrained=[True, False])] + + +class TestLPIPSHelpers(unittest.TestCase): + """Check public helpers and the sequential feature data flow.""" + + def test_feature_slices_are_chained(self): + """Later slices must consume preceding features rather than the RGB input.""" + first = nn.Conv2d(3, 4, 1, bias=False) + second = nn.Conv2d(4, 5, 1, bias=False) + with torch.no_grad(): + first.weight.fill_(1.0) + second.weight.fill_(0.5) + model = LPIPSFeatures(nn.Sequential(first, nn.ReLU(), second), (2, 3)) + self.assertIsInstance(model, nn.Sequential) + outputs = model(torch.ones(1, 3, 2, 2)) + torch.testing.assert_close(outputs[0], torch.full((1, 4, 2, 2), 3.0)) + torch.testing.assert_close(outputs[1], torch.full((1, 5, 2, 2), 6.0)) + self.assertTrue(all(not p.requires_grad for p in model.parameters())) + + def test_sequential_calibration_preserves_checkpoint_key(self): + """The Sequential implementation retains upstream model.1.weight keys.""" + model = NetLinLayer(3).eval() + self.assertIsInstance(model, nn.Sequential) + with torch.no_grad(): + model.model[1].weight.fill_(1.0) + torch.testing.assert_close(model(torch.ones(1, 3, 2, 2)), torch.full((1, 1, 2, 2), 3.0)) + self.assertEqual(set(model.state_dict()), {"model.1.weight"}) + + def test_grayscale_scaling(self): + """Grayscale input uses the original three channel scaling constants.""" + output = ScalingLayer()(torch.zeros(1, 1, 2, 2)) + expected = torch.tensor([0.0655021834, 0.1964285714, 0.4177777778]).view(1, 3, 1, 1) + torch.testing.assert_close(output, expected.expand(1, 3, 2, 2)) + + +@unittest.skipUnless(has_torchvision, "Requires torchvision") +@skip_if_quick +class TestLPIPSPretrained(unittest.TestCase): + @parameterized.expand(TEST_PRETRAINED) + def test_calibration_checkpoint(self, params): + net, factory_name = params["network"] + factory = getattr(torchvision.models, factory_name) + expected = {"lin0.model.1.weight": torch.full((1, 64, 1, 1), 0.125)} + with TemporaryDirectory() as directory: + checkpoint = Path(directory) / f"lpips_v0.1_{net}.pth" + + def save_checkpoint(url, filepath, **kwargs): + self.assertEqual(filepath, checkpoint) + torch.save(expected, filepath) + + with ( + patch("monai.networks.nets.perceptual.LPIPS_CHECKPOINTS_DIR", Path(directory)), + patch("monai.networks.nets.perceptual.download_url", side_effect=save_checkpoint), + patch.object(torchvision.models, factory_name, side_effect=lambda **kwargs: factory(weights=None)), + ): + model = LPIPS(net=net, pretrained=params["pretrained"]) + self.assertEqual(checkpoint.is_file(), params["pretrained"]) + self.assertEqual( + torch.equal(model.state_dict()["lin0.model.1.weight"], expected["lin0.model.1.weight"]), + params["pretrained"], + ) + + def test_perceptual_loss_cache_dir(self): + factory = torchvision.models.squeezenet1_1 + previous_cache = torch.hub.get_dir() + self.addCleanup(torch.hub.set_dir, previous_cache) + expected = {"lin0.model.1.weight": torch.full((1, 64, 1, 1), 0.125)} + with TemporaryDirectory() as directory: + checkpoint = Path(directory, "checkpoints", "lpips_v0.1_squeeze.pth") + + def save_checkpoint(url, filepath, **kwargs): + self.assertEqual(filepath, checkpoint) + filepath.parent.mkdir(parents=True, exist_ok=True) + torch.save(expected, filepath) + + with ( + patch("monai.networks.nets.perceptual.download_url", side_effect=save_checkpoint), + patch.object(torchvision.models, "squeezenet1_1", side_effect=lambda **kwargs: factory(weights=None)), + self.assertWarnsRegex(UserWarning, "Setting cache_dir"), + ): + loss = PerceptualLoss(spatial_dims=2, network_type="squeeze", cache_dir=directory) + self.assertTrue(checkpoint.is_file()) + torch.testing.assert_close( + loss.perceptual_function.state_dict()["lin0.model.1.weight"], expected["lin0.model.1.weight"] + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/testing_data/lpips_v0_1_state_dict.json b/tests/testing_data/lpips_v0_1_state_dict.json new file mode 100644 index 00000000000..db018e081f9 --- /dev/null +++ b/tests/testing_data/lpips_v0_1_state_dict.json @@ -0,0 +1,638 @@ +{ + "source": "richzhang/PerceptualSimilarity@082bb24f84c091ea94de2867d34c4544f68e0963", + "networks": { + "alex": { + "scaling_layer.shift": [ + 1, + 3, + 1, + 1 + ], + "scaling_layer.scale": [ + 1, + 3, + 1, + 1 + ], + "net.slice1.0.weight": [ + 64, + 3, + 11, + 11 + ], + "net.slice1.0.bias": [ + 64 + ], + "net.slice2.3.weight": [ + 192, + 64, + 5, + 5 + ], + "net.slice2.3.bias": [ + 192 + ], + "net.slice3.6.weight": [ + 384, + 192, + 3, + 3 + ], + "net.slice3.6.bias": [ + 384 + ], + "net.slice4.8.weight": [ + 256, + 384, + 3, + 3 + ], + "net.slice4.8.bias": [ + 256 + ], + "net.slice5.10.weight": [ + 256, + 256, + 3, + 3 + ], + "net.slice5.10.bias": [ + 256 + ], + "lin0.model.1.weight": [ + 1, + 64, + 1, + 1 + ], + "lin1.model.1.weight": [ + 1, + 192, + 1, + 1 + ], + "lin2.model.1.weight": [ + 1, + 384, + 1, + 1 + ], + "lin3.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lin4.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lins.0.model.1.weight": [ + 1, + 64, + 1, + 1 + ], + "lins.1.model.1.weight": [ + 1, + 192, + 1, + 1 + ], + "lins.2.model.1.weight": [ + 1, + 384, + 1, + 1 + ], + "lins.3.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lins.4.model.1.weight": [ + 1, + 256, + 1, + 1 + ] + }, + "vgg": { + "scaling_layer.shift": [ + 1, + 3, + 1, + 1 + ], + "scaling_layer.scale": [ + 1, + 3, + 1, + 1 + ], + "net.slice1.0.weight": [ + 64, + 3, + 3, + 3 + ], + "net.slice1.0.bias": [ + 64 + ], + "net.slice1.2.weight": [ + 64, + 64, + 3, + 3 + ], + "net.slice1.2.bias": [ + 64 + ], + "net.slice2.5.weight": [ + 128, + 64, + 3, + 3 + ], + "net.slice2.5.bias": [ + 128 + ], + "net.slice2.7.weight": [ + 128, + 128, + 3, + 3 + ], + "net.slice2.7.bias": [ + 128 + ], + "net.slice3.10.weight": [ + 256, + 128, + 3, + 3 + ], + "net.slice3.10.bias": [ + 256 + ], + "net.slice3.12.weight": [ + 256, + 256, + 3, + 3 + ], + "net.slice3.12.bias": [ + 256 + ], + "net.slice3.14.weight": [ + 256, + 256, + 3, + 3 + ], + "net.slice3.14.bias": [ + 256 + ], + "net.slice4.17.weight": [ + 512, + 256, + 3, + 3 + ], + "net.slice4.17.bias": [ + 512 + ], + "net.slice4.19.weight": [ + 512, + 512, + 3, + 3 + ], + "net.slice4.19.bias": [ + 512 + ], + "net.slice4.21.weight": [ + 512, + 512, + 3, + 3 + ], + "net.slice4.21.bias": [ + 512 + ], + "net.slice5.24.weight": [ + 512, + 512, + 3, + 3 + ], + "net.slice5.24.bias": [ + 512 + ], + "net.slice5.26.weight": [ + 512, + 512, + 3, + 3 + ], + "net.slice5.26.bias": [ + 512 + ], + "net.slice5.28.weight": [ + 512, + 512, + 3, + 3 + ], + "net.slice5.28.bias": [ + 512 + ], + "lin0.model.1.weight": [ + 1, + 64, + 1, + 1 + ], + "lin1.model.1.weight": [ + 1, + 128, + 1, + 1 + ], + "lin2.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lin3.model.1.weight": [ + 1, + 512, + 1, + 1 + ], + "lin4.model.1.weight": [ + 1, + 512, + 1, + 1 + ], + "lins.0.model.1.weight": [ + 1, + 64, + 1, + 1 + ], + "lins.1.model.1.weight": [ + 1, + 128, + 1, + 1 + ], + "lins.2.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lins.3.model.1.weight": [ + 1, + 512, + 1, + 1 + ], + "lins.4.model.1.weight": [ + 1, + 512, + 1, + 1 + ] + }, + "squeeze": { + "scaling_layer.shift": [ + 1, + 3, + 1, + 1 + ], + "scaling_layer.scale": [ + 1, + 3, + 1, + 1 + ], + "net.slice1.0.weight": [ + 64, + 3, + 3, + 3 + ], + "net.slice1.0.bias": [ + 64 + ], + "net.slice2.3.squeeze.weight": [ + 16, + 64, + 1, + 1 + ], + "net.slice2.3.squeeze.bias": [ + 16 + ], + "net.slice2.3.expand1x1.weight": [ + 64, + 16, + 1, + 1 + ], + "net.slice2.3.expand1x1.bias": [ + 64 + ], + "net.slice2.3.expand3x3.weight": [ + 64, + 16, + 3, + 3 + ], + "net.slice2.3.expand3x3.bias": [ + 64 + ], + "net.slice2.4.squeeze.weight": [ + 16, + 128, + 1, + 1 + ], + "net.slice2.4.squeeze.bias": [ + 16 + ], + "net.slice2.4.expand1x1.weight": [ + 64, + 16, + 1, + 1 + ], + "net.slice2.4.expand1x1.bias": [ + 64 + ], + "net.slice2.4.expand3x3.weight": [ + 64, + 16, + 3, + 3 + ], + "net.slice2.4.expand3x3.bias": [ + 64 + ], + "net.slice3.6.squeeze.weight": [ + 32, + 128, + 1, + 1 + ], + "net.slice3.6.squeeze.bias": [ + 32 + ], + "net.slice3.6.expand1x1.weight": [ + 128, + 32, + 1, + 1 + ], + "net.slice3.6.expand1x1.bias": [ + 128 + ], + "net.slice3.6.expand3x3.weight": [ + 128, + 32, + 3, + 3 + ], + "net.slice3.6.expand3x3.bias": [ + 128 + ], + "net.slice3.7.squeeze.weight": [ + 32, + 256, + 1, + 1 + ], + "net.slice3.7.squeeze.bias": [ + 32 + ], + "net.slice3.7.expand1x1.weight": [ + 128, + 32, + 1, + 1 + ], + "net.slice3.7.expand1x1.bias": [ + 128 + ], + "net.slice3.7.expand3x3.weight": [ + 128, + 32, + 3, + 3 + ], + "net.slice3.7.expand3x3.bias": [ + 128 + ], + "net.slice4.9.squeeze.weight": [ + 48, + 256, + 1, + 1 + ], + "net.slice4.9.squeeze.bias": [ + 48 + ], + "net.slice4.9.expand1x1.weight": [ + 192, + 48, + 1, + 1 + ], + "net.slice4.9.expand1x1.bias": [ + 192 + ], + "net.slice4.9.expand3x3.weight": [ + 192, + 48, + 3, + 3 + ], + "net.slice4.9.expand3x3.bias": [ + 192 + ], + "net.slice5.10.squeeze.weight": [ + 48, + 384, + 1, + 1 + ], + "net.slice5.10.squeeze.bias": [ + 48 + ], + "net.slice5.10.expand1x1.weight": [ + 192, + 48, + 1, + 1 + ], + "net.slice5.10.expand1x1.bias": [ + 192 + ], + "net.slice5.10.expand3x3.weight": [ + 192, + 48, + 3, + 3 + ], + "net.slice5.10.expand3x3.bias": [ + 192 + ], + "net.slice6.11.squeeze.weight": [ + 64, + 384, + 1, + 1 + ], + "net.slice6.11.squeeze.bias": [ + 64 + ], + "net.slice6.11.expand1x1.weight": [ + 256, + 64, + 1, + 1 + ], + "net.slice6.11.expand1x1.bias": [ + 256 + ], + "net.slice6.11.expand3x3.weight": [ + 256, + 64, + 3, + 3 + ], + "net.slice6.11.expand3x3.bias": [ + 256 + ], + "net.slice7.12.squeeze.weight": [ + 64, + 512, + 1, + 1 + ], + "net.slice7.12.squeeze.bias": [ + 64 + ], + "net.slice7.12.expand1x1.weight": [ + 256, + 64, + 1, + 1 + ], + "net.slice7.12.expand1x1.bias": [ + 256 + ], + "net.slice7.12.expand3x3.weight": [ + 256, + 64, + 3, + 3 + ], + "net.slice7.12.expand3x3.bias": [ + 256 + ], + "lin0.model.1.weight": [ + 1, + 64, + 1, + 1 + ], + "lin1.model.1.weight": [ + 1, + 128, + 1, + 1 + ], + "lin2.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lin3.model.1.weight": [ + 1, + 384, + 1, + 1 + ], + "lin4.model.1.weight": [ + 1, + 384, + 1, + 1 + ], + "lin5.model.1.weight": [ + 1, + 512, + 1, + 1 + ], + "lin6.model.1.weight": [ + 1, + 512, + 1, + 1 + ], + "lins.0.model.1.weight": [ + 1, + 64, + 1, + 1 + ], + "lins.1.model.1.weight": [ + 1, + 128, + 1, + 1 + ], + "lins.2.model.1.weight": [ + 1, + 256, + 1, + 1 + ], + "lins.3.model.1.weight": [ + 1, + 384, + 1, + 1 + ], + "lins.4.model.1.weight": [ + 1, + 384, + 1, + 1 + ], + "lins.5.model.1.weight": [ + 1, + 512, + 1, + 1 + ], + "lins.6.model.1.weight": [ + 1, + 512, + 1, + 1 + ] + } + } +}