From 8f35ed3fe7218495dcc8ab6095202e820d3a038f Mon Sep 17 00:00:00 2001 From: "R. Garcia-Dias" Date: Thu, 5 Nov 2026 13:41:55 +0000 Subject: [PATCH 1/6] Do not evaluate imports when reading an algorithm score `BundleAlgo.get_score()` builds a `ConfigParser` from `configs/hyper_parameters.yaml` in the algorithm output directory and evaluates `ckpt_path` with `eval()`. That directory is not always one this process wrote: `import_bundle_algo_history()` walks any work directory, and the caller wraps `get_score()` in a bare `except Exception: pass`, so a work directory from a collaborator, a shared mount or a downloaded archive could run code through a `$` expression and leave no trace. `ckpt_path` is now evaluated with no module globals and empty builtins: the documented string forms, including `$@bundle_root + '/model_fold_0'`, still resolve, while `__import__`, `monai`, `torch` and `numpy` are out of reach. A resolved value that is not a string raises a `ValueError` instead of reaching `os.path.join`, and the history importer logs the failure instead of swallowing it silently. Tests: new `tests/auto3dseg/test_algo_score_config.py` covers a `$` payload, module access, the two supported string forms, and an end-to-end import of a work directory with a malicious config. The three payload tests fail on `dev` (the expression runs) and pass here; `tests/auto3dseg` 41 passed, 1 skipped. Signed-off-by: R. Garcia-Dias --- monai/apps/auto3dseg/bundle_gen.py | 12 ++- monai/apps/auto3dseg/utils.py | 7 +- tests/auto3dseg/test_algo_score_config.py | 108 ++++++++++++++++++++++ 3 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 tests/auto3dseg/test_algo_score_config.py diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index 680de6c99f9..69a6abdb47b 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -360,11 +360,21 @@ def train( def get_score(self, *args, **kwargs): """ Returns validation scores of the model trained by the current Algo. + + The configuration is read from the algorithm output directory, which may belong to a work + directory this process did not create (``import_bundle_algo_history`` reads any work dir). + ``ckpt_path`` is therefore evaluated without imports or builtins, so string forms such as + ``$@bundle_root + '/model_fold_0'`` still work while an expression cannot import a module. """ config_yaml = os.path.join(self.output_path, "configs", "hyper_parameters.yaml") - parser = ConfigParser() + # ConfigParser(globals=False) keeps `monai`, `torch` and `numpy` out of the evaluation namespace; + # the empty builtins mapping removes `__import__`, `eval` and the rest. + parser = ConfigParser(globals=False) + parser.globals["__builtins__"] = {} parser.read_config(config_yaml) ckpt_path = parser.get_parsed_content("ckpt_path", default=self.output_path) + if not isinstance(ckpt_path, str): + raise ValueError(f"Invalid ckpt_path in {config_yaml}: expected a string, got {type(ckpt_path).__name__}.") dict_file = ConfigParser.load_config_file(os.path.join(ckpt_path, "progress.yaml")) # dict_file: a list of scores saved in the form of dict in progress.yaml diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 1946bb718a4..ff8243b1aca 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -11,6 +11,7 @@ from __future__ import annotations +import logging import os from monai.apps.auto3dseg.bundle_gen import BundleAlgo @@ -59,8 +60,10 @@ def import_bundle_algo_history( if best_metric is None: try: best_metric = algo.get_score() - except Exception: - pass + except Exception as e: + # a missing score is normal for an untrained algo; anything else means the directory + # could not be read as one, which is worth surfacing rather than swallowing + logging.warning(f"Could not read the validation score of {name!r} in {output_folder}: {e}") is_trained = best_metric is not None diff --git a/tests/auto3dseg/test_algo_score_config.py b/tests/auto3dseg/test_algo_score_config.py new file mode 100644 index 00000000000..24ae5b0776a --- /dev/null +++ b/tests/auto3dseg/test_algo_score_config.py @@ -0,0 +1,108 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import os +import tempfile +import unittest + +from monai.apps.auto3dseg.bundle_gen import BundleAlgo +from monai.apps.auto3dseg.utils import import_bundle_algo_history + + +class TestAlgoScoreConfig(unittest.TestCase): + """Regression tests for the score of an algorithm output directory. + + ``import_bundle_algo_history`` reads any work directory, so the directory and the config inside it + can come from someone else. Reading the saved score must not evaluate code from that config. + """ + + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.root = self.tempdir.name + self.marker = os.path.join(self.root, "PWNED") + self.algo_dir = os.path.join(self.root, "algo1") + os.makedirs(os.path.join(self.algo_dir, "configs")) + + def tearDown(self): + self.tempdir.cleanup() + + def _write_config(self, ckpt_value: str) -> None: + with open(os.path.join(self.algo_dir, "configs", "hyper_parameters.yaml"), "w") as f: + f.write(f"ckpt_path: {json.dumps(ckpt_value)}\n") + + def _write_progress(self, ckpt_dir: str, score: float) -> None: + os.makedirs(ckpt_dir, exist_ok=True) + with open(os.path.join(ckpt_dir, "progress.yaml"), "w") as f: + f.write(f"- best_avg_dice_score: {score}\n") + + def _algo(self) -> BundleAlgo: + algo = BundleAlgo(template_path=self.root) + algo.output_path = self.algo_dir + return algo + + def test_expression_in_ckpt_path_does_not_run(self): + """A ``$`` expression in ``ckpt_path`` cannot import or call anything.""" + payload = f"$__import__('pathlib').Path({self.marker!r}).write_text('pwned') or {self.algo_dir!r}" + self._write_config(payload) + + with self.assertRaises(RuntimeError): + self._algo().get_score() + self.assertFalse(os.path.exists(self.marker), "the ckpt_path expression was evaluated") + + def test_module_names_are_not_in_scope(self): + """The evaluation namespace has no monai/torch/numpy, even though the default parser provides them.""" + self._write_config("$str(torch.tensor(1))") + + with self.assertRaises(RuntimeError): + self._algo().get_score() + self.assertFalse(os.path.exists(self.marker)) + + def test_string_expression_still_resolves(self): + """The documented string form, for example ``$@bundle_root + '/model_fold_0'``, keeps working.""" + ckpt_dir = os.path.join(self.algo_dir, "model_fold_0") + self._write_progress(ckpt_dir, 0.42) + self._write_config(f"${self.algo_dir!r} + '/model_fold_0'") + + self.assertAlmostEqual(self._algo().get_score(), 0.42) + + def test_plain_ckpt_path_still_resolves(self): + ckpt_dir = os.path.join(self.algo_dir, "ckpt") + self._write_progress(ckpt_dir, 0.75) + self._write_config(ckpt_dir) + + self.assertAlmostEqual(self._algo().get_score(), 0.75) + + def test_history_import_does_not_run_a_config_payload(self): + """End to end: a work directory with a malicious config is imported without executing it.""" + payload = f"$__import__('pathlib').Path({self.marker!r}).write_text('pwned') or {self.algo_dir!r}" + self._write_config(payload) + with open(os.path.join(self.algo_dir, "algo_object.json"), "w") as f: + json.dump( + { + "_target_": "monai.apps.auto3dseg.bundle_gen.BundleAlgo", + "_state_": {"template_path": self.root, "output_path": self.algo_dir}, + }, + f, + ) + + with self.assertLogs(level="WARNING"): + history = import_bundle_algo_history(self.root, only_trained=False) + + self.assertFalse(os.path.exists(self.marker), "the work directory payload was evaluated") + self.assertEqual(len(history), 1) + self.assertFalse(history[0]["is_trained"]) + + +if __name__ == "__main__": + unittest.main() From 19eb89317a12de3b28f3cd4a11ce67d50dd7d241 Mon Sep 17 00:00:00 2001 From: "R. Garcia-Dias" Date: Thu, 5 Nov 2026 14:03:32 +0000 Subject: [PATCH 2/6] Resolve the algorithm score config as data instead of evaluating it Review feedback on the previous commit: `__builtins__ = {}` does not stop `eval`, since `().__class__.__base__.__subclasses__()` still walks to a class whose module exposes builtins, and `get_parsed_content` defaults to `instantiate=True`, so a `_target_` mapping in `ckpt_path` is constructed before any type check runs. `get_score()` no longer calls `eval` at all. `ckpt_path` is resolved by a small dedicated reader that accepts exactly the shapes the bundle templates use: a plain string, `+` concatenation of string literals, and `@references` to other plain-string entries, which covers `$@bundle_root + '/model_fold_0'`. Calls, attribute access, comprehensions and `_target_` mappings are refused with a `ValueError`. No component is instantiated and no expression is executed. Tests: `tests/auto3dseg/test_algo_score_config.py` now covers the `__subclasses__()` escape, a `_target_` mapping, module access, the reference form used by the shipped templates, and the end-to-end import of a malicious work directory. The three payload tests fail on `dev` and pass here; `tests/auto3dseg` 45 passed, 1 skipped. Signed-off-by: R. Garcia-Dias --- monai/apps/auto3dseg/bundle_gen.py | 82 ++++++++++++++++++++--- tests/auto3dseg/test_algo_score_config.py | 72 +++++++++++++------- 2 files changed, 123 insertions(+), 31 deletions(-) diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index 69a6abdb47b..18b8dddbf7f 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -11,6 +11,7 @@ from __future__ import annotations +import ast import importlib import ntpath import os @@ -39,7 +40,9 @@ _run_cmd_torchrun, algo_to_json, ) +from monai.bundle.config_item import ConfigExpression from monai.bundle.config_parser import ConfigParser +from monai.bundle.reference_resolver import ReferenceResolver from monai.config import PathLike from monai.utils import ensure_tuple, look_up_option, run_cmd from monai.utils.enums import AlgoKeys @@ -97,6 +100,71 @@ def _confined_join(root: str, name: str) -> str: return path +def _resolve_path_value(value: Any, parser: ConfigParser, what: str) -> str: + """ + Resolve a config value that must be a plain path string, without evaluating it with ``eval``. + + The value can come from a work directory this process did not write, so only the shapes the bundle + templates use are accepted: a plain string, or a ``$`` expression made of string literals, + ``+`` concatenation and references to other plain-string entries, for example + ``$@bundle_root + '/model'``. Calls, attribute access, subscripts to anything but the reference + table, comprehensions and ``_target_`` mappings are refused. + + Args: + value: the config value, as stored in the file. + parser: the parser that holds the other entries referenced with ``@``. + what: a description of the value, used in the error messages. + + Returns: + the resolved path string. + + Raises: + ValueError: if ``value`` is not a string, or is not one of the accepted shapes. + """ + if not isinstance(value, str): + raise ValueError(f"Invalid {what}: expected a string, got {type(value).__name__}.") + if value.startswith("@"): + return _resolve_path_reference(value[1:], parser, what) + if not value.startswith(ConfigExpression.prefix): + return value + + refs = {ref: _resolve_path_reference(ref, parser, what) for ref in ReferenceResolver.match_refs_pattern(value)} + expression = ReferenceResolver.update_refs_pattern(value, refs) + try: + node = ast.parse(expression[len(ConfigExpression.prefix) :], mode="eval").body + except SyntaxError as e: + raise ValueError(f"Invalid {what} expression: {value!r}.") from e + return _path_expression(node, refs, what) + + +def _resolve_path_reference(ref: str, parser: ConfigParser, what: str) -> str: + """Return the plain-string value of ``@ref``, refusing anything that would need evaluation itself.""" + try: + ref_value = parser.get(ref) + except KeyError as e: + raise ValueError(f"Invalid {what}: reference @{ref} does not exist.") from e + if not isinstance(ref_value, str) or ref_value.startswith(("@", "$")): + raise ValueError(f"Invalid {what}: reference @{ref} is not a plain string.") + return ref_value + + +def _path_expression(node: ast.AST, refs: dict[str, str], what: str) -> str: + """Evaluate the small expression grammar accepted by :func:`_resolve_path_value`.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return _path_expression(node.left, refs, what) + _path_expression(node.right, refs, what) + if ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Name) + and node.value.id == ReferenceResolver._vars + and isinstance(node.slice, ast.Constant) + and isinstance(node.slice.value, str) + ): + return refs[node.slice.value] + raise ValueError(f"Invalid {what}: only string literals, + concatenation and @references are allowed.") + + class BundleAlgo(Algo): """ An algorithm represented by a set of bundle configurations and scripts. @@ -363,18 +431,16 @@ def get_score(self, *args, **kwargs): The configuration is read from the algorithm output directory, which may belong to a work directory this process did not create (``import_bundle_algo_history`` reads any work dir). - ``ckpt_path`` is therefore evaluated without imports or builtins, so string forms such as - ``$@bundle_root + '/model_fold_0'`` still work while an expression cannot import a module. + ``ckpt_path`` is therefore resolved as data rather than evaluated: string literals, + ``+`` concatenation and ``@references`` such as ``$@bundle_root + '/model_fold_0'`` work, + while a call, an attribute, a comprehension or a ``_target_`` mapping is refused. """ config_yaml = os.path.join(self.output_path, "configs", "hyper_parameters.yaml") - # ConfigParser(globals=False) keeps `monai`, `torch` and `numpy` out of the evaluation namespace; - # the empty builtins mapping removes `__import__`, `eval` and the rest. parser = ConfigParser(globals=False) - parser.globals["__builtins__"] = {} parser.read_config(config_yaml) - ckpt_path = parser.get_parsed_content("ckpt_path", default=self.output_path) - if not isinstance(ckpt_path, str): - raise ValueError(f"Invalid ckpt_path in {config_yaml}: expected a string, got {type(ckpt_path).__name__}.") + # resolved as data, never with eval(): the file can come from a work directory this process + # did not write, and a `$` expression there must not be able to call or import anything + ckpt_path = _resolve_path_value(parser.get("ckpt_path", default=self.output_path), parser, "ckpt_path") dict_file = ConfigParser.load_config_file(os.path.join(ckpt_path, "progress.yaml")) # dict_file: a list of scores saved in the form of dict in progress.yaml diff --git a/tests/auto3dseg/test_algo_score_config.py b/tests/auto3dseg/test_algo_score_config.py index 24ae5b0776a..a0413a05827 100644 --- a/tests/auto3dseg/test_algo_score_config.py +++ b/tests/auto3dseg/test_algo_score_config.py @@ -37,9 +37,10 @@ def setUp(self): def tearDown(self): self.tempdir.cleanup() - def _write_config(self, ckpt_value: str) -> None: + def _write_config(self, **values) -> None: with open(os.path.join(self.algo_dir, "configs", "hyper_parameters.yaml"), "w") as f: - f.write(f"ckpt_path: {json.dumps(ckpt_value)}\n") + for key, value in values.items(): + f.write(f"{key}: {json.dumps(value)}\n") def _write_progress(self, ckpt_dir: str, score: float) -> None: os.makedirs(ckpt_dir, exist_ok=True) @@ -51,42 +52,67 @@ def _algo(self) -> BundleAlgo: algo.output_path = self.algo_dir return algo - def test_expression_in_ckpt_path_does_not_run(self): - """A ``$`` expression in ``ckpt_path`` cannot import or call anything.""" - payload = f"$__import__('pathlib').Path({self.marker!r}).write_text('pwned') or {self.algo_dir!r}" - self._write_config(payload) - - with self.assertRaises(RuntimeError): + def _assert_refused(self, **values) -> None: + self._write_config(**values) + with self.assertRaises(ValueError): self._algo().get_score() - self.assertFalse(os.path.exists(self.marker), "the ckpt_path expression was evaluated") + self.assertFalse(os.path.exists(self.marker), "the ckpt_path payload was executed") - def test_module_names_are_not_in_scope(self): - """The evaluation namespace has no monai/torch/numpy, even though the default parser provides them.""" - self._write_config("$str(torch.tensor(1))") + def test_plain_ckpt_path_still_resolves(self): + ckpt_dir = os.path.join(self.algo_dir, "ckpt") + self._write_progress(ckpt_dir, 0.75) + self._write_config(ckpt_path=ckpt_dir) - with self.assertRaises(RuntimeError): - self._algo().get_score() - self.assertFalse(os.path.exists(self.marker)) + self.assertAlmostEqual(self._algo().get_score(), 0.75) - def test_string_expression_still_resolves(self): - """The documented string form, for example ``$@bundle_root + '/model_fold_0'``, keeps working.""" + def test_string_concatenation_still_resolves(self): ckpt_dir = os.path.join(self.algo_dir, "model_fold_0") self._write_progress(ckpt_dir, 0.42) - self._write_config(f"${self.algo_dir!r} + '/model_fold_0'") + self._write_config(ckpt_path=f"${self.algo_dir!r} + '/model_fold_0'") self.assertAlmostEqual(self._algo().get_score(), 0.42) - def test_plain_ckpt_path_still_resolves(self): + def test_reference_expression_still_resolves(self): + """The form used by the shipped templates, ``$@bundle_root + '/model'``, keeps working.""" + ckpt_dir = os.path.join(self.algo_dir, "model") + self._write_progress(ckpt_dir, 0.33) + self._write_config(bundle_root=self.algo_dir, ckpt_path="$@bundle_root + '/model'") + + self.assertAlmostEqual(self._algo().get_score(), 0.33) + + def test_whole_value_reference_still_resolves(self): ckpt_dir = os.path.join(self.algo_dir, "ckpt") - self._write_progress(ckpt_dir, 0.75) - self._write_config(ckpt_dir) + self._write_progress(ckpt_dir, 0.5) + self._write_config(ckpt=ckpt_dir, ckpt_path="@ckpt") - self.assertAlmostEqual(self._algo().get_score(), 0.75) + self.assertAlmostEqual(self._algo().get_score(), 0.5) + + def test_expression_in_ckpt_path_does_not_run(self): + """A ``$`` expression in ``ckpt_path`` cannot import or call anything.""" + payload = f"$__import__('pathlib').Path({self.marker!r}).write_text('pwned') or {self.algo_dir!r}" + + self._assert_refused(ckpt_path=payload) + + def test_module_names_are_not_in_scope(self): + self._assert_refused(ckpt_path="$str(torch.tensor(1))") + + def test_subclasses_escape_is_refused(self): + """The usual ``__subclasses__()`` sandbox escape has no way in: no attribute or call is accepted.""" + payload = ( + f"$[c for c in ().__class__.__base__.__subclasses__() if c.__name__ == 'catch_warnings'][0]()" + f"._module.__builtins__['__import__']('pathlib').Path({self.marker!r}).write_text('pwned')" + ) + + self._assert_refused(ckpt_path=payload) + + def test_target_mapping_is_refused(self): + """A ``_target_`` mapping is not instantiated: ``get_score`` never asks for a component.""" + self._assert_refused(ckpt_path={"_target_": "subprocess.Popen", "args": [["touch", self.marker]]}) def test_history_import_does_not_run_a_config_payload(self): """End to end: a work directory with a malicious config is imported without executing it.""" payload = f"$__import__('pathlib').Path({self.marker!r}).write_text('pwned') or {self.algo_dir!r}" - self._write_config(payload) + self._write_config(ckpt_path=payload) with open(os.path.join(self.algo_dir, "algo_object.json"), "w") as f: json.dump( { From ee55a00c2bf6d900a950fee79efe17497628eaea Mon Sep 17 00:00:00 2001 From: "R. Garcia-Dias" Date: Thu, 5 Nov 2026 17:39:39 +0000 Subject: [PATCH 3/6] Accept str(@fold) and reject unknown reference keys in ckpt_path Review feedback on the score config reader. The `dints` and `swinunetr` templates set `ckpt_path: $@bundle_root + '/model_fold' + str(@fold)`, where `fold` is an integer. The reader refused the integer reference and had no `str()` branch, so those two templates could not resolve their score, and `import_bundle_algo_history` dropped them from the trained set. An integer reference is now accepted, but only when it is the single positional argument of `str()`; concatenating it directly is still refused. A subscript into the reference table whose key was not a matched `@reference` now raises `ValueError` instead of `KeyError`, matching the documented contract. Tests: the two template forms (`$@bundle_root + '/model_fold' + str(@fold)` and the segresnet form), the direct-concatenation refusal, and an unknown reference key. `tests/auto3dseg` 48 passed, 1 skipped. Signed-off-by: R. Garcia-Dias --- monai/apps/auto3dseg/bundle_gen.py | 66 +++++++++++++++++------ tests/auto3dseg/test_algo_score_config.py | 16 ++++++ 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index 18b8dddbf7f..d1fe9814297 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -105,10 +105,10 @@ def _resolve_path_value(value: Any, parser: ConfigParser, what: str) -> str: Resolve a config value that must be a plain path string, without evaluating it with ``eval``. The value can come from a work directory this process did not write, so only the shapes the bundle - templates use are accepted: a plain string, or a ``$`` expression made of string literals, - ``+`` concatenation and references to other plain-string entries, for example - ``$@bundle_root + '/model'``. Calls, attribute access, subscripts to anything but the reference - table, comprehensions and ``_target_`` mappings are refused. + templates use are accepted: a plain string, or a ``$`` expression made of string literals, ``+`` concatenation, ``str(...)`` of a reference, and references to other + entries, for example ``$@bundle_root + '/model_fold' + str(@fold)``. Calls other than ``str`` with one + positional argument, attribute access, subscripts to anything but the reference table, comprehensions + and ``_target_`` mappings are refused. Args: value: the config value, as stored in the file. @@ -124,45 +124,77 @@ def _resolve_path_value(value: Any, parser: ConfigParser, what: str) -> str: if not isinstance(value, str): raise ValueError(f"Invalid {what}: expected a string, got {type(value).__name__}.") if value.startswith("@"): - return _resolve_path_reference(value[1:], parser, what) + resolved = _resolve_path_reference(value[1:], parser, what) + if not isinstance(resolved, str): + raise ValueError(f"Invalid {what}: reference {value!r} is not a plain string.") + return resolved if not value.startswith(ConfigExpression.prefix): return value - refs = {ref: _resolve_path_reference(ref, parser, what) for ref in ReferenceResolver.match_refs_pattern(value)} + refs = { + ref: _resolve_path_reference(ref, parser, what, allow_integer=True) + for ref in ReferenceResolver.match_refs_pattern(value) + } expression = ReferenceResolver.update_refs_pattern(value, refs) try: node = ast.parse(expression[len(ConfigExpression.prefix) :], mode="eval").body except SyntaxError as e: raise ValueError(f"Invalid {what} expression: {value!r}.") from e - return _path_expression(node, refs, what) + resolved = _path_expression(node, refs, what) + if not isinstance(resolved, str): + raise ValueError(f"Invalid {what}: the expression does not resolve to a string.") + return resolved -def _resolve_path_reference(ref: str, parser: ConfigParser, what: str) -> str: - """Return the plain-string value of ``@ref``, refusing anything that would need evaluation itself.""" +def _resolve_path_reference(ref: str, parser: ConfigParser, what: str, allow_integer: bool = False) -> str | int: + """Return the value of ``@ref``, refusing anything that would need evaluation itself. + + A string is returned as is. An integer is only returned when ``allow_integer`` is set, because the + templates use it through ``str()``, as in ``str(@fold)``; it is never concatenated directly. + """ try: ref_value = parser.get(ref) except KeyError as e: raise ValueError(f"Invalid {what}: reference @{ref} does not exist.") from e - if not isinstance(ref_value, str) or ref_value.startswith(("@", "$")): - raise ValueError(f"Invalid {what}: reference @{ref} is not a plain string.") - return ref_value + if isinstance(ref_value, str): + if ref_value.startswith(("@", "$")): + raise ValueError(f"Invalid {what}: reference @{ref} is not a plain value.") + return ref_value + if allow_integer and isinstance(ref_value, int) and not isinstance(ref_value, bool): + return ref_value + raise ValueError(f"Invalid {what}: reference @{ref} is not a plain value.") -def _path_expression(node: ast.AST, refs: dict[str, str], what: str) -> str: +def _path_expression(node: ast.AST, refs: dict[str, str | int], what: str) -> str | int: """Evaluate the small expression grammar accepted by :func:`_resolve_path_value`.""" if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): - return _path_expression(node.left, refs, what) + _path_expression(node.right, refs, what) + left = _path_expression(node.left, refs, what) + right = _path_expression(node.right, refs, what) + if isinstance(left, str) and isinstance(right, str): + return left + right + raise ValueError(f"Invalid {what}: only strings can be concatenated.") + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "str" + and len(node.args) == 1 + and not node.keywords + ): + return str(_path_expression(node.args[0], refs, what)) if ( isinstance(node, ast.Subscript) and isinstance(node.value, ast.Name) and node.value.id == ReferenceResolver._vars and isinstance(node.slice, ast.Constant) and isinstance(node.slice.value, str) + and node.slice.value in refs ): return refs[node.slice.value] - raise ValueError(f"Invalid {what}: only string literals, + concatenation and @references are allowed.") + raise ValueError( + f"Invalid {what}: only string literals, + concatenation, str() of a reference and @references are allowed." + ) class BundleAlgo(Algo): @@ -432,8 +464,8 @@ def get_score(self, *args, **kwargs): The configuration is read from the algorithm output directory, which may belong to a work directory this process did not create (``import_bundle_algo_history`` reads any work dir). ``ckpt_path`` is therefore resolved as data rather than evaluated: string literals, - ``+`` concatenation and ``@references`` such as ``$@bundle_root + '/model_fold_0'`` work, - while a call, an attribute, a comprehension or a ``_target_`` mapping is refused. + ``+`` concatenation and ``@references`` such as ``$@bundle_root + '/model_fold' + str(@fold)`` + work, while any other call, an attribute, a comprehension or a ``_target_`` mapping is refused. """ config_yaml = os.path.join(self.output_path, "configs", "hyper_parameters.yaml") parser = ConfigParser(globals=False) diff --git a/tests/auto3dseg/test_algo_score_config.py b/tests/auto3dseg/test_algo_score_config.py index a0413a05827..49d258ebb81 100644 --- a/tests/auto3dseg/test_algo_score_config.py +++ b/tests/auto3dseg/test_algo_score_config.py @@ -80,6 +80,22 @@ def test_reference_expression_still_resolves(self): self.assertAlmostEqual(self._algo().get_score(), 0.33) + def test_integer_reference_through_str_still_resolves(self): + """The ``dints`` and ``swinunetr`` templates use ``$@bundle_root + '/model_fold' + str(@fold)``.""" + ckpt_dir = os.path.join(self.algo_dir, "model_fold0") + self._write_progress(ckpt_dir, 0.61) + self._write_config(bundle_root=self.algo_dir, fold=0, ckpt_path="$@bundle_root + '/model_fold' + str(@fold)") + + self.assertAlmostEqual(self._algo().get_score(), 0.61) + + def test_integer_reference_cannot_be_concatenated(self): + """An integer reference is only usable through str(); concatenating it directly is refused.""" + self._assert_refused(bundle_root=self.algo_dir, fold=0, ckpt_path="$@bundle_root + @fold") + + def test_unknown_reference_key_is_refused(self): + """A subscript that does not come from a matched @reference raises ValueError, not KeyError.""" + self._assert_refused(ckpt_path="$__local_refs['missing']") + def test_whole_value_reference_still_resolves(self): ckpt_dir = os.path.join(self.algo_dir, "ckpt") self._write_progress(ckpt_dir, 0.5) From 7430ea079d7b9f0b2b6318019e35954115791ebc Mon Sep 17 00:00:00 2001 From: "R. Garcia-Dias" Date: Thu, 5 Nov 2026 17:44:27 +0000 Subject: [PATCH 4/6] Report a missing algorithm config reference precisely Review feedback: ConfigParser.get() swallows KeyError and IndexError and returns the default, so a missing reference was reported as "not a plain value" and the KeyError branch was unreachable. Index the parser directly and catch both exceptions, so an id that does not exist in the file is reported as such. Adds a test for the message; tests/auto3dseg 49 passed, 1 skipped. Signed-off-by: R. Garcia-Dias --- monai/apps/auto3dseg/bundle_gen.py | 4 ++-- tests/auto3dseg/test_algo_score_config.py | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index d1fe9814297..2f568d04be9 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -153,8 +153,8 @@ def _resolve_path_reference(ref: str, parser: ConfigParser, what: str, allow_int templates use it through ``str()``, as in ``str(@fold)``; it is never concatenated directly. """ try: - ref_value = parser.get(ref) - except KeyError as e: + ref_value = parser[ref] + except (KeyError, IndexError) as e: raise ValueError(f"Invalid {what}: reference @{ref} does not exist.") from e if isinstance(ref_value, str): if ref_value.startswith(("@", "$")): diff --git a/tests/auto3dseg/test_algo_score_config.py b/tests/auto3dseg/test_algo_score_config.py index 49d258ebb81..cdff2858b80 100644 --- a/tests/auto3dseg/test_algo_score_config.py +++ b/tests/auto3dseg/test_algo_score_config.py @@ -92,6 +92,12 @@ def test_integer_reference_cannot_be_concatenated(self): """An integer reference is only usable through str(); concatenating it directly is refused.""" self._assert_refused(bundle_root=self.algo_dir, fold=0, ckpt_path="$@bundle_root + @fold") + def test_missing_reference_is_refused(self): + """A reference to an id that is not in the file raises ValueError with the documented message.""" + self._write_config(ckpt_path="$@no_such_entry + '/model'") + with self.assertRaisesRegex(ValueError, "does not exist"): + self._algo().get_score() + def test_unknown_reference_key_is_refused(self): """A subscript that does not come from a matched @reference raises ValueError, not KeyError.""" self._assert_refused(ckpt_path="$__local_refs['missing']") From 47117af9c2fcb21b88e7784dd25065acc2eecb4c Mon Sep 17 00:00:00 2001 From: "R. Garcia-Dias" Date: Fri, 6 Nov 2026 17:33:25 +0000 Subject: [PATCH 5/6] Document the args, return value and errors of get_score Review feedback: add Args, Returns and Raises to the get_score docstring, recording that *args/**kwargs are unused and that a non-plain ckpt_path raises ValueError. No behaviour change; the indexed reference lookup the review also mentions is already in place. Signed-off-by: R. Garcia-Dias --- monai/apps/auto3dseg/bundle_gen.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index 2f568d04be9..78d5dc35664 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -466,6 +466,16 @@ def get_score(self, *args, **kwargs): ``ckpt_path`` is therefore resolved as data rather than evaluated: string literals, ``+`` concatenation and ``@references`` such as ``$@bundle_root + '/model_fold' + str(@fold)`` work, while any other call, an attribute, a comprehension or a ``_target_`` mapping is refused. + + Args: + *args: unused, kept for compatibility with the ``Algo`` interface. + **kwargs: unused, kept for compatibility with the ``Algo`` interface. + + Returns: + the best average Dice score recorded in ``progress.yaml``. + + Raises: + ValueError: if ``ckpt_path`` in the saved configuration is not a plain path string. """ config_yaml = os.path.join(self.output_path, "configs", "hyper_parameters.yaml") parser = ConfigParser(globals=False) From e9997b0b09ab101f8fe13d177e0dbe71e607d67f Mon Sep 17 00:00:00 2001 From: "R. Garcia-Dias" Date: Tue, 17 Nov 2026 13:44:29 +0000 Subject: [PATCH 6/6] Document the path syntax accepted for a saved ckpt_path Review follow-up on the get_score change: - _resolve_path_value now states what the value is for, lists the accepted shapes, and records the consequence of refusing to evaluate: a config that needs os.path.join, an f-string or a reference to another expression is rejected rather than resolved, so a score written that way cannot be read back. It also answers the transitive reference question: references are one level deep, so $@model_folds_dir + '/fold_0' is refused when model_folds_dir is itself an expression. - _path_expression says why it can return an int: a bare reference such as @fold inside str(), never as the path itself. - BundleAlgo and get_score point at that restriction from the class docstring down, so it is not buried in a helper. - Direct tests for _resolve_path_value, _resolve_path_reference and _path_expression, including the expression forms that must be refused, so a regression in the grammar fails without going through get_score. Signed-off-by: R. Garcia-Dias --- monai/apps/auto3dseg/bundle_gen.py | 55 ++++++++++--- tests/auto3dseg/test_algo_score_config.py | 96 ++++++++++++++++++++++- 2 files changed, 137 insertions(+), 14 deletions(-) diff --git a/monai/apps/auto3dseg/bundle_gen.py b/monai/apps/auto3dseg/bundle_gen.py index 78d5dc35664..a76b14ff57d 100644 --- a/monai/apps/auto3dseg/bundle_gen.py +++ b/monai/apps/auto3dseg/bundle_gen.py @@ -102,13 +102,27 @@ def _confined_join(root: str, name: str) -> str: def _resolve_path_value(value: Any, parser: ConfigParser, what: str) -> str: """ - Resolve a config value that must be a plain path string, without evaluating it with ``eval``. + Resolve a config value that must be a path string, without evaluating it with ``eval``. - The value can come from a work directory this process did not write, so only the shapes the bundle - templates use are accepted: a plain string, or a ``$`` expression made of string literals, ``+`` concatenation, ``str(...)`` of a reference, and references to other - entries, for example ``$@bundle_root + '/model_fold' + str(@fold)``. Calls other than ``str`` with one - positional argument, attribute access, subscripts to anything but the reference table, comprehensions - and ``_target_`` mappings are refused. + The config is read from an algorithm output directory, and ``import_bundle_algo_history`` can read one from + a work directory this process did not write. The value is therefore treated as data, and only these shapes + are accepted: + + - a plain string, which is returned unchanged; + - a whole-value reference, ``@name``, where that entry holds a plain string; + - a ``$`` expression built only from string literals, ``+`` concatenation, ``str()`` with one argument, and + ``@name`` references to other entries, for example the ``$@bundle_root + '/model_fold' + str(@fold)`` + used by the shipped templates. + + Everything else raises ``ValueError``: any other call, attribute access, a subscript other than the internal + reference table, a comprehension, a ``_target_`` mapping, and a value that is not a string at all. + + Three consequences are deliberate. A config that relied on a wider expression, for example + ``$os.path.join(@bundle_root, 'model')`` or an f-string, is now refused, so such a path has to be written out + in full. References are resolved one level deep only: if ``model_folds_dir`` is itself + ``$@bundle_root + '/model_folds'`` then ``$@model_folds_dir + '/fold_0'`` is refused, because the referenced + entry holds an expression rather than a plain value. An integer entry is accepted only inside ``str()``, as + in ``str(@fold)``, because a path fragment has to be a string. Args: value: the config value, as stored in the file. @@ -149,7 +163,8 @@ def _resolve_path_value(value: Any, parser: ConfigParser, what: str) -> str: def _resolve_path_reference(ref: str, parser: ConfigParser, what: str, allow_integer: bool = False) -> str | int: """Return the value of ``@ref``, refusing anything that would need evaluation itself. - A string is returned as is. An integer is only returned when ``allow_integer`` is set, because the + A string is returned as is. A reference whose own value starts with ``$`` or ``@`` is refused, so references + are never resolved transitively. An integer is only returned when ``allow_integer`` is set, because the templates use it through ``str()``, as in ``str(@fold)``; it is never concatenated directly. """ try: @@ -166,7 +181,13 @@ def _resolve_path_reference(ref: str, parser: ConfigParser, what: str, allow_int def _path_expression(node: ast.AST, refs: dict[str, str | int], what: str) -> str | int: - """Evaluate the small expression grammar accepted by :func:`_resolve_path_value`.""" + """Evaluate the small expression grammar accepted by :func:`_resolve_path_value`. + + The result is a string for every accepted path fragment. The single exception is a bare reference such as + ``@fold``, which returns the integer held in the config so that a surrounding ``str(...)`` can format it: + an expression like ``$@bundle_root + '/model_fold' + str(@fold)`` visits ``@fold`` here and turns it into + ``'0'``. :func:`_resolve_path_value` rejects a non-string result, so an integer never becomes the path. + """ if isinstance(node, ast.Constant) and isinstance(node.value, str): return node.value if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): @@ -217,6 +238,15 @@ class BundleAlgo(Algo): format, the bundle template may contain placeholders that must be filled using ``fill_template_config`` during ``export_to_disk``. Then created bundle keeps the same file structure as the template. + Note: + ``get_score`` reads ``ckpt_path`` from ``configs/hyper_parameters.yaml`` in the algorithm output directory + as data rather than evaluating it, because that directory can come from a work directory this process did + not write. Only a plain string, a reference to a plain entry, or a ``$`` expression made of string + literals, ``+``, ``str()`` and ``@references`` are accepted, as in + ``$@bundle_root + '/model_fold' + str(@fold)``. A saved config that needs anything else, for example a + call to ``os.path.join`` or a reference to another expression, raises ``ValueError`` instead of being + evaluated, so a score written by a template that builds its path that way cannot be read back. + """ def __init__(self, template_path: PathLike | None = None): @@ -461,11 +491,10 @@ def get_score(self, *args, **kwargs): """ Returns validation scores of the model trained by the current Algo. - The configuration is read from the algorithm output directory, which may belong to a work - directory this process did not create (``import_bundle_algo_history`` reads any work dir). - ``ckpt_path`` is therefore resolved as data rather than evaluated: string literals, - ``+`` concatenation and ``@references`` such as ``$@bundle_root + '/model_fold' + str(@fold)`` - work, while any other call, an attribute, a comprehension or a ``_target_`` mapping is refused. + The configuration is read from the algorithm output directory, which may belong to a work directory this + process did not create (``import_bundle_algo_history`` reads any work dir), so ``ckpt_path`` is resolved + as data rather than evaluated. See the class docstring for the syntax that is accepted, which is a + narrower set than ``eval`` would allow. Args: *args: unused, kept for compatibility with the ``Algo`` interface. diff --git a/tests/auto3dseg/test_algo_score_config.py b/tests/auto3dseg/test_algo_score_config.py index cdff2858b80..cf99f47155b 100644 --- a/tests/auto3dseg/test_algo_score_config.py +++ b/tests/auto3dseg/test_algo_score_config.py @@ -11,13 +11,107 @@ from __future__ import annotations +import ast import json import os import tempfile import unittest -from monai.apps.auto3dseg.bundle_gen import BundleAlgo +from monai.apps.auto3dseg.bundle_gen import BundleAlgo, _path_expression, _resolve_path_reference, _resolve_path_value from monai.apps.auto3dseg.utils import import_bundle_algo_history +from monai.bundle.config_parser import ConfigParser + + +class TestPathValueHelpers(unittest.TestCase): + """Direct tests for the helpers that resolve ``ckpt_path`` as data instead of evaluating it.""" + + def setUp(self): + self.parser = ConfigParser(globals=False) + self.parser.update( + { + "bundle_root": "/work/bundle", + "fold": 0, + "model_folds_dir": "$@bundle_root + '/model_folds'", + "flag": True, + } + ) + + def test_plain_string_is_returned_unchanged(self): + self.assertEqual(_resolve_path_value("/work/ckpt", self.parser, "ckpt_path"), "/work/ckpt") + + def test_concatenation_of_literals(self): + self.assertEqual(_resolve_path_value("$'/work' + '/ckpt'", self.parser, "ckpt_path"), "/work/ckpt") + + def test_shipped_template_expression(self): + """The form used by the shipped templates, with an integer entry through ``str()``.""" + self.assertEqual( + _resolve_path_value("$@bundle_root + '/model_fold' + str(@fold)", self.parser, "ckpt_path"), + "/work/bundle/model_fold0", + ) + + def test_whole_value_reference(self): + self.assertEqual(_resolve_path_value("@bundle_root", self.parser, "ckpt_path"), "/work/bundle") + + def test_integer_reference_needs_str(self): + """An integer entry is not a path fragment, so it is only accepted inside ``str()``.""" + with self.assertRaisesRegex(ValueError, "does not resolve to a string"): + _resolve_path_value("$@fold", self.parser, "ckpt_path") + with self.assertRaisesRegex(ValueError, "only strings can be concatenated"): + _resolve_path_value("$@bundle_root + @fold", self.parser, "ckpt_path") + + def test_reference_to_another_expression_is_refused(self): + """References are resolved one level deep, so chaining an expression through a reference is refused.""" + with self.assertRaisesRegex(ValueError, "not a plain value"): + _resolve_path_value("$@model_folds_dir + '/fold_0'", self.parser, "ckpt_path") + with self.assertRaisesRegex(ValueError, "not a plain value"): + _resolve_path_value("@model_folds_dir", self.parser, "ckpt_path") + + def test_boolean_reference_is_refused(self): + """``bool`` is a subclass of ``int`` in Python but is never a path fragment.""" + with self.assertRaisesRegex(ValueError, "not a plain value"): + _resolve_path_value("$str(@flag)", self.parser, "ckpt_path") + + def test_non_string_value_is_refused(self): + with self.assertRaisesRegex(ValueError, "expected a string"): + _resolve_path_value({"_target_": "os.system"}, self.parser, "ckpt_path") + with self.assertRaisesRegex(ValueError, "expected a string"): + _resolve_path_value(1, self.parser, "ckpt_path") + + def test_missing_reference_is_refused(self): + with self.assertRaisesRegex(ValueError, "does not exist"): + _resolve_path_value("$@no_such_entry + '/x'", self.parser, "ckpt_path") + + def test_unknown_subscript_is_refused(self): + with self.assertRaisesRegex(ValueError, "only string literals"): + _resolve_path_value("$__local_refs['missing']", self.parser, "ckpt_path") + + def test_call_and_attribute_access_are_refused(self): + for value in ("$os.path.join(@bundle_root, 'model')", "$str(@fold).upper()", "$@bundle_root.strip()"): + with self.subTest(value=value), self.assertRaises(ValueError): + _resolve_path_value(value, self.parser, "ckpt_path") + + def test_unparseable_expression_is_refused(self): + with self.assertRaisesRegex(ValueError, "expression"): + _resolve_path_value("$@bundle_root +", self.parser, "ckpt_path") + + def test_reference_helper_allows_integers_only_on_request(self): + self.assertEqual(_resolve_path_reference("fold", self.parser, "ckpt_path", allow_integer=True), 0) + with self.assertRaisesRegex(ValueError, "not a plain value"): + _resolve_path_reference("fold", self.parser, "ckpt_path") + + def test_path_expression_returns_int_for_a_bare_reference(self): + """A bare reference is the only non-string result, so that a surrounding ``str()`` can format it.""" + node = ast.parse("__local_refs['fold']", mode="eval").body + self.assertEqual(_path_expression(node, {"fold": 0}, "ckpt_path"), 0) + node = ast.parse("str(__local_refs['fold'])", mode="eval").body + self.assertEqual(_path_expression(node, {"fold": 3}, "ckpt_path"), "3") + node = ast.parse("'a' + 'b'", mode="eval").body + self.assertEqual(_path_expression(node, {}, "ckpt_path"), "ab") + + def test_path_expression_refuses_other_nodes(self): + for source in ("len('a')", "1 + 2", "[1, 2]", "'abc'[0]"): + with self.subTest(source=source), self.assertRaisesRegex(ValueError, "Invalid ckpt_path"): + _path_expression(ast.parse(source, mode="eval").body, {}, "ckpt_path") class TestAlgoScoreConfig(unittest.TestCase):