From 625c4301dd01474a3bbc7495bac904e58cfa93c6 Mon Sep 17 00:00:00 2001 From: Sean Stoube <31805425+HoneyBearTech@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:59:38 +0000 Subject: [PATCH 1/2] fix relative imports inside an app's subfolders A module imported from a subfolder (from .subfolder.b import x) got its parent folder as the base for its own relative imports, so a sibling import (from .a import foo) failed, and imports from plain modules were registered under phantom contexts such as apps.myapp.subfolder.b.a that were unloaded on the next reload. Use the module's own folder as the base and resolve a plain module's relative imports against its package. Fixes #726. Signed-off-by: Sean Stoube <31805425+HoneyBearTech@users.noreply.github.com> --- custom_components/pyscript/global_ctx.py | 5 +- tests/test_apps_modules.py | 116 +++++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/custom_components/pyscript/global_ctx.py b/custom_components/pyscript/global_ctx.py index 0eb5c470..0d4ac5b3 100644 --- a/custom_components/pyscript/global_ctx.py +++ b/custom_components/pyscript/global_ctx.py @@ -178,6 +178,9 @@ def find_first_file(file_paths: list[list[str]]) -> list[str] | None: if path.endswith("/__init__"): path = os.path.dirname(path) ctx_name = self.name + if os.path.basename(self.file_path or "") != "__init__.py" and "." in ctx_name: + # a plain module is relative to its parent package, so drop its own name + ctx_name = ctx_name[0 : ctx_name.rfind(".")] for _ in range(import_level - 1): path = os.path.dirname(path) idx = ctx_name.rfind(".") @@ -185,7 +188,7 @@ def find_first_file(file_paths: list[list[str]]) -> list[str] | None: raise ImportError("attempted relative import above parent package") ctx_name = ctx_name[0:idx] ctx_name += f".{module_name}" - module_info = [ctx_name, f"{path}/{module_path}.py", path] + module_info = [ctx_name, f"{path}/{module_path}.py", os.path.dirname(f"{path}/{module_path}.py")] path += f"/{module_path}" file_paths.append([ctx_name, f"{path}/__init__.py", path]) file_paths.append(module_info) diff --git a/tests/test_apps_modules.py b/tests/test_apps_modules.py index 2ccf524d..7114f418 100644 --- a/tests/test_apps_modules.py +++ b/tests/test_apps_modules.py @@ -2,6 +2,7 @@ from ast import literal_eval import asyncio +from contextlib import ExitStack import re from unittest.mock import patch @@ -9,6 +10,7 @@ import pytest from custom_components.pyscript.const import DOMAIN, FOLDER +from custom_components.pyscript.global_ctx import GlobalContextMgr from homeassistant.const import EVENT_STATE_CHANGED from homeassistant.setup import async_setup_component @@ -213,3 +215,117 @@ async def state_changed(event): assert "ModuleNotFoundError: import of no_such_package not allowed" in caplog.text assert "SyntaxError: expected ':'" in caplog.text assert 'modules/bad_module.py", line 2' in caplog.text + + +@pytest.mark.asyncio +async def test_app_subfolder_relative_imports(hass, caplog): + """Test relative imports between modules in an app's subfolders, from #726.""" + + conf_dir = hass.config.path(FOLDER) + + file_contents = { + f"{conf_dir}/apps/myapp/__init__.py": """ +from .subfolder.a import foo +from .subfolder.b import bar, baz +from .subfolder.deep.c import qux + +@service(supports_response="only") +def myapp_func(): + return {"done": [foo(), bar(), baz(), qux()]} +""", + f"{conf_dir}/apps/myapp/subfolder/a.py": """ +def foo(): + return "foo" +""", + # + # a plain module in a subfolder importing its sibling + # + f"{conf_dir}/apps/myapp/subfolder/b.py": """ +from .a import foo + +def bar(): + return "bar" + foo() + +def baz(): + return __name__ +""", + # + # a deeper subfolder importing from the parent subfolder + # + f"{conf_dir}/apps/myapp/subfolder/deep/c.py": """ +from ..a import foo +from .. import b + +def qux(): + return foo() + b.bar() +""", + } + + mock_open = MockOpen() + for key, value in file_contents.items(): + mock_open[key].read_data = value + + def isfile_side_effect(arg): + return arg in file_contents + + def glob_side_effect(path, recursive=None, root_dir=None, dir_fd=None, include_hidden=False): + result = [] + path_re = path.replace("*", "[^/]*").replace(".", "\\.") + path_re = path_re.replace("[^/]*[^/]*/", ".*") + for this_path in file_contents: + if re.match(path_re, this_path): + result.append(this_path) + return result + + conf = {"apps": {"myapp": {}}} + + def file_patches(mtime): + stack = ExitStack() + for this_patch in ( + patch("custom_components.pyscript.os.path.isdir", return_value=True), + patch("custom_components.pyscript.glob.iglob", side_effect=glob_side_effect), + patch("custom_components.pyscript.global_ctx.open", mock_open), + patch("custom_components.pyscript.open", mock_open), + patch("homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}), + patch("custom_components.pyscript.watchdog_start", return_value=None), + patch("custom_components.pyscript.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os_path_isfile", side_effect=isfile_side_effect), + ): + stack.enter_context(this_patch) + return stack + + with file_patches(1000): + assert await async_setup_component(hass, "pyscript", {DOMAIN: conf}) + + await hass.async_start() + await hass.async_block_till_done() + + expected = {"done": ["foo", "barfoo", "myapp.subfolder.b", "foobarfoo"]} + + def ctx_names(): + return sorted(name for name, _ in GlobalContextMgr.items()) + + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + # + # no phantom contexts like apps.myapp.subfolder.b.a, so a reload keeps everything + # + names = ctx_names() + assert names == [ + "apps.myapp", + "apps.myapp.subfolder.a", + "apps.myapp.subfolder.b", + "apps.myapp.subfolder.deep.c", + ] + + with file_patches(2000): + await hass.services.async_call("pyscript", "reload", {}, blocking=True) + await hass.async_block_till_done() + + assert ctx_names() == names + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected From 18ae688ec97858a4c55b244c973d140ad384077b Mon Sep 17 00:00:00 2001 From: Sean Stoube <31805425+HoneyBearTech@users.noreply.github.com> Date: Sat, 21 Nov 2026 19:23:15 +0000 Subject: [PATCH 2/2] keep the old form of relative imports in subfolders working, with a warning Scripts written for the old rules import a subfolder module's siblings relative to the folder of the package that imported it (from .subfolder.a import foo inside subfolder/b.py). Resolve relative imports with the new rules first, and only if no file is found, retry from the base the old rules used, which each module's context now keeps. When that finds the file, log a warning that the old form is deprecated and will be removed in a future release, with the import written the new way. Name each module's context after its file's location, so both routes load the same context and the module's __name__ matches, and no phantom contexts are created. Contexts for imports from packages keep the names they had before; only the phantom names of imports made from plain modules change. Signed-off-by: Sean Stoube <31805425+HoneyBearTech@users.noreply.github.com> --- custom_components/pyscript/global_ctx.py | 98 ++++++-- tests/test_apps_modules.py | 276 +++++++++++++++++++++++ 2 files changed, 350 insertions(+), 24 deletions(-) diff --git a/custom_components/pyscript/global_ctx.py b/custom_components/pyscript/global_ctx.py index 0d4ac5b3..e25a95e2 100644 --- a/custom_components/pyscript/global_ctx.py +++ b/custom_components/pyscript/global_ctx.py @@ -45,6 +45,8 @@ def __init__( self.auto_start: bool = False self.module: ModuleType | None = None self.rel_import_path: str = rel_import_path + # the base the old rules used for relative imports, kept as a fallback for now + self.legacy_rel_import_path: str | None = rel_import_path self.source: str = source self.file_path: str | None = None self.mtime: float = mtime @@ -154,12 +156,24 @@ def get_trig_info(self, name: str, trig_args: dict[str, Any]) -> TrigInfo: """Return a new trigger info instance with the given args.""" return TrigInfo(name, trig_args, self) + def relative_import_name(self, target: str) -> str: + """Return the relative import name of a pyscript folder path, from this file's folder.""" + base = self.rel_import_path.removesuffix("/__init__").split("/") + parts = target.split("/") + common = len(os.path.commonprefix([base, parts])) + if common == len(parts): + # the target is our own folder or one above it, so name it from its parent + common -= 1 + return "." * (len(base) - common + 1) + ".".join(parts[common:]) + async def module_import(self, module_name: str, import_level: int) -> ModuleType | None: """Import a pyscript module from the pyscript/modules or apps folder.""" pyscript_dir = Function.hass.config.path(FOLDER) module_path = module_name.replace(".", "/") file_paths = [] + legacy_file_paths = [] + import_name = "." * import_level + module_name def find_first_file(file_paths: list[list[str]]) -> list[str] | None: for ctx_name, path, rel_path in file_paths: @@ -174,25 +188,32 @@ def find_first_file(file_paths: list[list[str]]) -> list[str] | None: if import_level > 0: if self.rel_import_path is None: raise ImportError("attempted relative import with no known parent package") - path = self.rel_import_path - if path.endswith("/__init__"): - path = os.path.dirname(path) - ctx_name = self.name - if os.path.basename(self.file_path or "") != "__init__.py" and "." in ctx_name: - # a plain module is relative to its parent package, so drop its own name - ctx_name = ctx_name[0 : ctx_name.rfind(".")] - for _ in range(import_level - 1): - path = os.path.dirname(path) - idx = ctx_name.rfind(".") - if path.find("/") < 0 or idx < 0: - raise ImportError("attempted relative import above parent package") - ctx_name = ctx_name[0:idx] - ctx_name += f".{module_name}" - module_info = [ctx_name, f"{path}/{module_path}.py", os.path.dirname(f"{path}/{module_path}.py")] - path += f"/{module_path}" - file_paths.append([ctx_name, f"{path}/__init__.py", path]) - file_paths.append(module_info) - module_name = ctx_name[ctx_name.find(".") + 1 :] + + def base_path(path: str) -> str: + path = path.removesuffix("/__init__") + for _ in range(import_level - 1): + path = os.path.dirname(path) + if path.find("/") < 0: + raise ImportError("attempted relative import above parent package") + return path + + def candidates(path: str) -> list[list[str]]: + # the context name follows the file's location, so each file has one context, + # and a module's relative imports start from its own folder + ctx_name = f"{path}/{module_path}".replace("/", ".") + return [ + [ctx_name, f"{path}/{module_path}/__init__.py", f"{path}/{module_path}"], + [ctx_name, f"{path}/{module_path}.py", os.path.dirname(f"{path}/{module_path}.py")], + ] + + path = base_path(self.rel_import_path) + file_paths = candidates(path) + try: + legacy_path = base_path(self.legacy_rel_import_path or self.rel_import_path) + except ImportError: + legacy_path = path + if legacy_path != path: + legacy_file_paths = candidates(legacy_path) else: if self.rel_import_path is not None and self.rel_import_path.startswith("apps/"): @@ -204,28 +225,57 @@ def find_first_file(file_paths: list[list[str]]) -> list[str] | None: file_paths.append([ctx_name, f"modules/{module_path}/__init__.py", f"modules/{module_path}"]) file_paths.append([ctx_name, f"modules/{module_path}.py", None]) + def find_loaded(file_paths: list[list[str]]) -> ModuleType | None: + for ctx_name, _, _ in file_paths: + mod_ctx = self.manager.get(ctx_name) + if mod_ctx and mod_ctx.module: + self.imports.add(mod_ctx.get_name()) + return mod_ctx.module + return None + # # now see if we have loaded it already # - for ctx_name, _, _ in file_paths: - mod_ctx = self.manager.get(ctx_name) - if mod_ctx and mod_ctx.module: - self.imports.add(mod_ctx.get_name()) - return mod_ctx.module + if mod := find_loaded(file_paths): + return mod # # not loaded already, so try to find and import it # file_info = await Function.hass.async_add_executor_job(find_first_file, file_paths) + if not file_info and legacy_file_paths: + # + # fall back to the old rules, where a plain module's relative imports started from + # the folder its importer's import started from, not from the module's own folder + # + path = legacy_path # this import starts here with the old rules + mod = find_loaded(legacy_file_paths) + if not mod: + file_info = await Function.hass.async_add_executor_job(find_first_file, legacy_file_paths) + if mod or file_info: + _LOGGER.warning( + "%s: the relative import %s was found only with the old rules, which are deprecated " + "and will be removed in a future release; relative imports now start from the " + "importing file's own folder, as in Python, so write it as %s", + self.name, + import_name, + self.relative_import_name(f"{path}/{module_path}"), + ) + if mod: + return mod if not file_info: return None [ctx_name, file_path, rel_import_path] = file_info + module_name = ctx_name[ctx_name.find(".") + 1 :] mod = ModuleType(module_name) global_ctx = GlobalContext( ctx_name, global_sym_table=mod.__dict__, manager=self.manager, rel_import_path=rel_import_path ) + if import_level > 0 and not file_path.endswith("/__init__.py"): + # with the old rules, a plain module's relative imports started where this one did + global_ctx.legacy_rel_import_path = path global_ctx.set_auto_start(self.auto_start) try: await self.manager.load_file(global_ctx, file_path) diff --git a/tests/test_apps_modules.py b/tests/test_apps_modules.py index 7114f418..c2b0b3ab 100644 --- a/tests/test_apps_modules.py +++ b/tests/test_apps_modules.py @@ -329,3 +329,279 @@ def ctx_names(): assert hass.services.has_service("pyscript", "myapp_func") ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) assert ret == expected + + +@pytest.mark.asyncio +async def test_app_subfolder_legacy_relative_imports(hass, caplog): + """Test that the old form of relative imports in an app's subfolders still works, with a warning.""" + + conf_dir = hass.config.path(FOLDER) + + file_contents = { + f"{conf_dir}/apps/myapp/__init__.py": """ +from .subfolder.a import foo +from .subfolder.b import bar, baz +from .subfolder.deep.c import qux + +@service(supports_response="only") +def myapp_func(): + return {"done": [foo(), bar(), baz(), qux()]} +""", + f"{conf_dir}/apps/myapp/subfolder/a.py": """ +def foo(): + return "foo" +""", + # + # a plain module in a subfolder importing its sibling the old way, relative to + # the folder of the package that imported it + # + f"{conf_dir}/apps/myapp/subfolder/b.py": """ +from .subfolder.a import foo + +def bar(): + return "bar" + foo() + +def baz(): + return __name__ +""", + # + # a deeper subfolder importing from the parent subfolder + # + f"{conf_dir}/apps/myapp/subfolder/deep/c.py": """ +from ..a import foo +from .. import b + +def qux(): + return foo() + b.bar() +""", + } + + mock_open = MockOpen() + for key, value in file_contents.items(): + mock_open[key].read_data = value + + def isfile_side_effect(arg): + return arg in file_contents + + def glob_side_effect(path, recursive=None, root_dir=None, dir_fd=None, include_hidden=False): + result = [] + path_re = path.replace("*", "[^/]*").replace(".", "\\.") + path_re = path_re.replace("[^/]*[^/]*/", ".*") + for this_path in file_contents: + if re.match(path_re, this_path): + result.append(this_path) + return result + + conf = {"apps": {"myapp": {}}} + + def file_patches(mtime): + stack = ExitStack() + for this_patch in ( + patch("custom_components.pyscript.os.path.isdir", return_value=True), + patch("custom_components.pyscript.glob.iglob", side_effect=glob_side_effect), + patch("custom_components.pyscript.global_ctx.open", mock_open), + patch("custom_components.pyscript.open", mock_open), + patch("homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}), + patch("custom_components.pyscript.watchdog_start", return_value=None), + patch("custom_components.pyscript.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os_path_isfile", side_effect=isfile_side_effect), + ): + stack.enter_context(this_patch) + return stack + + def old_rules_warnings(): + return [rec.getMessage() for rec in caplog.records if "only with the old rules" in rec.getMessage()] + + warning = ( + "apps.myapp.subfolder.b: the relative import .subfolder.a was found only with the old rules, " + "which are deprecated and will be removed in a future release; relative imports now start " + "from the importing file's own folder, as in Python, so write it as .a" + ) + + with file_patches(1000): + assert await async_setup_component(hass, "pyscript", {DOMAIN: conf}) + + await hass.async_start() + await hass.async_block_till_done() + + expected = {"done": ["foo", "barfoo", "myapp.subfolder.b", "foobarfoo"]} + + def ctx_names(): + return sorted(name for name, _ in GlobalContextMgr.items()) + + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + # + # only the old-form import is warned about, and both forms load the same contexts, + # so there are no phantom contexts like apps.myapp.subfolder.b.subfolder.a + # + assert old_rules_warnings() == [warning] + names = ctx_names() + assert names == [ + "apps.myapp", + "apps.myapp.subfolder.a", + "apps.myapp.subfolder.b", + "apps.myapp.subfolder.deep.c", + ] + + caplog.clear() + with file_patches(2000): + await hass.services.async_call("pyscript", "reload", {}, blocking=True) + await hass.async_block_till_done() + + assert ctx_names() == names + assert old_rules_warnings() == [warning] + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + +@pytest.mark.asyncio +async def test_relative_import_context_names(hass, caplog): + """Test that relative imports name each module's context after its file, with either rules.""" + + conf_dir = hass.config.path(FOLDER) + + file_contents = { + f"{conf_dir}/apps/myapp/__init__.py": """ +from . import a +from .sub import sub_names +from .lib.m import m_names +import other + +@service(supports_response="only") +def myapp_func(): + return {"done": [__name__, a.name(), sub_names(), m_names(), other.names()]} +""", + f"{conf_dir}/apps/myapp/a.py": """ +def name(): + return __name__ +""", + # + # a subpackage, importing from its own folder and the folder above it; it + # loads lib/m.py first, with the old rules relative to apps/myapp + # + f"{conf_dir}/apps/myapp/sub/__init__.py": """ +from .s import name as s_name +from ..a import name as a_name +from ..lib.m import m_names + +def sub_names(): + return [__name__, s_name(), a_name()] +""", + f"{conf_dir}/apps/myapp/sub/s.py": """ +def name(): + return __name__ +""", + # + # imports the old way; lib/p.py is found only with the old rules, and itself + # imports the new way + # + f"{conf_dir}/apps/myapp/lib/m.py": """ +from .lib.n import name as n_name +from .lib.p import p_names + +def m_names(): + return [__name__, n_name(), p_names()] +""", + f"{conf_dir}/apps/myapp/lib/n.py": """ +def name(): + return __name__ +""", + f"{conf_dir}/apps/myapp/lib/p.py": """ +from .n import name as n_name + +def p_names(): + return [__name__, n_name()] +""", + # + # a plain module in the apps folder, imported by name, imports from apps/other/ + # + f"{conf_dir}/apps/other.py": """ +from .y import name as y_name + +def names(): + return [__name__, y_name()] +""", + f"{conf_dir}/apps/other/y.py": """ +def name(): + return __name__ +""", + # + # an import that neither rule finds fails as before, without a warning + # + f"{conf_dir}/apps/badapp/__init__.py": """ +from .sub.b import x +""", + f"{conf_dir}/apps/badapp/sub/b.py": """ +from .missing import x +""", + } + + mock_open = MockOpen() + for key, value in file_contents.items(): + mock_open[key].read_data = value + + def isfile_side_effect(arg): + return arg in file_contents + + def glob_side_effect(path, recursive=None, root_dir=None, dir_fd=None, include_hidden=False): + result = [] + path_re = path.replace("*", "[^/]*").replace(".", "\\.") + path_re = path_re.replace("[^/]*[^/]*/", ".*") + for this_path in file_contents: + if re.match(path_re, this_path): + result.append(this_path) + return result + + conf = {"apps": {"myapp": {}, "badapp": {}}} + + with ( + patch("custom_components.pyscript.os.path.isdir", return_value=True), + patch("custom_components.pyscript.glob.iglob", side_effect=glob_side_effect), + patch("custom_components.pyscript.global_ctx.open", mock_open), + patch("custom_components.pyscript.open", mock_open), + patch("homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}), + patch("custom_components.pyscript.watchdog_start", return_value=None), + patch("custom_components.pyscript.os.path.getmtime", return_value=1000), + patch("custom_components.pyscript.global_ctx.os.path.getmtime", return_value=1000), + patch("custom_components.pyscript.global_ctx.os_path_isfile", side_effect=isfile_side_effect), + ): + assert await async_setup_component(hass, "pyscript", {DOMAIN: conf}) + + await hass.async_start() + await hass.async_block_till_done() + + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == { + "done": [ + "myapp", + "myapp.a", + ["myapp.sub", "myapp.sub.s", "myapp.a"], + ["myapp.lib.m", "myapp.lib.n", ["myapp.lib.p", "myapp.lib.n"]], + ["other", "other.y"], + ] + } + assert sorted(name for name, _ in GlobalContextMgr.items()) == [ + "apps.myapp", + "apps.myapp.a", + "apps.myapp.lib.m", + "apps.myapp.lib.n", + "apps.myapp.lib.p", + "apps.myapp.sub", + "apps.myapp.sub.s", + "apps.other", + "apps.other.y", + ] + + warnings = [rec.getMessage() for rec in caplog.records if "only with the old rules" in rec.getMessage()] + assert len(warnings) == 2 + assert warnings[0].startswith("apps.myapp.lib.m: the relative import .lib.n was found only") + assert warnings[0].endswith("so write it as .n") + assert warnings[1].startswith("apps.myapp.lib.m: the relative import .lib.p was found only") + assert warnings[1].endswith("so write it as .p") + assert "ModuleNotFoundError: import from missing not allowed" in caplog.text