diff --git a/custom_components/pyscript/global_ctx.py b/custom_components/pyscript/global_ctx.py index 0eb5c47..e25a95e 100644 --- a/custom_components/pyscript/global_ctx.py +++ b/custom_components/pyscript/global_ctx.py @@ -45,6 +45,8 @@ def __init__( self.auto_start: bool = False self.module: ModuleType | None = None self.rel_import_path: str = rel_import_path + # the base the old rules used for relative imports, kept as a fallback for now + self.legacy_rel_import_path: str | None = rel_import_path self.source: str = source self.file_path: str | None = None self.mtime: float = mtime @@ -154,12 +156,24 @@ def get_trig_info(self, name: str, trig_args: dict[str, Any]) -> TrigInfo: """Return a new trigger info instance with the given args.""" return TrigInfo(name, trig_args, self) + def relative_import_name(self, target: str) -> str: + """Return the relative import name of a pyscript folder path, from this file's folder.""" + base = self.rel_import_path.removesuffix("/__init__").split("/") + parts = target.split("/") + common = len(os.path.commonprefix([base, parts])) + if common == len(parts): + # the target is our own folder or one above it, so name it from its parent + common -= 1 + return "." * (len(base) - common + 1) + ".".join(parts[common:]) + async def module_import(self, module_name: str, import_level: int) -> ModuleType | None: """Import a pyscript module from the pyscript/modules or apps folder.""" pyscript_dir = Function.hass.config.path(FOLDER) module_path = module_name.replace(".", "/") file_paths = [] + legacy_file_paths = [] + import_name = "." * import_level + module_name def find_first_file(file_paths: list[list[str]]) -> list[str] | None: for ctx_name, path, rel_path in file_paths: @@ -174,22 +188,32 @@ def find_first_file(file_paths: list[list[str]]) -> list[str] | None: if import_level > 0: if self.rel_import_path is None: raise ImportError("attempted relative import with no known parent package") - path = self.rel_import_path - if path.endswith("/__init__"): - path = os.path.dirname(path) - ctx_name = self.name - for _ in range(import_level - 1): - path = os.path.dirname(path) - idx = ctx_name.rfind(".") - if path.find("/") < 0 or idx < 0: - raise ImportError("attempted relative import above parent package") - ctx_name = ctx_name[0:idx] - ctx_name += f".{module_name}" - module_info = [ctx_name, f"{path}/{module_path}.py", path] - path += f"/{module_path}" - file_paths.append([ctx_name, f"{path}/__init__.py", path]) - file_paths.append(module_info) - module_name = ctx_name[ctx_name.find(".") + 1 :] + + def base_path(path: str) -> str: + path = path.removesuffix("/__init__") + for _ in range(import_level - 1): + path = os.path.dirname(path) + if path.find("/") < 0: + raise ImportError("attempted relative import above parent package") + return path + + def candidates(path: str) -> list[list[str]]: + # the context name follows the file's location, so each file has one context, + # and a module's relative imports start from its own folder + ctx_name = f"{path}/{module_path}".replace("/", ".") + return [ + [ctx_name, f"{path}/{module_path}/__init__.py", f"{path}/{module_path}"], + [ctx_name, f"{path}/{module_path}.py", os.path.dirname(f"{path}/{module_path}.py")], + ] + + path = base_path(self.rel_import_path) + file_paths = candidates(path) + try: + legacy_path = base_path(self.legacy_rel_import_path or self.rel_import_path) + except ImportError: + legacy_path = path + if legacy_path != path: + legacy_file_paths = candidates(legacy_path) else: if self.rel_import_path is not None and self.rel_import_path.startswith("apps/"): @@ -201,28 +225,57 @@ def find_first_file(file_paths: list[list[str]]) -> list[str] | None: file_paths.append([ctx_name, f"modules/{module_path}/__init__.py", f"modules/{module_path}"]) file_paths.append([ctx_name, f"modules/{module_path}.py", None]) + def find_loaded(file_paths: list[list[str]]) -> ModuleType | None: + for ctx_name, _, _ in file_paths: + mod_ctx = self.manager.get(ctx_name) + if mod_ctx and mod_ctx.module: + self.imports.add(mod_ctx.get_name()) + return mod_ctx.module + return None + # # now see if we have loaded it already # - for ctx_name, _, _ in file_paths: - mod_ctx = self.manager.get(ctx_name) - if mod_ctx and mod_ctx.module: - self.imports.add(mod_ctx.get_name()) - return mod_ctx.module + if mod := find_loaded(file_paths): + return mod # # not loaded already, so try to find and import it # file_info = await Function.hass.async_add_executor_job(find_first_file, file_paths) + if not file_info and legacy_file_paths: + # + # fall back to the old rules, where a plain module's relative imports started from + # the folder its importer's import started from, not from the module's own folder + # + path = legacy_path # this import starts here with the old rules + mod = find_loaded(legacy_file_paths) + if not mod: + file_info = await Function.hass.async_add_executor_job(find_first_file, legacy_file_paths) + if mod or file_info: + _LOGGER.warning( + "%s: the relative import %s was found only with the old rules, which are deprecated " + "and will be removed in a future release; relative imports now start from the " + "importing file's own folder, as in Python, so write it as %s", + self.name, + import_name, + self.relative_import_name(f"{path}/{module_path}"), + ) + if mod: + return mod if not file_info: return None [ctx_name, file_path, rel_import_path] = file_info + module_name = ctx_name[ctx_name.find(".") + 1 :] mod = ModuleType(module_name) global_ctx = GlobalContext( ctx_name, global_sym_table=mod.__dict__, manager=self.manager, rel_import_path=rel_import_path ) + if import_level > 0 and not file_path.endswith("/__init__.py"): + # with the old rules, a plain module's relative imports started where this one did + global_ctx.legacy_rel_import_path = path global_ctx.set_auto_start(self.auto_start) try: await self.manager.load_file(global_ctx, file_path) diff --git a/tests/test_apps_modules.py b/tests/test_apps_modules.py index 2ccf524..c2b0b3a 100644 --- a/tests/test_apps_modules.py +++ b/tests/test_apps_modules.py @@ -2,6 +2,7 @@ from ast import literal_eval import asyncio +from contextlib import ExitStack import re from unittest.mock import patch @@ -9,6 +10,7 @@ import pytest from custom_components.pyscript.const import DOMAIN, FOLDER +from custom_components.pyscript.global_ctx import GlobalContextMgr from homeassistant.const import EVENT_STATE_CHANGED from homeassistant.setup import async_setup_component @@ -213,3 +215,393 @@ async def state_changed(event): assert "ModuleNotFoundError: import of no_such_package not allowed" in caplog.text assert "SyntaxError: expected ':'" in caplog.text assert 'modules/bad_module.py", line 2' in caplog.text + + +@pytest.mark.asyncio +async def test_app_subfolder_relative_imports(hass, caplog): + """Test relative imports between modules in an app's subfolders, from #726.""" + + conf_dir = hass.config.path(FOLDER) + + file_contents = { + f"{conf_dir}/apps/myapp/__init__.py": """ +from .subfolder.a import foo +from .subfolder.b import bar, baz +from .subfolder.deep.c import qux + +@service(supports_response="only") +def myapp_func(): + return {"done": [foo(), bar(), baz(), qux()]} +""", + f"{conf_dir}/apps/myapp/subfolder/a.py": """ +def foo(): + return "foo" +""", + # + # a plain module in a subfolder importing its sibling + # + f"{conf_dir}/apps/myapp/subfolder/b.py": """ +from .a import foo + +def bar(): + return "bar" + foo() + +def baz(): + return __name__ +""", + # + # a deeper subfolder importing from the parent subfolder + # + f"{conf_dir}/apps/myapp/subfolder/deep/c.py": """ +from ..a import foo +from .. import b + +def qux(): + return foo() + b.bar() +""", + } + + mock_open = MockOpen() + for key, value in file_contents.items(): + mock_open[key].read_data = value + + def isfile_side_effect(arg): + return arg in file_contents + + def glob_side_effect(path, recursive=None, root_dir=None, dir_fd=None, include_hidden=False): + result = [] + path_re = path.replace("*", "[^/]*").replace(".", "\\.") + path_re = path_re.replace("[^/]*[^/]*/", ".*") + for this_path in file_contents: + if re.match(path_re, this_path): + result.append(this_path) + return result + + conf = {"apps": {"myapp": {}}} + + def file_patches(mtime): + stack = ExitStack() + for this_patch in ( + patch("custom_components.pyscript.os.path.isdir", return_value=True), + patch("custom_components.pyscript.glob.iglob", side_effect=glob_side_effect), + patch("custom_components.pyscript.global_ctx.open", mock_open), + patch("custom_components.pyscript.open", mock_open), + patch("homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}), + patch("custom_components.pyscript.watchdog_start", return_value=None), + patch("custom_components.pyscript.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os_path_isfile", side_effect=isfile_side_effect), + ): + stack.enter_context(this_patch) + return stack + + with file_patches(1000): + assert await async_setup_component(hass, "pyscript", {DOMAIN: conf}) + + await hass.async_start() + await hass.async_block_till_done() + + expected = {"done": ["foo", "barfoo", "myapp.subfolder.b", "foobarfoo"]} + + def ctx_names(): + return sorted(name for name, _ in GlobalContextMgr.items()) + + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + # + # no phantom contexts like apps.myapp.subfolder.b.a, so a reload keeps everything + # + names = ctx_names() + assert names == [ + "apps.myapp", + "apps.myapp.subfolder.a", + "apps.myapp.subfolder.b", + "apps.myapp.subfolder.deep.c", + ] + + with file_patches(2000): + await hass.services.async_call("pyscript", "reload", {}, blocking=True) + await hass.async_block_till_done() + + assert ctx_names() == names + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + +@pytest.mark.asyncio +async def test_app_subfolder_legacy_relative_imports(hass, caplog): + """Test that the old form of relative imports in an app's subfolders still works, with a warning.""" + + conf_dir = hass.config.path(FOLDER) + + file_contents = { + f"{conf_dir}/apps/myapp/__init__.py": """ +from .subfolder.a import foo +from .subfolder.b import bar, baz +from .subfolder.deep.c import qux + +@service(supports_response="only") +def myapp_func(): + return {"done": [foo(), bar(), baz(), qux()]} +""", + f"{conf_dir}/apps/myapp/subfolder/a.py": """ +def foo(): + return "foo" +""", + # + # a plain module in a subfolder importing its sibling the old way, relative to + # the folder of the package that imported it + # + f"{conf_dir}/apps/myapp/subfolder/b.py": """ +from .subfolder.a import foo + +def bar(): + return "bar" + foo() + +def baz(): + return __name__ +""", + # + # a deeper subfolder importing from the parent subfolder + # + f"{conf_dir}/apps/myapp/subfolder/deep/c.py": """ +from ..a import foo +from .. import b + +def qux(): + return foo() + b.bar() +""", + } + + mock_open = MockOpen() + for key, value in file_contents.items(): + mock_open[key].read_data = value + + def isfile_side_effect(arg): + return arg in file_contents + + def glob_side_effect(path, recursive=None, root_dir=None, dir_fd=None, include_hidden=False): + result = [] + path_re = path.replace("*", "[^/]*").replace(".", "\\.") + path_re = path_re.replace("[^/]*[^/]*/", ".*") + for this_path in file_contents: + if re.match(path_re, this_path): + result.append(this_path) + return result + + conf = {"apps": {"myapp": {}}} + + def file_patches(mtime): + stack = ExitStack() + for this_patch in ( + patch("custom_components.pyscript.os.path.isdir", return_value=True), + patch("custom_components.pyscript.glob.iglob", side_effect=glob_side_effect), + patch("custom_components.pyscript.global_ctx.open", mock_open), + patch("custom_components.pyscript.open", mock_open), + patch("homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}), + patch("custom_components.pyscript.watchdog_start", return_value=None), + patch("custom_components.pyscript.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os.path.getmtime", return_value=mtime), + patch("custom_components.pyscript.global_ctx.os_path_isfile", side_effect=isfile_side_effect), + ): + stack.enter_context(this_patch) + return stack + + def old_rules_warnings(): + return [rec.getMessage() for rec in caplog.records if "only with the old rules" in rec.getMessage()] + + warning = ( + "apps.myapp.subfolder.b: the relative import .subfolder.a was found only with the old rules, " + "which are deprecated and will be removed in a future release; relative imports now start " + "from the importing file's own folder, as in Python, so write it as .a" + ) + + with file_patches(1000): + assert await async_setup_component(hass, "pyscript", {DOMAIN: conf}) + + await hass.async_start() + await hass.async_block_till_done() + + expected = {"done": ["foo", "barfoo", "myapp.subfolder.b", "foobarfoo"]} + + def ctx_names(): + return sorted(name for name, _ in GlobalContextMgr.items()) + + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + # + # only the old-form import is warned about, and both forms load the same contexts, + # so there are no phantom contexts like apps.myapp.subfolder.b.subfolder.a + # + assert old_rules_warnings() == [warning] + names = ctx_names() + assert names == [ + "apps.myapp", + "apps.myapp.subfolder.a", + "apps.myapp.subfolder.b", + "apps.myapp.subfolder.deep.c", + ] + + caplog.clear() + with file_patches(2000): + await hass.services.async_call("pyscript", "reload", {}, blocking=True) + await hass.async_block_till_done() + + assert ctx_names() == names + assert old_rules_warnings() == [warning] + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == expected + + +@pytest.mark.asyncio +async def test_relative_import_context_names(hass, caplog): + """Test that relative imports name each module's context after its file, with either rules.""" + + conf_dir = hass.config.path(FOLDER) + + file_contents = { + f"{conf_dir}/apps/myapp/__init__.py": """ +from . import a +from .sub import sub_names +from .lib.m import m_names +import other + +@service(supports_response="only") +def myapp_func(): + return {"done": [__name__, a.name(), sub_names(), m_names(), other.names()]} +""", + f"{conf_dir}/apps/myapp/a.py": """ +def name(): + return __name__ +""", + # + # a subpackage, importing from its own folder and the folder above it; it + # loads lib/m.py first, with the old rules relative to apps/myapp + # + f"{conf_dir}/apps/myapp/sub/__init__.py": """ +from .s import name as s_name +from ..a import name as a_name +from ..lib.m import m_names + +def sub_names(): + return [__name__, s_name(), a_name()] +""", + f"{conf_dir}/apps/myapp/sub/s.py": """ +def name(): + return __name__ +""", + # + # imports the old way; lib/p.py is found only with the old rules, and itself + # imports the new way + # + f"{conf_dir}/apps/myapp/lib/m.py": """ +from .lib.n import name as n_name +from .lib.p import p_names + +def m_names(): + return [__name__, n_name(), p_names()] +""", + f"{conf_dir}/apps/myapp/lib/n.py": """ +def name(): + return __name__ +""", + f"{conf_dir}/apps/myapp/lib/p.py": """ +from .n import name as n_name + +def p_names(): + return [__name__, n_name()] +""", + # + # a plain module in the apps folder, imported by name, imports from apps/other/ + # + f"{conf_dir}/apps/other.py": """ +from .y import name as y_name + +def names(): + return [__name__, y_name()] +""", + f"{conf_dir}/apps/other/y.py": """ +def name(): + return __name__ +""", + # + # an import that neither rule finds fails as before, without a warning + # + f"{conf_dir}/apps/badapp/__init__.py": """ +from .sub.b import x +""", + f"{conf_dir}/apps/badapp/sub/b.py": """ +from .missing import x +""", + } + + mock_open = MockOpen() + for key, value in file_contents.items(): + mock_open[key].read_data = value + + def isfile_side_effect(arg): + return arg in file_contents + + def glob_side_effect(path, recursive=None, root_dir=None, dir_fd=None, include_hidden=False): + result = [] + path_re = path.replace("*", "[^/]*").replace(".", "\\.") + path_re = path_re.replace("[^/]*[^/]*/", ".*") + for this_path in file_contents: + if re.match(path_re, this_path): + result.append(this_path) + return result + + conf = {"apps": {"myapp": {}, "badapp": {}}} + + with ( + patch("custom_components.pyscript.os.path.isdir", return_value=True), + patch("custom_components.pyscript.glob.iglob", side_effect=glob_side_effect), + patch("custom_components.pyscript.global_ctx.open", mock_open), + patch("custom_components.pyscript.open", mock_open), + patch("homeassistant.config.load_yaml_config_file", return_value={"pyscript": conf}), + patch("custom_components.pyscript.watchdog_start", return_value=None), + patch("custom_components.pyscript.os.path.getmtime", return_value=1000), + patch("custom_components.pyscript.global_ctx.os.path.getmtime", return_value=1000), + patch("custom_components.pyscript.global_ctx.os_path_isfile", side_effect=isfile_side_effect), + ): + assert await async_setup_component(hass, "pyscript", {DOMAIN: conf}) + + await hass.async_start() + await hass.async_block_till_done() + + assert hass.services.has_service("pyscript", "myapp_func") + ret = await hass.services.async_call("pyscript", "myapp_func", {}, return_response=True, blocking=True) + assert ret == { + "done": [ + "myapp", + "myapp.a", + ["myapp.sub", "myapp.sub.s", "myapp.a"], + ["myapp.lib.m", "myapp.lib.n", ["myapp.lib.p", "myapp.lib.n"]], + ["other", "other.y"], + ] + } + assert sorted(name for name, _ in GlobalContextMgr.items()) == [ + "apps.myapp", + "apps.myapp.a", + "apps.myapp.lib.m", + "apps.myapp.lib.n", + "apps.myapp.lib.p", + "apps.myapp.sub", + "apps.myapp.sub.s", + "apps.other", + "apps.other.y", + ] + + warnings = [rec.getMessage() for rec in caplog.records if "only with the old rules" in rec.getMessage()] + assert len(warnings) == 2 + assert warnings[0].startswith("apps.myapp.lib.m: the relative import .lib.n was found only") + assert warnings[0].endswith("so write it as .n") + assert warnings[1].startswith("apps.myapp.lib.m: the relative import .lib.p was found only") + assert warnings[1].endswith("so write it as .p") + assert "ModuleNotFoundError: import from missing not allowed" in caplog.text