fix: use explicit encodings in cz commit --edit and cz init

commitizen/commands/check.py

@@ -124,7 +124,8 @@ def _get_commit_message(self) -> str | None:
 
         # Get commit message from file (--commit-msg-file)
         return Path(self.commit_msg_file).read_text(
-            encoding=self.config.settings["encoding"]
+            encoding=self.config.settings["commit_encoding"]
+            or self.config.settings["encoding"]
         )
 
     def _get_commits(self) -> list[git.GitCommit]:

commitizen/commands/commit.py

@@ -57,14 +57,23 @@ def __init__(self, config: BaseConfig, arguments: CommitArgs) -> None:
         self.arguments = arguments
         self.backup_file_path = get_backup_file_path()
 
+    def _get_commit_encoding(self) -> str:
+        """Return the encoding of the files that hold the commit message.
+
+        `commit_encoding` falls back to `encoding` when it is not set.
+        """
+        return (
+            self.config.settings["commit_encoding"] or self.config.settings["encoding"]
+        )
+
     def _read_backup_message(self) -> str | None:
         # Check the commit backup file exists
         if not self.backup_file_path.is_file():
             return None
 
         # Read commit message from backup
         return self.backup_file_path.read_text(
-            encoding=self.config.settings["encoding"]
+            encoding=self._get_commit_encoding()
         ).strip()
 
     def _get_message_by_prompt_commit_questions(self) -> str:
@@ -130,18 +139,30 @@ def _wrap_body(self, message: str) -> str:
         return "\n".join(chain(lines[:2], wrapped_body_lines))
 
     def manual_edit(self, message: str) -> str:
+        """Open the message in the git editor and return the edited message.
+
+        The temporary file uses `commit_encoding` (or `encoding`) instead of the
+        locale encoding. On Windows, the locale encoding is a legacy code page
+        that cannot encode all characters.
+
+        Raises:
+            RuntimeError: If no editor is set or the editor is not found.
+        """
         editor = git.get_core_editor()
         if editor is None:
             raise RuntimeError("No 'editor' value given and no default available.")
         exec_path = shutil.which(editor)
         if exec_path is None:
             raise RuntimeError(f"Editor '{editor}' not found.")
-        with tempfile.NamedTemporaryFile(mode="w", delete=False) as file:
+        encoding = self._get_commit_encoding()
+        with tempfile.NamedTemporaryFile(
+            mode="w", delete=False, encoding=encoding
+        ) as file:
             file.write(message)
         file_path = file.name
         argv = [exec_path, file_path]
         subprocess.call(argv)
-        message = Path(file_path).read_text().strip()
+        message = Path(file_path).read_text(encoding=encoding).strip()
         os.unlink(file.name)
         return message
 
@@ -188,7 +209,7 @@ def __call__(self) -> None:
 
         if write_message_to_file:
             with smart_open(
-                write_message_to_file, "w", encoding=self.config.settings["encoding"]
+                write_message_to_file, "w", encoding=self._get_commit_encoding()
             ) as file:
                 file.write(commit_message)
 
@@ -204,7 +225,7 @@ def __call__(self) -> None:
 
             # Create commit backup
             with smart_open(
-                self.backup_file_path, "w", encoding=self.config.settings["encoding"]
+                self.backup_file_path, "w", encoding=self._get_commit_encoding()
             ) as f:
                 f.write(commit_message)
 

commitizen/defaults.py

@@ -41,6 +41,7 @@ class Settings(TypedDict, total=False):
     changelog_incremental: bool
     changelog_merge_prerelease: bool
     changelog_start_rev: str | None
+    commit_encoding: str | None
     customize: CzSettings
     encoding: str
     extras: dict[str, Any]
@@ -111,6 +112,7 @@ class Settings(TypedDict, total=False):
     "post_bump_hooks": [],
     "prerelease_offset": 0,
     "encoding": ENCODING,
+    "commit_encoding": None,  # default to encoding
     "always_signoff": False,
     "template": None,  # default provided by plugin
     "extras": {},

commitizen/project_info.py

@@ -27,7 +27,8 @@ def get_default_version_provider() -> Literal[
 ]:
     pyproject_path = Path("pyproject.toml")
     if pyproject_path.is_file():
-        if "[tool.poetry]" in pyproject_path.read_text():
+        # TOML files are UTF-8 by specification, whatever the locale encoding
+        if "[tool.poetry]" in pyproject_path.read_text(encoding="utf-8"):
             return "poetry"
         if Path("uv.lock").is_file():
             return "uv"

docs/config/commit.md

@@ -11,6 +11,20 @@ When true, breaking changes will be also indicated by an exclamation mark in the
 
 When false, breaking changes will be only indicated by `BREAKING CHANGE:` in the footer. See [writing commits](../tutorials/writing_commits.md) for more details.
 
+## `commit_encoding`
+
+- Type: `str`
+- Default: `None`
+
+Sets the character encoding of the files that hold a commit message:
+
+- the file that `cz commit --edit` opens in the editor
+- the file that `cz commit --write-message-to-file` writes
+- the backup file that `cz commit --retry` reads
+- the file that `cz check --commit-msg-file` reads
+
+If it is not set, Commitizen uses [`encoding`](#encoding). Set it when your editor saves commit messages in a different encoding from your other files.
+
 ## `encoding`
 
 - Type: `str`

docs/tutorials/writing_commits.md

@@ -147,6 +147,8 @@ Commitizen supports Unicode characters (including emojis) in commit messages. Th
 
 By default, Commitizen uses `utf-8` encoding. You can configure a different encoding through the `encoding` [configuration option][configuration].
 
+Files that hold a commit message, such as the file that `cz commit --edit` opens in your editor, use the [`commit_encoding`][commit-encoding] option. If it is not set, they use `encoding`.
+
 ## Related Documentation
 
 - [Conventional Commits Specification][conventional_commits]
@@ -157,4 +159,5 @@ By default, Commitizen uses `utf-8` encoding. You can configure a different enco
 [conventional_commits]: https://www.conventionalcommits.org
 [cz_emoji]: ../third-party-plugins/cz-emoji.md
 [configuration]: ../config/commit.md#encoding
+[commit-encoding]: ../config/commit.md#commit_encoding
 [breaking-change-config]: ../config/commit.md#breaking_change_exclamation_in_title

tests/commands/test_check_command.py

@@ -144,6 +144,22 @@ def test_check_conventional_commit(
     success_mock.assert_called_once()
 
 
+def test_check_commit_msg_file_uses_commit_encoding(
+    config, success_mock: MockType, tmp_path
+):
+    """`--commit-msg-file` reads the file with `commit_encoding`."""
+    # Arrange
+    config.settings["commit_encoding"] = "utf-16"
+    commit_msg_file = tmp_path / "temp_commit_file"
+    commit_msg_file.write_text("feat: add 안녕하세요 ✨", encoding="utf-16")
+
+    # Act
+    commands.Check(config=config, arguments={"commit_msg_file": commit_msg_file})()
+
+    # Assert
+    success_mock.assert_called_once()
+
+
 def test_check_command_when_commit_file_not_found(config):
     with pytest.raises(FileNotFoundError):
         commands.Check(config=config, arguments={"commit_msg_file": "no_such_file"})()

tests/commands/test_commit_command.py

@@ -83,6 +83,43 @@ def test_commit_backup_on_failure(
     assert Path(temp_file).exists()
 
 
+@pytest.mark.usefixtures("staging_is_clean", "prompt_mock_feat")
+def test_commit_backup_uses_commit_encoding(config, mocker: MockFixture):
+    """A failed commit writes the backup file with `commit_encoding`."""
+    # Arrange
+    config.settings["commit_encoding"] = "utf-16"
+    mocker.patch(
+        "commitizen.git.commit", return_value=cmd.Command("", "error", b"", b"", 9)
+    )
+    mocker.patch("commitizen.out.error")
+    commit_cmd = commands.Commit(config, {})
+
+    # Act
+    with pytest.raises(CommitError):
+        commit_cmd()
+
+    # Assert
+    assert (
+        commit_cmd.backup_file_path.read_text(encoding="utf-16")
+        == "feat: user created\n\ncloses #21"
+    )
+
+
+@pytest.mark.usefixtures("staging_is_clean")
+def test_commit_retry_reads_backup_with_commit_encoding(config, commit_mock: MockType):
+    """`cz commit --retry` reads the backup file with `commit_encoding`."""
+    # Arrange
+    config.settings["commit_encoding"] = "utf-16"
+    commit_cmd = commands.Commit(config, {"retry": True})
+    commit_cmd.backup_file_path.write_text("feat: add 안녕하세요 ✨", encoding="utf-16")
+
+    # Act
+    commit_cmd()
+
+    # Assert
+    commit_mock.assert_called_with("feat: add 안녕하세요 ✨", args=[])
+
+
 @pytest.mark.usefixtures("staging_is_clean", "commit_mock")
 def test_commit_retry_fails_no_backup(config):
     with pytest.raises(
@@ -168,6 +205,22 @@ def test_commit_command_with_write_message_to_file_option(
     assert "feat: user created" in tmp_file.read_text()
 
 
+@pytest.mark.usefixtures("staging_is_clean", "commit_mock", "prompt_mock_feat")
+def test_commit_command_with_write_message_to_file_uses_commit_encoding(
+    config, tmp_path
+):
+    """`--write-message-to-file` writes the message with `commit_encoding`."""
+    # Arrange
+    config.settings["commit_encoding"] = "utf-16"
+    tmp_file = tmp_path / "message"
+
+    # Act
+    commands.Commit(config, {"write_message_to_file": tmp_file})()
+
+    # Assert
+    assert tmp_file.read_text(encoding="utf-16") == "feat: user created\n\ncloses #21"
+
+
 @pytest.mark.usefixtures("staging_is_clean", "prompt_mock_feat")
 def test_commit_command_with_invalid_write_message_to_file_option(config, tmp_path):
     with pytest.raises(NotAllowed):
@@ -320,6 +373,51 @@ def test_manual_edit(editor, config, mocker: MockFixture, tmp_path):
         assert edited_message == test_message.strip()
 
 
+@pytest.mark.usefixtures("staging_is_clean")
+@pytest.mark.parametrize(
+    ("encoding", "commit_encoding", "file_encoding"),
+    [
+        ("utf-8", None, "utf-8"),
+        ("utf-16", None, "utf-16"),
+        ("utf-8", "utf-16", "utf-16"),
+    ],
+)
+def test_manual_edit_keeps_non_ascii_message(
+    encoding: str,
+    commit_encoding: str | None,
+    file_encoding: str,
+    config,
+    mocker: MockFixture,
+):
+    """The editor file uses `commit_encoding`, or `encoding` if it is not set.
+
+    The locale encoding is not used. On Windows, it is a legacy code page
+    (e.g. cp1252), which cannot encode this message.
+    """
+    # Arrange
+    config.settings["encoding"] = encoding
+    config.settings["commit_encoding"] = commit_encoding
+    message = "feat: add 안녕하세요 greeting ✨"
+    edited_message = "feat: add 안녕하세요 greeting ✨ (edited)"
+    mocker.patch("commitizen.git.get_core_editor", return_value="vim")
+    mocker.patch("shutil.which", return_value="vim")
+
+    def edit_in_editor(argv: list[str]) -> int:
+        message_file = Path(argv[1])
+        assert message_file.read_text(encoding=file_encoding) == message
+        message_file.write_text(edited_message, encoding=file_encoding)
+        return 0
+
+    mocker.patch("subprocess.call", side_effect=edit_in_editor)
+    commit_cmd = commands.Commit(config, {"edit": True})
+
+    # Act
+    result = commit_cmd.manual_edit(message)
+
+    # Assert
+    assert result == edited_message
+
+
 @pytest.mark.usefixtures("staging_is_clean", "prompt_mock_feat")
 @pytest.mark.parametrize(
     "out", ["no changes added to commit", "nothing added to commit"]

tests/test_conf.py

@@ -108,6 +108,7 @@
     "post_bump_hooks": ["scripts/slack_notification.sh"],
     "prerelease_offset": 0,
     "encoding": "utf-8",
+    "commit_encoding": None,
     "always_signoff": False,
     "template": None,
     "extras": {},
@@ -149,6 +150,7 @@
     "post_bump_hooks": ["scripts/slack_notification.sh"],
     "prerelease_offset": 0,
     "encoding": "utf-8",
+    "commit_encoding": None,
     "always_signoff": False,
     "template": None,
     "extras": {},

tests/test_project_info.py

@@ -15,7 +15,7 @@ def _create_project_files(files: dict[str, str | None]) -> None:
         if content is None:
             path.touch()
         else:
-            path.write_text(content)
+            path.write_text(content, encoding="utf-8")
 
 
 @pytest.mark.parametrize(
@@ -48,6 +48,11 @@ def test_available_hook_installers(mocker, which_map, expected):
             {"pyproject.toml": '[tool.poetry]\nname = "test"\nversion = "0.1.0"'},
             "poetry",
         ),
+        (
+            # Non-ASCII text that the cp949 and cp1252 code pages cannot decode
+            {"pyproject.toml": '[tool.poetry]\nname = "test"\nauthors = ["Łukasz ✨"]'},
+            "poetry",
+        ),
         ({"pyproject.toml": "", "uv.lock": ""}, "uv"),
         (
             {"pyproject.toml": '[tool.commitizen]\nversion = "0.1.0"'},