From 2f982f6df28e5ff616427f4d85047490a4aef4fa Mon Sep 17 00:00:00 2001 From: Lorchie Date: Fri, 10 Jul 2026 00:41:31 +0200 Subject: [PATCH] fix(extensions): force UTF-8 worker stdio to prevent deadlock on non-UTF-8 Windows locales The extension worker was launched with text=True but no explicit encoding, so pipes were decoded with the system locale (cp932 on Japanese Windows). The worker emits UTF-8, so a UnicodeDecodeError killed the stderr reader thread, the stderr buffer filled up, and the worker blocked inside tqdm - a silent, permanent hang around 80% progress with 0% CPU. - _build_env(): set PYTHONUTF8=1 so the worker emits UTF-8 regardless of locale - Popen/subprocess.run: use encoding="utf-8", errors="replace" so reader threads can never die on undecodable bytes Affects all non-UTF-8 Windows locales (Japan, China, Korea, ...). Fixes #214 --- api/services/extension_process.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/services/extension_process.py b/api/services/extension_process.py index 60ce6787..672623cb 100644 --- a/api/services/extension_process.py +++ b/api/services/extension_process.py @@ -73,6 +73,10 @@ def _build_env(self) -> dict: env["MODELS_DIR"] = str(MODELS_DIR) env["WORKSPACE_DIR"] = str(WORKSPACE_DIR) env["MODLY_API_DIR"] = str(Path(__file__).parent.parent) + # Force UTF-8 for the worker's stdio regardless of the system locale. + # On non-UTF-8 Windows locales (cp932/cp936/cp949) the default encoding + # would otherwise mismatch our UTF-8 pipe readers and deadlock the worker. + env["PYTHONUTF8"] = "1" if sys.platform == "darwin": env.setdefault("NUMBA_DISABLE_JIT", "1") # Pass the exact model_dir so runner.py doesn't have to re-derive it @@ -109,7 +113,8 @@ def _start(self) -> None: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, + encoding="utf-8", + errors="replace", bufsize=1, env=self._build_env(), ) @@ -172,7 +177,8 @@ def _install_missing_package(self, python: Path, module_name: str, package_name: check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, + encoding="utf-8", + errors="replace", ) except subprocess.CalledProcessError as exc: details = (exc.stderr or exc.stdout or "").strip()