fix: support formats other than utf-8 while decoding text [CM-1078] (#3968)

Signed-off-by: Mouad BANI <mouad-mb@outlook.com>

Author: mbani01

services/apps/git_integration/src/crowdgit/services/maintainer/maintainer_service.py

@@ -38,7 +38,7 @@
 from crowdgit.services.base.base_service import BaseService
 from crowdgit.services.maintainer.bedrock import invoke_bedrock
 from crowdgit.services.maintainer.section_extractor import SectionExtractor
-from crowdgit.services.utils import run_shell_command
+from crowdgit.services.utils import run_shell_command, safe_decode
 from crowdgit.settings import MAINTAINER_RETRY_INTERVAL_DAYS, MAINTAINER_UPDATE_INTERVAL_HOURS
 
 
@@ -140,6 +140,11 @@ class MaintainerService(BaseService):
 
     _section_extractor = SectionExtractor()
 
+    @staticmethod
+    async def _read_text_file(file_path: str) -> str:
+        async with aiofiles.open(file_path, "rb") as f:
+            return safe_decode(await f.read())
+
     def make_role(self, title: str):
         title = title.lower()
         title = (
@@ -554,8 +559,7 @@ async def find_candidate_files(
         for candidate_path in all_paths:
             file_path = os.path.join(repo_path, candidate_path)
             try:
-                async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
-                    content = await f.read()
+                content = await self._read_text_file(file_path)
             except Exception as e:
                 self.logger.warning(f"Failed to read candidate {candidate_path}: {repr(e)}")
                 continue
@@ -638,9 +642,7 @@ async def try_saved_maintainer_file(
             f"Saved maintainer file exists, reading content: '{saved_maintainer_file}'"
         )
         try:
-            async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
-                content = await f.read()
-
+            content = await self._read_text_file(file_path)
             result = await self.analyze_and_build_result(saved_maintainer_file, content)
             cost += result.total_cost
             return result, cost
@@ -784,8 +786,7 @@ def _attach_metadata(result: MaintainerResult) -> MaintainerResult:
                 )
             else:
                 try:
-                    async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
-                        content = await f.read()
+                    content = await self._read_text_file(file_path)
                     result = await self.analyze_and_build_result(ai_file_name, content)
                     total_cost += result.total_cost
                     return _attach_metadata(result)

services/apps/git_integration/src/crowdgit/services/utils.py

@@ -13,7 +13,7 @@
 from crowdgit.logger import logger
 
 
-def _safe_decode(data: bytes) -> str:
+def safe_decode(data: bytes) -> str:
     """
     Safely decode bytes to string, handling various encodings that might be present in git output.
 
@@ -229,7 +229,7 @@ async def run_shell_command(
             async def _run_with_stderr_logging() -> bytes:
                 async def _stream() -> None:
                     async for raw_line in process.stderr:
-                        line = _safe_decode(raw_line).rstrip()
+                        line = safe_decode(raw_line).rstrip()
                         if line:
                             stderr_logger.log(stderr_log_level, line)
                             stderr_lines.append(line)
@@ -240,7 +240,7 @@ async def _stream() -> None:
 
             coro = _run_with_stderr_logging()
             stdout = await (asyncio.wait_for(coro, timeout=timeout) if timeout else coro)
-            stdout_text = _safe_decode(stdout).strip() if stdout else ""
+            stdout_text = safe_decode(stdout).strip() if stdout else ""
             stderr_text = "\n".join(stderr_lines)
         else:
             # Wait for completion with optional timeout
@@ -252,8 +252,8 @@ async def _stream() -> None:
                 stdout, stderr = await process.communicate(input=stdin_input)
 
             # Handle potentially non-UTF-8 encoded output from git commands
-            stdout_text = _safe_decode(stdout).strip() if stdout else ""
-            stderr_text = _safe_decode(stderr).strip() if stderr else ""
+            stdout_text = safe_decode(stdout).strip() if stdout else ""
+            stderr_text = safe_decode(stderr).strip() if stderr else ""
 
         # Check return code
         if process.returncode == 0: