fix: add explicit UTF-8 encoding to all file I/O to prevent UnicodeDecodeError (#57) (#58)

On systems where the default locale encoding is not UTF-8 (e.g. GBK on
Chinese Windows), open() and Path.read_text() without an explicit
encoding parameter fail with UnicodeDecodeError when reading UTF-8 files.

Add encoding="utf-8" to every text-mode open() and read_text() call
that was missing it across production code, tests, eval runners, and
scripts.

Closes #57

Author: vineethsai7

evals/runners/benchmark_runner.py

@@ -127,7 +127,7 @@ def _find_evaluation_skills(self) -> list[tuple]:
     def _evaluate_skill(self, skill_path: Path, expected_file: Path):
         """Evaluate a single skill."""
         # Load expected results
-        with open(expected_file) as f:
+        with open(expected_file, encoding="utf-8") as f:
             expected = json.load(f)
 
         skill_name = expected.get("skill_name", skill_path.name)
@@ -324,7 +324,7 @@ def main():
     # Save JSON if requested
     if args.output:
         output_data = {"benchmark": asdict(result), "individual_results": runner.results}
-        with open(args.output, "w") as f:
+        with open(args.output, "w", encoding="utf-8") as f:
             json.dump(output_data, f, indent=2)
         print(f"Results saved to: {args.output}")
 

evals/runners/eval_runner.py

@@ -521,7 +521,7 @@ def main():
                     k: v for k, v in comparison_results["with_meta"].items() if k != "eval_results_with_scan"
                 },
             }
-            with open(args.output, "w") as f:
+            with open(args.output, "w", encoding="utf-8") as f:
                 json.dump(output_data, f, indent=2)
             print(f"\nResults saved to: {args.output}")
 
@@ -576,7 +576,7 @@ def main():
     if args.output:
         # Remove non-serializable data
         output_results = {k: v for k, v in results.items() if k != "eval_results_with_scan"}
-        with open(args.output, "w") as f:
+        with open(args.output, "w", encoding="utf-8") as f:
             json.dump(output_results, f, indent=2)
         print(f"\nResults saved to: {args.output}")
 

evals/runners/policy_benchmark.py

@@ -119,7 +119,7 @@ def run_eval_benchmark(scanner: SkillScanner, eval_dir: Path) -> dict:
         if not (skill_dir / "SKILL.md").exists():
             continue
 
-        with open(expected_file) as f:
+        with open(expected_file, encoding="utf-8") as f:
             expected = json.load(f)
 
         skill_name = expected.get("skill_name", skill_dir.name)

evals/runners/update_expected_findings.py

@@ -58,7 +58,7 @@ def load_expected(skill_dir: Path):
     if not expected_file.exists():
         return None
 
-    with open(expected_file) as f:
+    with open(expected_file, encoding="utf-8") as f:
         return json.load(f)
 
 
@@ -182,7 +182,7 @@ def main():
                     )
 
                 # Save updated file
-                with open(expected_file, "w") as f:
+                with open(expected_file, "w", encoding="utf-8") as f:
                     json.dump(existing, f, indent=2)
                 print(f"  [OK] Updated {expected_file}")
                 updates_needed.append(skill_name)

scripts/fp_analysis_collect.py

@@ -122,7 +122,7 @@ def main():
 
     out_path = Path(__file__).parent.parent / ".local_benchmark" / "fp_analysis_collect.json"
     out_path.parent.mkdir(parents=True, exist_ok=True)
-    with open(out_path, "w") as f:
+    with open(out_path, "w", encoding="utf-8") as f:
         json.dump(out, f, indent=2)
 
     print(f"Wrote {out_path}")

scripts/update_brew_formula.py

@@ -241,7 +241,7 @@ def render_formula(
 
 def read_local_version() -> str:
     """Read __version__ from skill_scanner/_version.py."""
-    text = VERSION_PATH.read_text()
+    text = VERSION_PATH.read_text(encoding="utf-8")
     match = re.search(r'__version__\s*=\s*["\']([^"\']+)["\']', text)
     if not match:
         print(f"Could not parse version from {VERSION_PATH}", file=sys.stderr)

skill_scanner/config/config.py

@@ -142,7 +142,7 @@ def from_file(cls, config_file: Path) -> "Config":
         """
         # Load .env file
         if config_file.exists():
-            with open(config_file) as f:
+            with open(config_file, encoding="utf-8") as f:
                 for line in f:
                     line = line.strip()
                     if line and not line.startswith("#") and "=" in line:

skill_scanner/core/scan_policy.py

@@ -470,7 +470,7 @@ def from_yaml(cls, path: str | Path) -> ScanPolicy:
         if not path.exists():
             raise FileNotFoundError(f"Policy file not found: {path}")
 
-        with open(path) as fh:
+        with open(path, encoding="utf-8") as fh:
             raw: dict[str, Any] = yaml.safe_load(fh) or {}
 
         # If this IS the default file, just parse directly
@@ -488,7 +488,7 @@ def from_yaml(cls, path: str | Path) -> ScanPolicy:
     def to_yaml(self, path: str | Path) -> None:
         """Dump the full policy to a YAML file for editing."""
         data = self._to_dict()
-        with open(path, "w") as fh:
+        with open(path, "w", encoding="utf-8") as fh:
             fh.write("# Skill Scanner – Scan Policy\n")
             fh.write("# Customise this file to match your organisation's security bar.\n")
             fh.write("# Only include sections you want to override; omitted sections\n")
@@ -502,7 +502,7 @@ def to_yaml(self, path: str | Path) -> None:
     @classmethod
     def _load_default_raw(cls) -> dict[str, Any]:
         if _DEFAULT_POLICY_PATH.exists():
-            with open(_DEFAULT_POLICY_PATH) as fh:
+            with open(_DEFAULT_POLICY_PATH, encoding="utf-8") as fh:
                 return yaml.safe_load(fh) or {}
         return {}
 

skill_scanner/hooks/pre_commit.py

@@ -92,7 +92,7 @@ def load_config(repo_root: Path) -> dict:
     for config_path in config_paths:
         if config_path.exists():
             try:
-                with open(config_path) as f:
+                with open(config_path, encoding="utf-8") as f:
                     user_config = json.load(f)
                     config.update(user_config)
                     break

tests/test_api_server_config.py

@@ -42,7 +42,7 @@ def _extract_uvicorn_paths_from_file(self, filepath: Path) -> list[tuple[int, st
         Returns list of (line_number, module_path) tuples.
         """
         paths = []
-        content = filepath.read_text()
+        content = filepath.read_text(encoding="utf-8")
 
         # Parse the AST to find uvicorn.run calls
         try:
@@ -142,7 +142,7 @@ class TestModulePathFormat:
     def test_api_server_path_includes_api_subpackage(self):
         """Test that api_server.py delegates to the correct module path."""
         api_server_path = Path(__file__).parent.parent / "skill_scanner" / "api" / "api_server.py"
-        content = api_server_path.read_text()
+        content = api_server_path.read_text(encoding="utf-8")
 
         # api_server.py is a thin wrapper; it should reference the canonical
         # app location: skill_scanner.api.api:app (not skill_scanner.api_server:app)
@@ -157,7 +157,7 @@ def test_api_server_path_includes_api_subpackage(self):
     def test_api_cli_path_includes_api_subpackage(self):
         """Test that api_cli.py path includes 'api' subpackage."""
         api_cli_path = Path(__file__).parent.parent / "skill_scanner" / "api" / "api_cli.py"
-        content = api_cli_path.read_text()
+        content = api_cli_path.read_text(encoding="utf-8")
 
         # The path should be skill_scanner.api.api, not skill_scanner.api
         assert "skill_scanner.api.api" in content, "api_cli.py should use 'skill_scanner.api.api:app' path"
@@ -273,7 +273,7 @@ def test_no_old_skillanalyzer_references(self):
         api_dir = Path(__file__).parent.parent / "skill_scanner" / "api"
 
         for py_file in api_dir.glob("*.py"):
-            content = py_file.read_text()
+            content = py_file.read_text(encoding="utf-8")
 
             # Check for old module references in uvicorn paths
             if "skillanalyzer" in content.lower():
@@ -287,7 +287,7 @@ def test_module_paths_use_skill_scanner_package(self):
         api_dir = Path(__file__).parent.parent / "skill_scanner" / "api"
 
         for py_file in api_dir.glob("*.py"):
-            content = py_file.read_text()
+            content = py_file.read_text(encoding="utf-8")
 
             # Find uvicorn.run calls with string arguments
             uvicorn_pattern = r'uvicorn\.run\(["\']([^"\']+)["\']'