feat(param files): Add support for UTF-16 and UTF-32 parameter files

Author: amilcarlucas

ardupilot_methodic_configurator/annotate_params.py

@@ -43,6 +43,7 @@
     PARAM_NAME_REGEX,
     ParamFileError,
     ParDict,
+    read_param_file_lines,
 )
 
 # URL of the XML file
@@ -461,8 +462,7 @@ def update_parameter_documentation(
             continue
 
         # Read the entire file contents
-        with open(param_file, encoding="utf-8") as file:
-            lines = file.readlines()
+        lines = list(read_param_file_lines(param_file))
 
         update_parameter_documentation_file(
             doc, sort_type, param_default_dict, param_file, lines, delete_documentation_annotations

ardupilot_methodic_configurator/data_model_par_dict.py

@@ -11,7 +11,9 @@
 
 import logging
 import re
-from collections.abc import Callable
+from codecs import BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE
+from collections.abc import Callable, Iterator
+from io import TextIOWrapper
 from math import isfinite as math_isfinite
 from os import path as os_path
 from shutil import get_terminal_size
@@ -41,6 +43,47 @@ class ParamFileError(ValueError):
     """Raised when a .param file contains invalid or malformed data."""
 
 
+def _has_zero_byte_pattern(sample: bytes, stride: int, zero_offsets: tuple[int, ...]) -> bool:
+    """Return whether the sample matches zero-byte offsets for a fixed-width encoding."""
+    usable_length = len(sample) - len(sample) % stride
+    return usable_length >= stride * 2 and all(
+        sample[offset] == 0 for zero_offset in zero_offsets for offset in range(zero_offset, usable_length, stride)
+    )
+
+
+def _detect_param_file_encoding(sample: bytes) -> str:
+    """Detect UTF-8/16/32 using a BOM or the zero-byte pattern of ASCII parameter text."""
+    for byte_order_mark, encoding in (
+        (BOM_UTF32_LE, "utf-32"),
+        (BOM_UTF32_BE, "utf-32"),
+        (BOM_UTF16_LE, "utf-16"),
+        (BOM_UTF16_BE, "utf-16"),
+    ):
+        if sample.startswith(byte_order_mark):
+            return encoding
+
+    for encoding, stride, zero_offsets in (
+        ("utf-32-le", 4, (1, 2, 3)),
+        ("utf-32-be", 4, (0, 1, 2)),
+        ("utf-16-le", 2, (1,)),
+        ("utf-16-be", 2, (0,)),
+    ):
+        if _has_zero_byte_pattern(sample, stride, zero_offsets):
+            return encoding
+
+    return "utf-8-sig"
+
+
+def read_param_file_lines(param_file: str) -> Iterator[str]:
+    """Yield parameter-file lines from UTF-8/16/32 files, with or without BOMs."""
+    with open(param_file, "rb") as binary_handle:
+        sample = binary_handle.read(256)
+        binary_handle.seek(0)
+        encoding = _detect_param_file_encoding(sample)
+        with TextIOWrapper(binary_handle, encoding=encoding) as text_handle:
+            yield from text_handle
+
+
 def validate_param_name(param_name: str) -> tuple[bool, str]:
     """
     Validate parameter name according to ArduPilot standards.
@@ -151,37 +194,36 @@ def load_param_file_into_dict(param_file: str) -> "ParDict":
         """
         parameter_dict = ParDict()
         try:
-            with open(param_file, encoding="utf-8-sig") as f_handle:
-                for i, f_line in enumerate(f_handle, start=1):
-                    original_line = f_line
-                    line = f_line.strip()
-                    comment = None
-                    if not line:
-                        continue  # skip empty lines
-                    if line[0] == "#":
-                        continue  # skip comments
-                    if "#" in line:
-                        line, comment = line.split("#", 1)  # strip trailing comments
-                        comment = comment.strip()
-                    if "," in line:
-                        # parse mission planner style parameter files
-                        parameter, value = line.split(",", 1)
-                    elif " " in line:
-                        # parse mavproxy style parameter files
-                        parameter, value = line.split(" ", 1)
-                    elif "\t" in line:
-                        parameter, value = line.split("\t", 1)
-                    else:
-                        msg = _("Missing parameter-value separator: {line} in {param_file} line {i}").format(
-                            line=line, param_file=param_file, i=i
-                        )
-                        raise ParamFileError(msg)
-                    # Strip whitespace from both parameter name and value immediately after splitting
-                    parameter = parameter.strip()
-                    value = value.strip()
-                    ParDict._validate_parameter(param_file, parameter_dict, i, original_line, comment, parameter, value)
+            for i, f_line in enumerate(read_param_file_lines(param_file), start=1):
+                original_line = f_line
+                line = f_line.strip()
+                comment = None
+                if not line:
+                    continue  # skip empty lines
+                if line[0] == "#":
+                    continue  # skip comments
+                if "#" in line:
+                    line, comment = line.split("#", 1)  # strip trailing comments
+                    comment = comment.strip()
+                if "," in line:
+                    # parse mission planner style parameter files
+                    parameter, value = line.split(",", 1)
+                elif " " in line:
+                    # parse mavproxy style parameter files
+                    parameter, value = line.split(" ", 1)
+                elif "\t" in line:
+                    parameter, value = line.split("\t", 1)
+                else:
+                    msg = _("Missing parameter-value separator: {line} in {param_file} line {i}").format(
+                        line=line, param_file=param_file, i=i
+                    )
+                    raise ParamFileError(msg)
+                # Strip whitespace from both parameter name and value immediately after splitting
+                parameter = parameter.strip()
+                value = value.strip()
+                ParDict._validate_parameter(param_file, parameter_dict, i, original_line, comment, parameter, value)
         except UnicodeDecodeError as exp:
-            msg = _("Fatal error reading {param_file}, file must be UTF-8 encoded: {exp}").format(
+            msg = _("Fatal error reading {param_file}, file must be UTF-8, UTF-16, or UTF-32 encoded: {exp}").format(
                 param_file=param_file, exp=exp
             )
             raise ParamFileError(msg) from exp

ardupilot_methodic_configurator/param_pid_adjustment_update.py

@@ -22,7 +22,13 @@
 import argcomplete
 from argcomplete.completers import DirectoriesCompleter, FilesCompleter
 
-from ardupilot_methodic_configurator.data_model_par_dict import PARAM_NAME_MAX_LEN, PARAM_NAME_REGEX, Par, ParDict
+from ardupilot_methodic_configurator.data_model_par_dict import (
+    PARAM_NAME_MAX_LEN,
+    PARAM_NAME_REGEX,
+    Par,
+    ParDict,
+    read_param_file_lines,
+)
 
 VERSION = "1.1"
 
@@ -114,45 +120,44 @@ def load_param_file_with_content(param_file: str) -> tuple[ParDict, list[str]]:
     parameter_dict = ParDict()
     content = []
     try:
-        with open(param_file, encoding="utf-8-sig") as f_handle:
-            for n, f_line in enumerate(f_handle, start=1):
-                line = f_line.strip()
-                content.append(line)
-                comment = None
-                if not line or line.startswith("#"):
-                    continue
-                if "#" in line:
-                    line, comment = line.split("#", 1)
-                    comment = comment.strip()
-                if "," in line:
-                    parameter, value = line.split(",", 1)
-                elif " " in line:
-                    parameter, value = line.split(" ", 1)
-                elif "\t" in line:
-                    parameter, value = line.split("\t", 1)
-                else:
-                    msg = f"Missing parameter-value separator: {line} in {param_file} line {n}"
-                    raise SystemExit(msg)
-                # Strip whitespace from both parameter name and value immediately after splitting
-                parameter = parameter.strip()
-                value = value.strip()
-                if len(parameter) > PARAM_NAME_MAX_LEN:
-                    msg = f"Too long parameter name: {parameter} in {param_file} line {n}"
-                    raise SystemExit(msg)
-                if not re.match(PARAM_NAME_REGEX, parameter):
-                    msg = f"Invalid characters in parameter name {parameter} in {param_file} line {n}"
-                    raise SystemExit(msg)
-                try:
-                    fvalue = float(value)
-                except ValueError as exc:
-                    msg = f"Invalid parameter value {value} in {param_file} line {n}"
-                    raise SystemExit(msg) from exc
-                if parameter in parameter_dict:
-                    msg = f"Duplicated parameter {parameter} in {param_file} line {n}"
-                    raise SystemExit(msg)
-                parameter_dict[parameter] = Par(fvalue, comment)
+        for n, f_line in enumerate(read_param_file_lines(param_file), start=1):
+            line = f_line.strip()
+            content.append(line)
+            comment = None
+            if not line or line.startswith("#"):
+                continue
+            if "#" in line:
+                line, comment = line.split("#", 1)
+                comment = comment.strip()
+            if "," in line:
+                parameter, value = line.split(",", 1)
+            elif " " in line:
+                parameter, value = line.split(" ", 1)
+            elif "\t" in line:
+                parameter, value = line.split("\t", 1)
+            else:
+                msg = f"Missing parameter-value separator: {line} in {param_file} line {n}"
+                raise SystemExit(msg)
+            # Strip whitespace from both parameter name and value immediately after splitting
+            parameter = parameter.strip()
+            value = value.strip()
+            if len(parameter) > PARAM_NAME_MAX_LEN:
+                msg = f"Too long parameter name: {parameter} in {param_file} line {n}"
+                raise SystemExit(msg)
+            if not re.match(PARAM_NAME_REGEX, parameter):
+                msg = f"Invalid characters in parameter name {parameter} in {param_file} line {n}"
+                raise SystemExit(msg)
+            try:
+                fvalue = float(value)
+            except ValueError as exc:
+                msg = f"Invalid parameter value {value} in {param_file} line {n}"
+                raise SystemExit(msg) from exc
+            if parameter in parameter_dict:
+                msg = f"Duplicated parameter {parameter} in {param_file} line {n}"
+                raise SystemExit(msg)
+            parameter_dict[parameter] = Par(fvalue, comment)
     except UnicodeDecodeError as exp:
-        msg = f"Fatal error reading {param_file}, file must be UTF-8 encoded: {exp}"
+        msg = f"Fatal error reading {param_file}, file must be UTF-8, UTF-16, or UTF-32 encoded: {exp}"
         raise SystemExit(msg) from exp
     return parameter_dict, content
 

tests/test_annotate_params.py

@@ -347,6 +347,44 @@ def test_update_parameter_documentation(self) -> None:
         assert "Code1: Value1" in updated_content
         assert "Code2: Value2" in updated_content
 
+    def test_user_can_annotate_utf16_parameter_file(self) -> None:
+        """
+        User can annotate a UTF-16 parameter file.
+
+        GIVEN: A valid UTF-16 parameter file containing a documented parameter
+        WHEN: Parameter documentation is added
+        THEN: The annotated output is written successfully as UTF-8 text
+        """
+        with open(self.temp_file.name, "wb") as file:
+            file.write("PARAM1,100\r\n".encode("utf-16"))
+
+        update_parameter_documentation(self.doc_dict, self.temp_file.name)
+
+        with open(self.temp_file.name, encoding="utf-8") as file:
+            updated_content = file.read()
+
+        assert "# Param 1" in updated_content
+        assert "PARAM1,100" in updated_content
+
+    def test_user_can_annotate_utf32_parameter_file(self) -> None:
+        """
+        User can annotate a UTF-32 parameter file.
+
+        GIVEN: A valid UTF-32 parameter file containing a documented parameter
+        WHEN: Parameter documentation is added
+        THEN: The annotated output is written successfully as UTF-8 text
+        """
+        with open(self.temp_file.name, "wb") as file:
+            file.write("PARAM1,100\r\n".encode("utf-32"))
+
+        update_parameter_documentation(self.doc_dict, self.temp_file.name)
+
+        with open(self.temp_file.name, encoding="utf-8") as file:
+            updated_content = file.read()
+
+        assert "# Param 1" in updated_content
+        assert "PARAM1,100" in updated_content
+
     def test_update_parameter_documentation_sorting_none(self) -> None:
         # Write some initial content to the temporary file
         # With stray leading and trailing whitespaces