backend_filesystem.py

# pylint: disable=too-many-lines
"""
Filesystem operations.

This file is part of ArduPilot Methodic Configurator. https://github.com/ArduPilot/MethodicConfigurator

SPDX-FileCopyrightText: 2024-2026 Amilcar do Carmo Lucas <amilcar.lucas@iav.de>

SPDX-License-Identifier: GPL-3.0-or-later
"""

# from sys import exit as sys_exit
from argparse import ArgumentParser
from logging import debug as logging_debug
from logging import error as logging_error
from logging import exception as logging_exception
from logging import info as logging_info
from logging import warning as logging_warning
from os import getcwd as os_getcwd
from os import listdir as os_listdir
from os import path as os_path
from os import remove as os_remove
from os import rename as os_rename
from os import rmdir as os_rmdir
from pathlib import Path
from platform import system as platform_system
from re import compile as re_compile
from shutil import Error as shutil_Error
from shutil import copy2 as shutil_copy2
from shutil import copytree as shutil_copytree
from shutil import rmtree as shutil_rmtree
from subprocess import SubprocessError, run
from typing import Any
from zipfile import ZipFile

from argcomplete.completers import DirectoriesCompleter

from ardupilot_methodic_configurator import _
from ardupilot_methodic_configurator.annotate_params import (
    PARAM_DEFINITION_XML_FILE,
    format_columns,
    get_fallback_xml_url,
    get_xml_dir,
    get_xml_url,
    load_default_param_file,
    parse_parameter_metadata,
    split_into_lines,
    update_parameter_documentation,
)
from ardupilot_methodic_configurator.backend_filesystem_configuration_steps import ConfigurationSteps
from ardupilot_methodic_configurator.backend_filesystem_program_settings import ProgramSettings
from ardupilot_methodic_configurator.backend_filesystem_vehicle_components import VehicleComponents
from ardupilot_methodic_configurator.data_model_par_dict import MANUAL_OVERRIDE_PREFIX, Par, ParDict, is_within_tolerance

PARAMETER_FILE_REGEXP = r"^\d{2}_.*\.param$"
TOOLTIP_MAX_LENGTH = 105


class LocalFilesystem(VehicleComponents, ConfigurationSteps, ProgramSettings):  # pylint: disable=too-many-public-methods
    """
    A class to manage local filesystem operations for the ArduPilot methodic configurator.

    This class provides methods for initializing and re-initializing the filesystem context,
    reading parameters from files, and handling configuration steps. It is designed to simplify
    the interaction with the local filesystem for managing ArduPilot configuration files.

    Args:
        vehicle_dir (str): The directory path where the vehicle configuration files are stored.
        vehicle_type (str): The type of the vehicle (e.g., "ArduCopter", "Rover").

    """

    def __init__(  # pylint: disable=too-many-arguments, too-many-positional-arguments
        self,
        vehicle_dir: str,
        vehicle_type: str,
        fw_version: str,
        allow_editing_template_files: bool,
        save_component_to_system_templates: bool,
    ) -> None:
        self.file_parameters: dict[str, ParDict] = {}
        VehicleComponents.__init__(self, save_component_to_system_templates)
        ConfigurationSteps.__init__(self, vehicle_dir, vehicle_type)
        ProgramSettings.__init__(self)
        self.vehicle_type = vehicle_type
        self.fw_version = fw_version
        self.allow_editing_template_files = allow_editing_template_files
        self.param_default_dict: ParDict = ParDict()
        self.vehicle_dir = vehicle_dir
        self.doc_dict: dict[str, Any] = {}
        if vehicle_dir is not None:
            self.re_init(vehicle_dir, vehicle_type)

    def re_init(self, vehicle_dir: str, vehicle_type: str, blank_component_data: bool = False) -> None:
        self.vehicle_dir = vehicle_dir
        self.doc_dict = {}

        if not self.load_vehicle_components_json_data(vehicle_dir):
            return

        if blank_component_data:
            self.wipe_component_info()
            if self.vehicle_components_fs.data and "Components" in self.vehicle_components_fs.data:
                self.save_vehicle_components_json_data(self.vehicle_components_fs.data, self.vehicle_dir)

        if not self.fw_version:
            self.fw_version = self.get_fc_fw_version_from_vehicle_components_json()

        if vehicle_type == "":
            vehicle_type = self.get_fc_fw_type_from_vehicle_components_json()
        if vehicle_type == "":
            vehicle_type = "ArduCopter"
            logging_warning(_("Could not detect vehicle type. Defaulting to %s."), vehicle_type)
        self.vehicle_type = vehicle_type

        ConfigurationSteps.re_init(self, vehicle_dir, vehicle_type)

        # Rename parameter files if some new files got added to the vehicle directory
        self.rename_parameter_files()

        # Read intermediate parameters from files
        self.file_parameters = self.read_params_from_files()
        if not self.file_parameters:
            return  # No files intermediate parameters files found, no need to continue, the rest needs them

        fw_version = re_compile(r"[ _-]").split(self.fw_version, 1)[0]
        # Read ArduPilot parameter documentation
        xml_url = get_xml_url(vehicle_type, fw_version)
        fallback_xml_url = get_fallback_xml_url(vehicle_type, fw_version)
        xml_dir = get_xml_dir(vehicle_dir)
        self.doc_dict = parse_parameter_metadata(
            xml_url, xml_dir, PARAM_DEFINITION_XML_FILE, vehicle_type, TOOLTIP_MAX_LENGTH, fallback_xml_url
        )
        self.param_default_dict = load_default_param_file(vehicle_dir)

        # Extend parameter documentation metadata if <parameter_file>.pdef.xml exists
        for filename in self.file_parameters:
            pdef_xml_file = filename.replace(".param", ".pdef.xml")
            if os_path.exists(os_path.join(xml_dir, pdef_xml_file)):
                doc_dict = parse_parameter_metadata("", xml_dir, pdef_xml_file, vehicle_type, TOOLTIP_MAX_LENGTH)
                self.doc_dict.update(doc_dict)

        self.__extend_and_reformat_parameter_documentation_metadata()

    def vehicle_configuration_files_exist(self, vehicle_dir: str) -> bool:
        vehicle_path = Path(vehicle_dir)
        if not (vehicle_path.exists() and vehicle_path.is_dir()):
            return False

        # Use generator expression for better memory efficiency
        files = (f.name for f in vehicle_path.iterdir() if f.is_file())

        if platform_system() == "Windows":
            files = (f.lower() for f in files)

        # Convert to set for O(1) lookup performance and compile pattern once
        file_set = set(files)
        pattern = re_compile(PARAMETER_FILE_REGEXP)

        return self.vehicle_components_fs.json_filename in file_set and any(pattern.match(f) for f in file_set)

    def rename_parameter_files(self) -> None:
        if self.vehicle_dir is None or self.configuration_steps is None:
            return
        # Rename parameter files if some new files got added to the vehicle directory
        for new_filename in self.configuration_steps:
            if "old_filenames" in self.configuration_steps[new_filename]:
                for old_filename in self.configuration_steps[new_filename]["old_filenames"]:
                    if self.vehicle_configuration_file_exists(old_filename) and old_filename != new_filename:
                        if self.vehicle_configuration_file_exists(new_filename):
                            logging_error(
                                _("File %s already exists. Will not rename file %s to %s."),
                                new_filename,
                                old_filename,
                                new_filename,
                            )
                            continue
                        new_filename_path = os_path.join(self.vehicle_dir, new_filename)
                        old_filename_path = os_path.join(self.vehicle_dir, old_filename)
                        os_rename(old_filename_path, new_filename_path)
                        logging_info("Renamed %s to %s", old_filename, new_filename)

    def _format_columns_sorted_numerically(  # pylint: disable=too-many-locals
        self, values: dict[str, Any], max_width: int = 105, max_columns: int = 4
    ) -> list[str]:
        """
        Formats a dictionary of values into column-major horizontally aligned columns with numeric sorting.

        This is similar to format_columns from annotate_params.py but sorts the values numerically by key.

        Args:
            values (Dict[str, Any]): The dictionary of values to format.
            max_width (int, optional): The maximum number of characters on all columns. Default is 105.
            max_columns (int): Maximum number of columns

        Returns:
            List[str]: The list of formatted strings.

        """
        if not values:
            return []

        # Sort values numerically by key
        def sort_key(item: tuple[str, Any]) -> tuple[int, int | float | str]:
            key = item[0]
            try:
                return (0, int(key))  # sort integers and floats together
            except ValueError:
                try:
                    return (0, float(key))  # sort integers and floats together
                except ValueError:
                    return (1, str(key))  # Fall back to string sorting and sort them after

        sorted_items = sorted(values.items(), key=sort_key)

        # Format each key-value pair using f-strings
        formatted_items = [f"{key}: {value}" for key, value in sorted_items]

        if not formatted_items:
            return []

        # Calculate optimal column width and number of columns
        max_item_length = max(len(item) for item in formatted_items)
        optimal_columns = min(max_columns, max(1, max_width // (max_item_length + 2)))  # +2 for spacing

        # Ensure we don't exceed max_width with the chosen number of columns
        while optimal_columns > 1 and (max_item_length + 2) * optimal_columns > max_width:
            optimal_columns -= 1

        # Arrange items in column-major order
        num_items = len(formatted_items)
        rows_per_column = (num_items + optimal_columns - 1) // optimal_columns  # Ceiling division

        result = []
        for row in range(rows_per_column):
            row_items = []
            for col in range(optimal_columns):
                index = col * rows_per_column + row
                if index < num_items:
                    row_items.append(formatted_items[index].ljust(max_item_length))
            if row_items:
                result.append("  ".join(row_items).rstrip())

        return result

    def __extend_and_reformat_parameter_documentation_metadata(self) -> None:
        for param_name, param_info in self.doc_dict.items():
            self._process_parameter_fields(param_info)
            self._process_parameter_values(param_name, param_info)
            self._create_parameter_tooltips(param_name, param_info)

    def _process_parameter_fields(self, param_info: dict[str, Any]) -> None:
        """Process and extract parameter fields like Units, Range, etc."""
        if "fields" not in param_info:
            return

        param_fields = param_info["fields"]

        # Process Units
        if "Units" in param_fields:
            units_list = param_fields["Units"].split("(")
            param_info["unit"] = units_list[0].strip()
            if len(units_list) > 1:
                param_info["unit_tooltip"] = units_list[1].strip(")").strip()

        # Process Range
        if "Range" in param_fields:
            range_parts = param_fields["Range"].split(" ")
            if len(range_parts) >= 2:
                param_info["min"] = float(range_parts[0].strip())
                param_info["max"] = float(range_parts[1].strip())

        # Process boolean fields using dict comprehension for better performance
        boolean_fields = ["Calibration", "ReadOnly", "RebootRequired"]
        param_info.update(
            {
                field_name: self.str_to_bool(param_fields[field_name].strip())
                for field_name in boolean_fields
                if field_name in param_fields
            }
        )

        # Process Bitmask using dict comprehension for better performance
        if "Bitmask" in param_fields:
            bitmask_items = param_fields["Bitmask"].split(",")
            param_info["Bitmask"] = {
                int(key.strip()): value.strip() for item in bitmask_items if ":" in item for key, value in [item.split(":", 1)]
            }

    def _process_parameter_values(self, param_name: str, param_info: dict[str, Any]) -> None:
        """Process and convert parameter values to appropriate numeric types."""
        if not param_info.get("values"):
            return

        try:
            param_info["Values"] = {int(k): v for k, v in param_info["values"].items()}
        except ValueError:
            try:
                param_info["Values"] = {float(k): v for k, v in param_info["values"].items()}
            except ValueError:
                logging_warning(_("Could not convert values to int or float for %s"), param_name)
                logging_warning(
                    _("Parameter %s has invalid metadata. Please file a bug at %s"),
                    param_name,
                    "https://github.com/ArduPilot/ardupilot/issues",
                )

    def _create_parameter_tooltips(self, param_name: str, param_info: dict[str, Any]) -> None:
        """Create tooltip documentation for parameters."""
        # Create common prefix parts
        prefix_parts = [f"{param_info['humanName']}"]
        prefix_parts += param_info["documentation"]

        # Add field information (excluding Units and UnitText)
        for key, value in param_info["fields"].items():
            if key not in {"Units", "UnitText"}:
                prefix_parts += split_into_lines(f"{key}: {value}", TOOLTIP_MAX_LENGTH)

        # Add default value if available
        default_suffix = []
        if param_name in self.param_default_dict:
            default_value = format(self.param_default_dict[param_name].value, ".6f").rstrip("0").rstrip(".")
            default_suffix = [f"Default: {default_value}"]

        # Create standard tooltip with alphabetically sorted values
        prefix_parts_with_values = prefix_parts + format_columns(param_info["values"], TOOLTIP_MAX_LENGTH) + default_suffix
        param_info["doc_tooltip"] = "\n".join(prefix_parts_with_values)

        # Create numerically sorted tooltip for Current Value column
        prefix_parts_with_sorted_values = (
            prefix_parts + self._format_columns_sorted_numerically(param_info["values"], TOOLTIP_MAX_LENGTH) + default_suffix
        )
        param_info["doc_tooltip_sorted_numerically"] = "\n".join(prefix_parts_with_sorted_values)

    def read_params_from_files(self) -> dict[str, ParDict]:
        """
        Reads intermediate parameter files from a directory and stores their contents in a dictionary.

        This function scans the specified directory for files matching a specific pattern,
        reads each file, and stores the parameter names and values in a dictionary.
        Files named '00_default.param' and '01_ignore_readonly.param' are ignored.

        Returns:
        - Dict[str, ParDict]: A dictionary with filenames as keys and ParDict as values.

        """
        parameters: dict[str, ParDict] = {}
        if os_path.isdir(self.vehicle_dir):
            # Regular expression pattern for filenames starting with two digits followed by an underscore and ending in .param
            pattern = re_compile(PARAMETER_FILE_REGEXP)

            for filename in sorted(os_listdir(self.vehicle_dir)):
                if pattern.match(filename):
                    if filename in {"00_default.param", "01_ignore_readonly.param"}:
                        continue
                    parameters[filename] = ParDict.from_file(os_path.join(self.vehicle_dir, filename))
        else:
            logging_error(_("Error: %s is not a directory."), self.vehicle_dir)
        return parameters

    def compound_params(self, last_filename: str | None = None, skip_default: bool = True) -> tuple[ParDict, str | None]:
        """
        Compound parameters from multiple .param files into a single ParDict.

        This method iterates through file_parameters (loaded via read_params_from_files)
        and compounds them into a single ParDict. By default, it excludes 00_default.param
        and stops at the specified last_filename if provided.

        Args:
            last_filename: Optional filename to stop processing at (inclusive).
                          If None, processes all files.
            skip_default: If True, skips 00_default.param. Default is True.

        Returns:
            tuple[ParDict, Optional[str]]: A tuple containing:
                - The compounded ParDict with all parameters
                - The first config step filename (excluding 00_default.param if skip_default is True)

        """
        compound = ParDict()
        first_config_step_filename = None

        for file_name, file_params in self.file_parameters.items():
            # Skip default file if requested
            if skip_default and file_name == "00_default.param":
                continue

            # Track the first config step filename
            if first_config_step_filename is None:
                first_config_step_filename = file_name

            # Append parameters from this file
            compound.append(file_params)

            # Stop at the specified filename if provided
            if last_filename and file_name == last_filename:
                break

        return compound, first_config_step_filename

    @staticmethod
    def str_to_bool(s: str) -> bool | None:
        """
        Converts a string representation of a boolean value to a boolean.

        This function interprets the string 'true', 'yes', '1' as True, and 'false', 'no', '0' as False.
        Any other input will return None.

        Args:
          s (str): The string to convert.

        Returns:
          Optional[bool]: True, False, or None if the string does not match any known boolean representation.

        """
        # Use sets for faster O(1) lookup instead of multiple comparisons
        lower_s = s.lower()
        if lower_s in {"true", "yes", "1"}:
            return True
        if lower_s in {"false", "no", "0"}:
            return False
        return None

    def export_to_param(self, params: ParDict, filename_out: str, annotate_doc: bool = True) -> None:
        """
        Exports a dictionary of parameters to a .param file and optionally annotates the documentation.

        This function formats the provided parameters into a string suitable for a .param file,
        writes the string to the specified output file, and optionally updates the parameter documentation.

        Args:
          params (ParDict): A ParDict of parameters to export.
          filename_out (str): The name of the output file.
          annotate_doc (bool, optional): Whether to update the parameter documentation. Default is True.

        """
        params.export_to_param(os_path.join(self.vehicle_dir, filename_out))
        if annotate_doc:
            update_parameter_documentation(
                self.doc_dict, os_path.join(self.vehicle_dir, filename_out), "missionplanner", self.param_default_dict
            )

    def vehicle_configuration_file_exists(self, filename: str) -> bool:
        """
        Check if a vehicle configuration file exists in the vehicle directory.

        Args:
          filename (str): The name of the file to check.

        Returns:
          bool: True if the file exists and is a file (not a directory) and is not empty, False otherwise.

        """
        file_path = os_path.join(self.vehicle_dir, filename)
        return os_path.exists(file_path) and os_path.isfile(file_path) and os_path.getsize(file_path) > 0

    def __all_intermediate_parameter_file_comments(self) -> dict[str, str]:
        """
        Retrieves all comments associated with parameters from intermediate parameter files.

        This method iterates through all intermediate parameter files, collects comments for each parameter,
        and returns them as a dictionary where the keys are parameter names and the values are the comments.
        Comments from the same parameter in different files are not merged; only the comment from the last file is returned.

        Returns:
        - Dict[str, str]: A dictionary mapping parameter names to their comments.

        """
        ret = {}
        for params in self.file_parameters.values():
            ret.update({param: info.comment for param, info in params.items() if info.comment})
        return ret

    def annotate_intermediate_comments_to_param_dict(self, param_dict: dict[str, float]) -> ParDict:
        """
        Annotates comments from intermediate parameter files to a parameter value-only dictionary.

        This function takes a dictionary of parameters with only values and adds comments from
        intermediate parameter files to create a new ParDict where each parameter is represented
        by a 'Par' object containing both the value and the comment.

        Args:
          param_dict (Dict[str, float]): A dictionary of parameters with only values.

        Returns:
          ParDict: A ParDict of parameters with intermediate parameter file comments.

        """
        ip_comments = self.__all_intermediate_parameter_file_comments()
        return ParDict.from_float_dict(param_dict).annotate_with_comments(ip_comments)

    def categorize_parameters(self, param: ParDict) -> tuple[ParDict, ParDict, ParDict, ParDict]:
        """
        Categorize parameters into four categories based on their default values and documentation attributes.

        This method iterates through the provided ParDict of parameters and categorizes them into four groups:
        - Non-default, read-only parameters
        - Non-default, writable calibrations
        - Non-default, writable IDs
        - Non-default, writable non-calibrations, non-IDs

        Args:
          param (ParDict): A ParDict mapping parameter names to their 'Par' objects.

        Returns:
          Tuple[ParDict, ParDict, ParDict, ParDict]: A tuple of four ParDict objects.
                                  Each ParDict represents one of the categories mentioned above.

        """
        return param.categorize_by_documentation(self.doc_dict, self.param_default_dict, is_within_tolerance)

    @staticmethod
    def get_directory_name_from_full_path(full_path: str) -> str:
        # Normalize the path to ensure it's in a standard format
        normalized_path = os_path.normpath(full_path)

        # Split the path into head and tail, then get the basename of the tail
        return os_path.basename(os_path.split(normalized_path)[1])

    # Extract the vehicle name from the directory path
    def get_vehicle_directory_name(self) -> str:
        return self.get_directory_name_from_full_path(self.vehicle_dir)

    def zip_file_path(self, zip_file_name: str = "") -> str:
        vehicle_name = self.get_vehicle_directory_name()
        return os_path.join(self.vehicle_dir, zip_file_name or f"{vehicle_name}.zip")

    def zip_file_exists(self, zip_file_name: str = "") -> bool:
        zip_file_path = self.zip_file_path(zip_file_name)
        return os_path.exists(zip_file_path) and os_path.isfile(zip_file_path)

    def add_configuration_file_to_zip(self, zipf: ZipFile, filename: str) -> None:
        if self.vehicle_configuration_file_exists(filename):
            zipf.write(os_path.join(self.vehicle_dir, filename), arcname=filename)

    def zip_files(self, files_to_zip: list[tuple[bool, str]], zip_file_name: str = "", include_apm_pdef: bool = True) -> str:
        """
        Zips the intermediate parameter files that were written to, including specific summary files.

        This method creates a zip archive containing all intermediate parameter files, along with
        specific summary files if they were written. The zip file is saved in the same directory as the
        intermediate parameter files. The method checks for the existence of each file before
        attempting to add it to the zip archive.

        Args:
          files_to_zip (List[Tuple[bool, str]]): A list of tuples, where each tuple contains a boolean
                                            indicating if the file was written and a string for the filename.
          zip_file_name (str, optional): The name where the zip file will be saved.
                                         If empty, defaults to the path returned by self.zip_file_path().
          include_apm_pdef (bool): Whether to include the 'apm.pdef.xml' file in the zip archive. Default is True.

        """
        zip_file_path = self.zip_file_path(zip_file_name)
        with ZipFile(zip_file_path, "w") as zipf:
            # Add all intermediate parameter files
            for file_name in self.file_parameters:
                self.add_configuration_file_to_zip(zipf, file_name)
                # Add step-specific documentation metadata files
                pdef_xml_file = file_name.replace(".param", ".pdef.xml")
                self.add_configuration_file_to_zip(zipf, pdef_xml_file)

            # Check for and add specific files if they exist
            specific_files = [
                "00_default.param",
                self.configuration_steps_filename,
                self.vehicle_components_fs.json_filename,
                "vehicle.jpg",
                "last_uploaded_filename.txt",
                "tempcal_gyro.png",
                "tempcal_acc.png",
                "tuning_report.csv",
                "complete.param",
            ]
            if include_apm_pdef:
                specific_files.append("apm.pdef.xml")
            for file_name in specific_files:
                self.add_configuration_file_to_zip(zipf, file_name)
            zipped_files = zipf.namelist()

            # Add conditional files using generator expression filtering
            for filename in (filename for wrote, filename in files_to_zip if wrote):
                if filename not in zipped_files:  # avoid duplicating files
                    self.add_configuration_file_to_zip(zipf, filename)

        logging_info(_("Intermediate parameter files and summary files zipped to %s"), zip_file_path)
        return zip_file_path

    def vehicle_image_filepath(self) -> str:
        return os_path.join(self.vehicle_dir, "vehicle.jpg")

    def vehicle_image_exists(self) -> bool:
        return os_path.exists(self.vehicle_image_filepath()) and os_path.isfile(self.vehicle_image_filepath())

    @staticmethod
    def new_vehicle_dir(base_dir: str, new_dir: str) -> str:
        return os_path.join(base_dir, new_dir)

    @staticmethod
    def directory_exists(directory: str) -> bool:
        return os_path.exists(directory) and os_path.isdir(directory)

    def copy_template_files_to_new_vehicle_dir(  # pylint: disable=too-many-arguments, too-many-positional-arguments
        self,
        template_dir: str,
        new_vehicle_dir: str,
        blank_change_reason: bool,
        copy_vehicle_image: bool,
        use_fc_params: bool = False,
        fc_parameters: dict[str, float] | None = None,
    ) -> str:
        # Copy the template files to the new vehicle directory
        try:
            if not os_path.exists(template_dir):
                error_msg = _("Template directory does not exist: {template_dir}")
                error_msg = error_msg.format(**locals())
                logging_error(error_msg)
                return error_msg

            if not os_path.exists(new_vehicle_dir):
                error_msg = _("New vehicle directory does not exist: {new_vehicle_dir}")
                error_msg = error_msg.format(**locals())
                logging_error(error_msg)
                return error_msg

            skip_files = {
                "apm.pdef.xml",
                "last_uploaded_filename.txt",
                "tempcal_acc.png",
                "tempcal_gyro.png",
            }
            if not copy_vehicle_image:
                skip_files.add("vehicle.jpg")

            for item in os_listdir(template_dir):
                if item in skip_files:
                    continue
                if item.endswith(".param") and not item[0:2].isdigit():
                    # Skip non-intermediate parameter files that do not start with NN_
                    continue
                source = os_path.join(template_dir, item)
                dest = os_path.join(new_vehicle_dir, item)
                if (
                    item.endswith(".param")
                    and (blank_change_reason or (use_fc_params and fc_parameters))
                    and item != "00_default.param"
                ):
                    # Parse source into memory, apply transformations, write the result to dest in one pass
                    params = ParDict.load_param_file_into_dict(source)
                    LocalFilesystem._transform_param_dict(params, blank_change_reason, use_fc_params, fc_parameters)
                    params.export_to_param(dest)
                elif os_path.isdir(source):
                    shutil_copytree(source, dest)
                else:
                    shutil_copy2(source, dest)
        except (OSError, shutil_Error) as _e:
            error_msg = _("Error copying template files to new vehicle directory: {_e}")
            return error_msg.format(**locals())
        return ""

    @staticmethod
    def _transform_param_dict(
        params: ParDict,
        blank_change_reason: bool,
        use_fc_params: bool,
        fc_parameters: dict[str, float] | None,
    ) -> None:
        """
        Apply in-place transformations to a parameter dict during template copy.

        Args:
            params: The parameter dictionary to transform (modified in place).
            blank_change_reason: When True, strips all comments from the parameters.
            use_fc_params: When True, replaces parameter values with FC values where they differ.
            fc_parameters: Flight controller parameter values; only used when use_fc_params is True.

        """
        if blank_change_reason:
            for param in params.values():
                param.comment = None
        if use_fc_params and fc_parameters:
            for param_name, param in params.items():
                if param_name in fc_parameters:
                    new_value = fc_parameters[param_name]
                    if not is_within_tolerance(param.value, new_value):
                        param.value = new_value

    def remove_created_files_and_vehicle_dir(self) -> str:
        # Remove the created files and the vehicle directory itself
        try:
            if not os_path.exists(self.vehicle_dir):
                # Use the actual vehicle_dir value to avoid KeyError from missing format keys
                error_msg = _("Vehicle directory to remove does not exist: {vehicle_dir}")
                error_msg = error_msg.format(vehicle_dir=self.vehicle_dir)
                logging_error(error_msg)
                return error_msg

            # delete all files in the vehicle directory and delete the vehicle directory
            errors: list[str] = []
            for item in os_listdir(self.vehicle_dir):
                item_path = os_path.join(self.vehicle_dir, item)
                try:
                    # If the entry is a symlink, remove the link instead of recursing into the target
                    if os_path.islink(item_path):
                        os_remove(item_path)
                    elif os_path.isdir(item_path):
                        shutil_rmtree(item_path)
                    else:
                        os_remove(item_path)
                except OSError as e:
                    logging_exception(_("Error removing %s"), item_path, e)
                    errors.append(str(e))

            # Try to remove the now-empty vehicle directory
            try:
                os_rmdir(self.vehicle_dir)
            except OSError as e:
                logging_exception(_("Error removing directory %s"), self.vehicle_dir, e)
                errors.append(str(e))

            if errors:
                # Return a combined, localized error message
                error_msg = _("Error removing created files: {error_list}")
                return error_msg.format(error_list="; ".join(errors))
        except OSError as _e:  # filesystem-related errors
            error_msg = _("Error removing created files: {_e}")
            logging_exception(_("Error removing created files:"), exc_info=_e)
            return error_msg.format(**locals())
        return ""

    @staticmethod
    def getcwd() -> str:
        return os_getcwd()

    def tempcal_imu_result_param_tuple(self) -> tuple[str, str]:
        tempcal_imu_result_param_filename = "03_imu_temperature_calibration_results.param"
        return tempcal_imu_result_param_filename, os_path.join(self.vehicle_dir, tempcal_imu_result_param_filename)

    def write_last_uploaded_filename(self, current_file: str) -> None:
        try:
            with open(
                os_path.join(self.vehicle_dir, "last_uploaded_filename.txt"), "w", encoding="utf-8", newline="\n"
            ) as file:
                file.write(current_file)
        except Exception as e:  # pylint: disable=broad-except
            logging_error(_("Error writing last uploaded filename: %s"), e)

    def __read_last_uploaded_filename(self) -> str:
        try:
            with open(os_path.join(self.vehicle_dir, "last_uploaded_filename.txt"), encoding="utf-8") as file:
                return file.read().strip()
        except FileNotFoundError as e:
            logging_debug(_("last_uploaded_filename.txt not found: %s"), e)
        except Exception as e:  # pylint: disable=broad-except
            logging_error(_("Error reading last uploaded filename: %s"), e)
        return ""

    def get_start_file(self, explicit_index: int, tcal_available: bool) -> str:
        # Get the list of intermediate parameter files files that will be processed sequentially
        files = list(self.file_parameters.keys())

        if not files:
            return ""

        if explicit_index >= 0:
            # Determine the starting file based on the --n command line argument
            start_file_index = explicit_index  # Ensure the index is within the range of available files
            if start_file_index >= len(files):
                start_file_index = len(files) - 1
                logging_warning(
                    _("Starting file index %s is out of range. Starting with file %s instead."),
                    explicit_index,
                    files[start_file_index],
                )
            return files[start_file_index]

        # In the no-tcal branch the historical behaviour is files[2], i.e. skip the first two files,
        # typically 02_imu_temperature_calibration_setup.param + 03_imu_temperature_calibration_results.param.
        # Fall back to the last available file when fewer than 3 files are present
        # so we never crash on a small or non-standard file set.
        if tcal_available:
            start_file = files[0]
            info_msg = _("Starting with the first file.")
        elif len(files) >= 3:
            start_file = files[2]
            info_msg = _("Starting with the first non-tcal file.")
        else:
            start_file = files[-1]
            info_msg = _("Fewer than three files available; starting with the last file.")

        last_uploaded_filename = self.__read_last_uploaded_filename()
        if last_uploaded_filename:
            logging_info(_("Last uploaded file was %s."), last_uploaded_filename)
        else:
            logging_info(_("No last uploaded file found. %s."), info_msg)
            return start_file

        if last_uploaded_filename not in files:
            # Handle the case where last_uploaded_filename is not found in the list
            logging_warning(_("Last uploaded file not found in the list of files.  %s."), info_msg)
            return start_file

        # Find the index of last_uploaded_filename in files
        last_uploaded_index = files.index(last_uploaded_filename)
        # Check if there is a file following last_uploaded_filename
        start_file_index = last_uploaded_index + 1
        if start_file_index >= len(files):
            # Last uploaded file is the last file in the list. Respect the existing
            # tcal_available branching (files[2] already skips 00_default+01_tcal
            # when tcal is not available) and only adjust when start_file is the
            # read-only 00_default.param snapshot -- see #1507.
            logging_warning(_("Last uploaded file is the last file in the list. Starting from the beginning."))
            if start_file == "00_default.param":
                start_file = next((c for c in files if c != "00_default.param"), "")
                if not start_file:
                    msg = _("Cannot restart configuration: 00_default.param is the only available file and is not editable.")
                    raise ValueError(msg)
            return start_file
        return files[start_file_index]

    def backup_fc_parameters_to_file(
        self,
        param_dict: dict[str, float],
        filename: str,
        overwrite_existing_file: bool = False,
        even_if_last_uploaded_filename_exists: bool = True,
    ) -> None:
        if (even_if_last_uploaded_filename_exists or not self.__read_last_uploaded_filename()) and (
            overwrite_existing_file or not self.vehicle_configuration_file_exists(filename)
        ):
            param_dict_as_par = ParDict({param: Par(float(value), "") for param, value in param_dict.items()})
            param_dict_as_par.export_to_param(os_path.join(self.vehicle_dir, filename))

    def get_eval_variables(self) -> dict[str, dict[str, Any]]:
        variables = {}
        if (
            hasattr(self, "vehicle_components_fs")
            and self.vehicle_components_fs.data
            and "Components" in self.vehicle_components_fs.data
        ):
            variables["vehicle_components"] = self.vehicle_components_fs.data["Components"]
        if hasattr(self, "doc_dict") and self.doc_dict:
            variables["doc_dict"] = self.doc_dict
        return variables

    def calculate_derived_and_forced_param_changes(
        self,
        fc_param_names: list[str],
        fc_parameters: dict[str, float] | None = None,
    ) -> dict[str, ParDict]:
        """
        Compute updated parameter values for all configuration files.

        ``self.file_parameters`` is never mutated by this method. The caller
        decides whether to apply the returned changes via
        :meth:`apply_computed_changes`.

        Side effects: ``self.forced_parameters`` and ``self.derived_parameters``
        are updated in place as a result of the internal :meth:`compute_parameters`
        calls. These side effects are intentional and required by
        :meth:`merge_forced_or_derived_parameters`.

        To apply accepted changes to the data model call
        :meth:`apply_computed_changes` with the returned dict. To persist them to
        disk call :meth:`save_vehicle_params_to_files` afterwards.

        Args:
            fc_param_names: List of parameter names that exist in the FC.
                            If empty or None all parameters are assumed to exist.
            fc_parameters: Optional dictionary mapping parameter names to their current FC values.
                           When provided, enables evaluation of conditions referencing ``fc_parameters``
                           and allows add-from-FC shorthand ``derived_parameters`` entries
                           (those without a ``New Value``) to be populated with current FC values.

        Returns:
            dict[str, ParDict]: Mapping of filenames to their fully-computed ``ParDict``
                                for every file whose computed state differs from the
                                loaded in-memory original.
                                An empty dict means no changes were detected.

        Raises:
            ValueError: If there is an error computing forced or derived parameters.

        Example:
            computed_changes = fs.calculate_derived_and_forced_param_changes(
                fc_param_names=["PARAM1"],
            )
            if computed_changes:
                if user_confirms:
                    fs.apply_computed_changes(computed_changes)
                    fs.save_vehicle_params_to_files(list(fs.file_parameters))
                # else: nothing - self.file_parameters was never mutated
            else:
                # No changes detected - nothing to do
                pass

        """
        eval_variables = self.get_eval_variables()
        if fc_parameters is not None:
            eval_variables["fc_parameters"] = fc_parameters
        # fc_parameters are intentionally kept in eval_variables only when provided,
        # so add-from-FC derived entries without fc_parameters silently skip.

        computed_changes: dict[str, ParDict] = {}

        for param_filename, param_dict in self.file_parameters.items():
            # Build a working copy - Phase 1 must never mutate self.file_parameters
            working = param_dict.deep_copy()

            # Compute and merge forced / derived parameters into the working copy
            if self.configuration_steps and param_filename in self.configuration_steps:
                step_dict = self.configuration_steps[param_filename]
                forced_error, derived_error = self.compute_forced_and_derived_parameters(
                    param_filename, step_dict, eval_variables, ignore_fc_derived_param_warnings=True
                )
                if forced_error:
                    msg = f"Error computing forced parameters for {param_filename}: {forced_error}"
                    raise ValueError(msg)
                self.merge_forced_or_derived_parameters(param_filename, self.forced_parameters, fc_param_names, target=working)

                if derived_error:
                    msg = f"Error computing derived parameters for {param_filename}: {derived_error}"
                    raise ValueError(msg)
                self.merge_forced_or_derived_parameters(
                    param_filename, self.derived_parameters, fc_param_names, target=working
                )
                # Compute deletions once and reuse in compute_add_parameters to avoid redundant evaluation
                to_delete = self.compute_deletions(param_filename, step_dict, eval_variables)
                self.compute_add_parameters(
                    param_filename, step_dict, eval_variables, existing_params=working, parameters_to_delete=to_delete
                )
                self.merge_forced_or_derived_parameters(param_filename, self.add_parameters, None, target=working)

                # Apply deletions from delete_parameters
                actually_deleted = [p for p in sorted(to_delete) if p in working]
                if actually_deleted:
                    logging_info(_("Deleting parameters %s from '%s'"), actually_deleted, param_filename)
                for param_name in to_delete:
                    working.pop(param_name, None)

            # Include in computed_changes if the working copy differs from the loaded in-memory state
            if working.differs_from(param_dict):
                computed_changes[param_filename] = working

        return computed_changes

    def save_vehicle_params_to_files(self, filenames: list[str]) -> None:
        """
        Write the current in-memory parameter values for the given files to disk.

        Args:
            filenames: List of parameter filenames (keys of :attr:`file_parameters`) to save.

        """
        annotate_docs = bool(ProgramSettings.get_setting("annotate_docs_into_param_files"))
        for filename in filenames:
            self.export_to_param(self.file_parameters[filename], filename, annotate_doc=annotate_docs)

    def apply_computed_changes(self, computed_changes: dict[str, ParDict]) -> None:
        """
        Apply pre-computed parameter changes to the in-memory data model.

        Args:
            computed_changes: Mapping of filenames to their fully-computed ``ParDict`` as
                     returned by :meth:`calculate_derived_and_forced_param_changes`.

        """
        self.file_parameters.update(computed_changes)

    def merge_forced_or_derived_parameters(
        self,
        filename: str,
        new_parameters: dict[str, ParDict],
        fc_param_names: list[str] | None,
        target: ParDict | None = None,
    ) -> bool:
        """
        Merge forced or derived parameter values into a target parameter dict.

        Args:
            filename: The name of the parameter file.
            new_parameters: Dictionary of new parameters to potentially merge.
            fc_param_names: Optional list of flight controller parameter names.
            target: ParDict to merge into.  When *None* (default), merges into
                    ``self.file_parameters[filename]`` (legacy behaviour used by the
                    parameter-editor upload path).  Pass the working copy from
                    :meth:`calculate_derived_and_forced_param_changes` to keep that
                    method's computation non-mutating.

        """
        if new_parameters is None or filename not in new_parameters:
            return False

        dest = target if target is not None else self.file_parameters.get(filename)
        if dest is None:
            return False

        at_least_one_param_changed = False
        for param_name, param in new_parameters[filename].items():
            if fc_param_names is None or not fc_param_names or param_name in fc_param_names:
                if param_name in dest:
                    # Skip parameters that have manual override active — user has explicitly
                    # opted out of automated forced/derived value management for this param
                    if (dest[param_name].comment or "").startswith(MANUAL_OVERRIDE_PREFIX):
                        continue
                    if not is_within_tolerance(dest[param_name].value, param.value):
                        at_least_one_param_changed = True
                else:
                    at_least_one_param_changed = True
                dest[param_name] = param
        return at_least_one_param_changed

    def set_param_default_values_if_different(self, param_default_values: ParDict) -> bool:
        """
        Set the default parameter values if they are different from the current ones.

        Args:    param_default_values: A ParDict containing the default parameter values to compare against the current ones.
        Returns: bool: True if the default parameter values were updated, False if they were the same as the current ones.
        """
        param_default_values = ParDict(dict(sorted(param_default_values.items())))
        if self.param_default_dict != param_default_values:
            self.param_default_dict = param_default_values
            return True
        return False

    def write_param_default_values_to_file(
        self, param_default_values: ParDict, filename: str = "00_default.param", vehicle_dir: str = ""
    ) -> None:
        if self.set_param_default_values_if_different(param_default_values):
            if len(vehicle_dir) == 0:
                vehicle_dir = self.vehicle_dir
            self.param_default_dict.export_to_param(os_path.join(vehicle_dir, filename))

    @staticmethod
    def _safe_path_join(base_dir: str, untrusted_path: str) -> str:
        """
        Safely join a base directory with an untrusted path component.

        Validates that the resolved path is a proper child of base_dir to prevent
        path traversal attacks via values like '../../.bashrc' from JSON configs.
        Uses os.path.commonpath for cross-platform robustness (handles case-insensitive
        filesystems on Windows/macOS).

        Raises:
            ValueError: If the resolved path escapes base_dir or resolves to base_dir itself.

        """
        base_real = os_path.normcase(os_path.realpath(base_dir))
        resolved = os_path.normcase(os_path.realpath(os_path.join(base_dir, untrusted_path)))
        try:
            common = os_path.commonpath([base_real, resolved])
        except ValueError:
            # On Windows, commonpath raises ValueError for paths on different drives
            msg = f"Path escapes vehicle directory: {untrusted_path!r}"
            raise ValueError(msg) from None
        if common != base_real or resolved == base_real:
            msg = f"Path escapes vehicle directory: {untrusted_path!r}"
            raise ValueError(msg)
        return os_path.realpath(os_path.join(base_dir, untrusted_path))

    def get_download_url_and_local_filename(self, selected_file: str) -> tuple[str, str]:
        if selected_file in self.configuration_steps and self.configuration_steps[selected_file].get("download_file"):
            src = self.configuration_steps[selected_file]["download_file"].get("source_url", "")
            dst = self.configuration_steps[selected_file]["download_file"].get("dest_local", "")
            if self.vehicle_dir and src and dst:
                return src, self._safe_path_join(self.vehicle_dir, dst)
        return "", ""

    def get_upload_local_and_remote_filenames(self, selected_file: str) -> tuple[str, str]:
        if selected_file in self.configuration_steps and self.configuration_steps[selected_file].get("upload_file"):
            src = self.configuration_steps[selected_file]["upload_file"].get("source_local", "")
            dst = self.configuration_steps[selected_file]["upload_file"].get("dest_on_fc", "")
            if self.vehicle_dir and src and dst:
                return self._safe_path_join(self.vehicle_dir, src), dst
        return "", ""

    @staticmethod
    def get_git_commit_hash() -> str:
        # Try to get git hash using git command
        try:
            result = run(
                ["git", "rev-parse", "HEAD"],  # noqa: S607
                capture_output=True,
                text=True,
                cwd=os_path.dirname(__file__),
                check=True,
            )
            if result.returncode == 0:
                return result.stdout.strip()
        except (SubprocessError, FileNotFoundError) as e:
            # Log the specific error and fallback to reading the git_hash.txt file
            msg = f"Failed to get git hash via command: {e}"
            logging_debug(msg)
            git_hash_file = os_path.join(os_path.dirname(__file__), "git_hash.txt")
            try:
                with open(git_hash_file, encoding="utf-8") as file:
                    return file.read().strip()
            except OSError as exp:
                msg = f"Failed to read git hash from file: {exp}"
                logging_debug(msg)
        return ""

    def find_lowest_available_backup_number(self) -> int:
        """
        Find the lowest non-existing backup number.

        Returns:
            The lowest available backup number

        """
        backup_num = 1  # Start from 01
        while self.vehicle_configuration_file_exists(f"autobackup_{backup_num:02d}.param"):
            backup_num += 1
            if backup_num > 99:  # Prevent endless loop, cap at 99
                # Overwrite the highest number if we reach the cap
                return 99
        return backup_num

    @staticmethod
    def add_argparse_arguments(parser: ArgumentParser) -> ArgumentParser:
        parser.add_argument(
            "-t",
            "--vehicle-type",
            choices=VehicleComponents.supported_vehicles(),
            default="",
            help=_("The type of the vehicle. Default is ArduCopter"),
        )
        parser.add_argument(  # type: ignore[attr-defined]
            "--vehicle-dir",
            type=str,
            default=os_getcwd(),
            help=_(
                "Directory containing vehicle-specific intermediate parameter files. Default is the current working directory"
            ),
        ).completer = DirectoriesCompleter()  # pyright: ignore[reportAttributeAccessIssue]
        parser.add_argument(
            "--n",
            type=int,
            default=-1,
            help=_(
                "Start directly on the nth intermediate parameter file (skips previous files). "
                "Default is to start on the file next to the last that you wrote to the flight controller. "
                "If the file does not exist, it will start on the first file."
            ),
        )
        parser.add_argument(
            "--allow-editing-template-files",
            action="store_true",
            help=_(
                "Allow opening and editing template files directly. "
                "Only for software developers that know what they are doing. "
                "Default is %(default)s"
            ),
        )
        parser.add_argument(
            "--save-component-to-system-templates",
            action="store_true",
            help=_(
                "Save the vehicle component to the system templates. "
                "Only for software developers that know what they are doing. "
                "Default is %(default)s"
            ),
        )
        return parser