update IMDB validation and add SAPEXE validation

Author: marcelmamula

plugins/module_utils/software_center/search.py

@@ -135,8 +135,8 @@ def find_file(client, name, deduplicate, search_alternatives, search_upgrades=Fa
             last_option = software_search_alternatives_filtered[-1]['Title']
 
             raise FileNotFoundError(
-                f'More than one alternative was found: '
-                f'{", ".join(names)}.\n'
+                f'More than one alternative was found: {alternatives_count} files\n'
+                f'Files: {", ".join(names)}\n'
                 f'Please use a more specific filename '
                 f'or set deduplicate parameter.\n'
                 f'Options for deduplicate:\n'
@@ -164,7 +164,8 @@ def find_file(client, name, deduplicate, search_alternatives, search_upgrades=Fa
         last_option = software_filtered[-1]['Title']
 
         raise FileNotFoundError(
-            f'More than one result was found: {", ".join(names)}.\n'
+            f'More than one result was found: {files_count} files\n'
+            f'Files: {", ".join(names)}\n'
             f'Please use the correct full filename '
             f'or set deduplicate parameter.\n'
             f'Options for deduplicate:\n'
@@ -235,8 +236,12 @@ def _search_software_fuzzy(client, query, search_upgrades=False):
             if f'-{filename_id}' in r.get('Title', ''):
                 fuzzy_results.append(_remove_useless_keys(r))
 
+        # Check if prefix search hit the 50-result limit (may have missed results)
+        # If yes, fallback to paginated ID search to ensure completeness
+        if len(results) >= 50:
+            fuzzy_results = []  # Clear partial results, will use paginated ID search below
         # If empty and suggested_filename_next exists, try incremented version
-        if len(fuzzy_results) == 0 and suggested_filename_next:
+        elif len(fuzzy_results) == 0 and suggested_filename_next:
             results = _search_software(client, suggested_filename_next)
             for r in results:
                 if f'-{filename_id}' in r.get('Title', ''):

roles/sap_software_download/README.md

@@ -9,6 +9,10 @@ sap_software_download_suser_id: ''
 # The password for the SAP S-User specified in 'sap_software_download_suser_id' (String).
 sap_software_download_suser_password: ''
 
+# The directory where downloaded SAP software files will be stored (String).
+# It will be created if it does not exist. Must be an absolute path.
+sap_software_download_directory: '/software'
+
 # A list of SAP software file names to download (List of Strings).
 sap_software_download_files: []
 
@@ -25,13 +29,18 @@ sap_software_download_mp_stack_xml: true
 ### Optional variables for controlling the download process
 
 # Enables searching for alternative files if the requested file is not found (Boolean).
-# Only applies to files specified in 'sap_software_download_files'.
+# Only applies to files specified in 'sap_software_download_files', not Maintenance Plan files.
 # If set to 'false', the role will not search for alternatives.
+# NOTE: Files like IMDB_SERVER have strict naming and the variable 'sap_software_download_find_upgrades' for broader search.
+# Example: IMDB_SERVER20_067_4-80002046.SAR searches for: IMDB_SERVER20_067
 sap_software_download_find_alternatives: true
 
-# The directory where downloaded SAP software files will be stored (String).
-# It will be created if it does not exist. Must be an absolute path.
-sap_software_download_directory: '/software'
+# Enables broader searching for alternative files if the requested file is not found (Boolean).
+# Only applies to files specified in 'sap_software_download_files', not Maintenance Plan files.
+# Only applies when file was not found with 'sap_software_download_find_alternatives set to 'true'.
+# If set to 'false', the role will not search for upgrade candidates.
+# Example: IMDB_SERVER20_067_4-80002046.SAR searches for: IMDB_SERVER20_067, IMDB_SERVER20_068, IMDB_SERVER20_
+sap_software_download_find_upgrades: false
 
 # Determines whether to ignore errors when a requested file is not found (Boolean).
 # If set to 'true', the role will continue execution and download other files, even if some files are not found.
@@ -44,24 +53,19 @@ sap_software_download_ignore_file_not_found: false
 # If set to 'false', the role will fail if the specified Maintenance Plan is not found.
 sap_software_download_ignore_plan_not_found: false
 
-# Enables validation of relationships between SAP software files (Boolean).
+# Enables validation of relationships between known combinations of SAP files (Boolean).
 # Only applies to files specified in 'sap_software_download_files', not Maintenance Plan files.
 # If set to 'false', no relationship validation will be performed.
-# Example: Verify version of IMDB_LCAPPS against IMDB_SERVER if IMDB_SERVER was found.
-# NOTE: This option will result in longer execution time
-# when 'sap_software_download_find_alternatives' is set to 'true',
-# because it attempts download in Dry run to gather list of alternative files for validation.
+# See section 'Explanation of relationship validation logic' in README.md for more information.
 sap_software_download_validate_relationships: true
 
-# Enables automatic upgrade of related HANA components when SERVER version changes.
-# Only applies when 'sap_software_download_validate_relationships: true'.
-# If IMDB_SERVER is not found and 'search_upgrades' finds a newer revision,
-# all related components (AFL, LCAPPS) will be searched with matching revision.
-# Example: If SERVER 084 is removed and only available revision is 089,
-# then AFL/LCAPPS will search for 089* revisions.
-# WARNING: This does not guarantee installation compatibility due to SAP's
-# inconsistent versioning (filename vs. actual package version).
-# Recommended to test before production use.
+# Enables automatic upgrade of related known combinations of SAP files when leading component version changes.
+# Only applies when 'sap_software_download_validate_relationships' is set to 'true'.
+# If set to 'false', no search for upgrade candidates before validation will be performed.
+# See section 'Explanation of relationship validation logic' in README.md for more information.
+# NOTE: Important for when SAP releases new patch for SAP HANA Revision, because:
+# - Previous patch of 'IMDB_SERVER' will be removed and only new patch will be available to download.
+# - Previous patch for other components like 'IMDB_AFL' will remain available to download as well as new patch.
 sap_software_download_upgrade_relationships: false
 
 # Determines whether to ignore warnings during file relationship validation (Boolean).
@@ -80,6 +84,7 @@ sap_software_download_ignore_validate_credentials: false
 # - 'first': Download the first file found (generally oldest).
 # - 'last': Download the last file found (generally newest).
 # - '': No deduplication, will cause an error if multiple files with the same name are found.
+#       Can be used to find list of all available files without downloading them.
 sap_software_download_deduplicate: last
 
 # Enables checksum validation of existing files present in 'sap_software_download_directory' (Boolean).
@@ -112,7 +117,7 @@ sap_software_download_use_venv: true
 # sap_software_download_python_package: ''
 
 # The list of the OS packages that provide modules for the desired Python version (List of Strings).
-# Required modules: wheel, urllib3, requests, beautifulsoup4, lxml
+# Required modules: 'wheel', 'urllib3', 'requests', 'beautifulsoup4', 'lxml'
 # The listed package versions must match the Python version specified by 'sap_software_download_python_interpreter'.
 # The default value is determined by the operating system and is set in the corresponding OS-specific variables file.
 # Examples:

roles/sap_software_download/defaults/main.yml

@@ -1,13 +1,22 @@
 # SPDX-License-Identifier: Apache-2.0
 ---
 
+- name: SAP Software Download - Download - SAP Software Files - Notice before download
+  ansible.builtin.debug:
+    msg: |
+      INFO: Download will be started for {{ __sap_software_download_files | d([]) | length }} file(s).
+      Large files (Media CD/DVD, Exports, Databases, etc.) may take considerably longer to download.
+      Following task might appear unresponsive during download,
+      but it is working in the background. Please wait for results.
+
 - name: SAP Software Download - Download - SAP Software Files - Get files
   community.sap_launchpad.software_center_download:
     suser_id: "{{ sap_software_download_suser_id }}"
     suser_password: "{{ sap_software_download_suser_password }}"
     search_query: "{{ item }}"
     dest: "{{ sap_software_download_directory }}"
     search_alternatives: "{{ sap_software_download_find_alternatives | d(true) }}"
+    search_upgrades: "{{ sap_software_download_find_upgrades | d(false) }}"
     deduplicate: "{{ sap_software_download_deduplicate }}"
     validate_checksum: "{{ sap_software_download_validate_checksum | d(false) }}"
   # Loop condition acts as when conditional

roles/sap_software_download/tasks/download_files.yml

@@ -1,6 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 ---
 
+- name: SAP Software Download - Download - SAP Software Files - Notice before download
+  ansible.builtin.debug:
+    msg: |
+      INFO: Download will be started for {{ __sap_software_download_register_mp_files.download_basket | d([]) | length }} file(s).
+      Large files (Media CD/DVD, Exports, Databases, etc.) may take considerably longer to download.
+      Following task might appear unresponsive during download,
+      but it is working in the background. Please wait for results.
+
 - name: SAP Software Download - Download - Maintenance Plan - Get files
   community.sap_launchpad.software_center_download:
     suser_id: "{{ sap_software_download_suser_id }}"
@@ -10,7 +18,7 @@
     dest: "{{ sap_software_download_directory }}"
     validate_checksum: "{{ sap_software_download_validate_checksum | d(false) }}"
   # Loop condition acts as when conditional
-  loop: "{{ __sap_software_download_fact_mp_files.download_basket | d([]) }}"
+  loop: "{{ __sap_software_download_register_mp_files.download_basket | d([]) }}"
   loop_control:
     label: "{{ item.Filename }} : {{ __sap_software_download_files_plan_results.msg | d('') }}"
   register: __sap_software_download_files_plan_results

roles/sap_software_download/tasks/download_plan.yml

@@ -2,7 +2,6 @@
 ---
 
 - name: SAP Software Download - Download - Maintenance Plan Stack XML - Get file
-  when: sap_software_download_use_venv | d(true)
   community.sap_launchpad.maintenance_planner_stack_xml_download:
     suser_id: "{{ sap_software_download_suser_id }}"
     suser_password: "{{ sap_software_download_suser_password }}"

roles/sap_software_download/tasks/download_stack.yml

@@ -73,8 +73,8 @@
         file: download_plan.yml
       when:
         - sap_software_download_mp_transaction | length > 0
-        - __sap_software_download_fact_mp_files.download_basket is defined
-          and __sap_software_download_fact_mp_files.download_basket | length > 0
+        - __sap_software_download_register_mp_files.download_basket is defined
+          and __sap_software_download_register_mp_files.download_basket | length > 0
 
     - name: SAP Software Download - Include tasks to download files in the variable 'sap_software_download_files'
       ansible.builtin.include_tasks:

roles/sap_software_download/tasks/main.yml

@@ -1,46 +1,23 @@
 # SPDX-License-Identifier: Apache-2.0
 ---
 
-- name: SAP Software Download - Maintenance Plan - Python venv - Get list of files
-  when: sap_software_download_use_venv | d(true)
+- name: SAP Software Download - Maintenance Plan - Get list of files
   community.sap_launchpad.maintenance_planner_files:
     suser_id: "{{ sap_software_download_suser_id }}"
     suser_password: "{{ sap_software_download_suser_password }}"
     transaction_name: "{{ sap_software_download_mp_transaction }}"
-  register: __sap_software_download_register_mp_files_venv
+  register: __sap_software_download_register_mp_files
   retries: 1
-  environment:
-    PATH: "{{ __sap_software_download_fact_venv_path }}"
-    PYTHONPATH: "{{ __sap_software_download_fact_venv_pythonpath }}"
-    VIRTUAL_ENV: "{{ __sap_software_download_register_temp_venv.path }}"
+  environment: "{{ __venv_environment if sap_software_download_use_venv | d(true) else {} }}"
   vars:
     ansible_python_interpreter: "{{ __sap_software_download_fact_python_interpreter_path }}"
+    __venv_environment:
+      PATH: "{{ __sap_software_download_fact_venv_path }}"
+      PYTHONPATH: "{{ __sap_software_download_fact_venv_pythonpath }}"
+      VIRTUAL_ENV: "{{ __sap_software_download_register_temp_venv.path }}"
   changed_when: false  # Getting list of files does not change anything
   ignore_errors: true  # Errors are ignored and validated afterwards
 
-
-- name: SAP Software Download - Maintenance Plan - Default Python - Get list of files
-  when: not sap_software_download_use_venv | d(true)
-  community.sap_launchpad.maintenance_planner_files:
-    suser_id: "{{ sap_software_download_suser_id }}"
-    suser_password: "{{ sap_software_download_suser_password }}"
-    transaction_name: "{{ sap_software_download_mp_transaction }}"
-  register: __sap_software_download_register_mp_files_default
-  retries: 1
-  vars:
-    ansible_python_interpreter: "{{ __sap_software_download_fact_python_interpreter_path }}"
-  changed_when: false  # Getting list of files does not change anything
-  ignore_errors: true  # Errors are ignored and validated afterwards
-
-
-- name: SAP Software Download - Maintenance Plan - Set fact with list of files
-  ansible.builtin.set_fact:
-    __sap_software_download_fact_mp_files:
-      "{{ __sap_software_download_register_mp_files_venv
-        if sap_software_download_use_venv | d(true)
-        else __sap_software_download_register_mp_files_default }}"
-
-
 - name: SAP Software Download - Maintenance Plan - Fail if Maintenance Plan was not found
   ansible.builtin.fail:
     msg: |
@@ -49,7 +26,7 @@
       or ignore this error with 'sap_software_download_ignore_plan_not_found' set to 'true'.
   when:
     - not sap_software_download_ignore_plan_not_found | d(false)
-    - __sap_software_download_fact_mp_files.failed
+    - __sap_software_download_register_mp_files.failed
 
 - name: SAP Software Download - Maintenance Plan - Inform that Maintenance Plan was not found but error is ignored
   ansible.builtin.debug:
@@ -58,4 +35,4 @@
       Error was ignored with 'sap_software_download_ignore_plan_not_found' set to 'true'.
   when:
     - sap_software_download_ignore_plan_not_found | d(false)
-    - __sap_software_download_fact_mp_files.failed
+    - __sap_software_download_register_mp_files.failed