update filter_translations with fastlane support

Fixes #568

Author: pxeemo

filter_translations

@@ -2,13 +2,15 @@
 from lxml import etree
 import requests
 from requests.adapters import HTTPAdapter
+from concurrent.futures import ThreadPoolExecutor
 from urllib3.util.retry import Retry
 import sys
 import os
+import shutil
+from pathlib import Path
 
 WEBLATE_URL = "https://hosted.weblate.org"
 PROJECT_SLUG = "gramophone"
-COMPONENT_SLUG = "strings-xml"
 API_TOKEN = os.getenv("WEBLATE_TOKEN")
 headers = {
     "Authorization": f"Token {API_TOKEN}",
@@ -26,29 +28,47 @@ session.mount("http://", adapter)
 session.mount("https://", adapter)
 
 
-def get_languages() -> list:
-    url = f"{WEBLATE_URL}/api/components/{PROJECT_SLUG}/{COMPONENT_SLUG}/translations/"
+def get_languages(component_slug: str) -> list:
+    url = f"{WEBLATE_URL}/api/components/{PROJECT_SLUG}/{component_slug}/translations/"
     response = session.get(url, headers=headers, timeout=5)
     response.raise_for_status()  # Raise an error for bad responses
     languages = response.json().get('results')
     return languages
 
 
+def get_pages_units(page: int, language_code: str) -> list:
+    url = f"{
+        WEBLATE_URL}/api/translations/{PROJECT_SLUG}/strings-xml/{language_code}/units/"
+    response = session.get(url, headers=headers,
+                           timeout=5, params={'page': page})
+    response.raise_for_status()
+    data = response.json()
+    return data.get("results")
+
+
 # Fetch all units
-def get_units_list(language_code: str) -> list:
+def get_units_list(language_code: str, component_slug: str) -> list:
     units = []
-    page_url = f"{
-        WEBLATE_URL}/api/translations/{PROJECT_SLUG}/{COMPONENT_SLUG}/{language_code}/units/"
+    url = f"{
+        WEBLATE_URL}/api/translations/{PROJECT_SLUG}/{component_slug}/{language_code}/units/"
 
     print(f'\nGetting units of language code "{language_code}"... ', end="")
-    while page_url:
-        response = session.get(page_url, headers=headers, timeout=5)
-        response.raise_for_status()
-        data = response.json()
-        units.extend(data["results"])
-        page_url = data.get("next")
-    print("Done.")
+    response = session.get(url, headers=headers, timeout=5)
+    response.raise_for_status()
+    data = response.json()
+    units.extend(data.get("results"))
 
+    count = data.get("count")
+    if count > 50:
+        page_params = [{'page': i, 'language_code': language_code}
+                       for i in range(2, count // 50 + 2)]
+        with ThreadPoolExecutor(max_workers=10) as executor:
+            futures = [executor.submit(get_pages_units, **params) for params in page_params]
+
+        for future in futures:
+            units.extend(future.result())
+
+    print("Done.")
     return units
 
 
@@ -66,7 +86,8 @@ def delete_strings(xml_file, names_to_delete):
         resolve_entities=False
     )
 
-    tree = etree.parse(sys.argv[1] + '/' + xml_file, parser)
+    source_file_path = Path(sys.argv[1] + "/" + str(xml_file))
+    tree = etree.parse(source_file_path, parser)
     root = tree.getroot()
 
     # Find all string elements
@@ -91,19 +112,68 @@ def delete_strings(xml_file, names_to_delete):
     )
 
 
-def filter_translations(language: dict) -> None:
-    units = get_units_list(language.get('language').get('code'))
+def filter_editings(language_code: dict, component_slug: str) -> list:
+    units = get_units_list(language_code, component_slug)
     string_names = [unit.get('context')
                     for unit in units if unit.get('state') == 10]
     print("  Strings to delete: " + str(string_names))
+    return string_names
 
-    delete_strings(language.get("filename"), string_names)
 
+def manage_fastlane_files(files_path: Path, editings: list) -> None:
+    source_files_path = Path(sys.argv[1] + "/" + files_path)
+    if not source_files_path.exists():
+        print(f"  Directory does not exist: {source_files_path}")
+        return
 
-if __name__ == "__main__":
-    languages = get_languages()
+    if sys.argv[1] != os.getcwd():
+        shutil.rmtree(files_path)
+        shutil.copytree(source_files_path, files_path)
 
+    files = ["short_description.txt", "full_description.txt", "title.txt"]
+    delete_files = any("description" in s for s in editings)
+    # true if a file has "description" in its name
+    if delete_files:
+        print(f"  Deleting files in {files_path}")
+        for file in files:
+            file_path = Path(files_path + "/" + file)
+            if file_path.exists():
+                os.remove(file_path)
+
+    # if delete_changelogs:
+    #     changelogs_dir = Path(files_path + "/changelogs/")
+    #     if changelogs_dir.exists() and changelogs_dir.is_dir():
+    #         print(f"  Deleting changelog {changelogs_dir}")
+    #         shutil.rmtree(changelogs_dir)
+
+    if not delete_files:
+        print("  Skipped deleting")
+
+
+def filter_strings_xml():
+    languages = get_languages("strings-xml")
     for language in languages:
-        if not language.get('is_source'):
-            filter_translations(language)
+        if language.get('is_source'):
+            continue
+        language_code = language.get('language').get('code')
+        files_path = Path(language.get('filename'))
+        editings = filter_editings(language_code, "strings-xml")
+        delete_strings(files_path, editings)
 
+
+def filter_fastlane():
+    fastlane_languages = get_languages('fastlane')
+    for language in fastlane_languages:
+        if language.get("is_source"):
+            continue
+        language_code = language.get('language').get('code')
+        files_path = language.get('filename')
+        editings = filter_editings(language_code, "fastlane")
+        manage_fastlane_files(files_path, editings)
+
+
+if __name__ == "__main__":
+    print("Filtering strings-xml...")
+    filter_strings_xml()
+    print("Filtering fastlane...")
+    filter_fastlane()