chg: split logic out to make it fs independent

Author: nicokant

src/datasync/services.py

@@ -1,4 +1,5 @@
 import copy
+import pathlib
 from collections import OrderedDict
 
 import duckdb
@@ -8,7 +9,7 @@
 import typer
 import yaml
 
-from .settings import env
+from .settings import env, log
 
 app = typer.Typer(help="Miljødata Infrastructure as Code pipelines")
 
@@ -167,48 +168,7 @@ def services_to_parquet(
 DASHBOARD_ORG = env("SERVICES_DASHBOARD_ORG", default="ninanor")
 
 
-@app.command(
-    help="Produce a Homer Dashbord using the Miljødata Infrastructure as Code "
-    "repository as data source"
-)
-def dashboard(
-    org=SERVICES_ORG,
-    repo: str | None = SERVICES_REPO,
-    config_org=DASHBOARD_ORG,
-    config_repo=DASHBOARD_REPO,
-    bucket: str | None = AWS_BUCKET,
-    endpoint: str | None = AWS_ENDPOINT,
-    access_key: str | None = AWS_ACCESS_KEY,
-    secret_key: str | None = AWS_SECRET_KEY,
-    git_username=GIT_USERNAME,
-    git_token=GIT_TOKEN,
-    prefix=DASHBOARD_PREFIX,
-):
-    s3 = s3fs.S3FileSystem(
-        endpoint_url=f"https://{endpoint}",
-        secret=secret_key,
-        key=access_key,
-    )
-
-    infrastucture_repo = fsspec.filesystem(
-        "github",
-        org=org,
-        repo=repo,
-        username=git_username,
-        token=git_token,
-    )
-
-    dashboard_repo = fsspec.filesystem(
-        "github",
-        org=config_org,
-        repo=config_repo,
-        username=git_username,
-        token=git_token,
-    )
-
-    with dashboard_repo.open("base-config.yml", "r") as base_conf_file:
-        base_conf = yaml.load(base_conf_file, Loader=yaml.SafeLoader)
-
+def parse_services(fs: fsspec.AbstractFileSystem):
     curated = OrderedDict()
     all_services = OrderedDict()
 
@@ -228,16 +188,19 @@ def dashboard(
         "public": "is-success",
     }
 
-    services_paths = infrastucture_repo.glob(path="services/*/*/metadata.yml")
+    services_paths = fs.glob(path="services/*/*/metadata.yml")
     for spath in services_paths:
-        with infrastucture_repo.open(spath, "r") as f:
+        log.info("found path", path=spath)
+        with fs.open(spath, "r") as f:
             data = yaml.load(f, Loader=yaml.SafeLoader)
-            _, project_name, module_name, _ = spath.split("/")
-            for resource in data.get("resources"):
+            parts = pathlib.Path(spath).parts
+            project_name = parts[-3]
+            for resource in data.get("resources", []):
+                log.info("Parsing resource", resource=resource)
                 if not resource["uri"].startswith("http") or resource.get(
                     "external", False
                 ):
-                    print(resource)
+                    log.info("Skipping resource", resource=resource)
                     # skip non http resources and external resources
                     continue
 
@@ -278,6 +241,52 @@ def dashboard(
                         "icon": ICON[resource.get("type", "web")],
                     }
                 )
+    return curated, all_services
+
+
+@app.command(
+    help="Produce a Homer Dashbord using the Miljødata Infrastructure as Code "
+    "repository as data source"
+)
+def dashboard(
+    org=SERVICES_ORG,
+    repo: str | None = SERVICES_REPO,
+    config_org=DASHBOARD_ORG,
+    config_repo=DASHBOARD_REPO,
+    bucket: str | None = AWS_BUCKET,
+    endpoint: str | None = AWS_ENDPOINT,
+    access_key: str | None = AWS_ACCESS_KEY,
+    secret_key: str | None = AWS_SECRET_KEY,
+    git_username=GIT_USERNAME,
+    git_token=GIT_TOKEN,
+    prefix=DASHBOARD_PREFIX,
+):
+    s3 = s3fs.S3FileSystem(
+        endpoint_url=f"https://{endpoint}",
+        secret=secret_key,
+        key=access_key,
+    )
+
+    infrastucture_repo = fsspec.filesystem(
+        "github",
+        org=org,
+        repo=repo,
+        username=git_username,
+        token=git_token,
+    )
+
+    dashboard_repo = fsspec.filesystem(
+        "github",
+        org=config_org,
+        repo=config_repo,
+        username=git_username,
+        token=git_token,
+    )
+
+    with dashboard_repo.open("base-config.yml", "r") as base_conf_file:
+        base_conf = yaml.load(base_conf_file, Loader=yaml.SafeLoader)
+
+    curated, all_services = parse_services(infrastucture_repo)
 
     curated_conf = copy.deepcopy(base_conf)
     curated_conf["services"] = [group for _, group in curated.items()]
@@ -298,3 +307,12 @@ def dashboard(
             all_conf,
             f,
         )
+
+
+@app.command(
+    help="Test the parsing of the services metadata "
+    "and the generation of the dashboard configuration"
+)
+def dashboard_test():
+    fs = fsspec.filesystem("file")
+    parse_services(fs)