Skip to content

Commit dc2a293

Browse files
nicokanttaheeraahmed
nicokant
authored and
taheeraahmed
committed
feat: implement services scripts
1 parent d5a5557 commit dc2a293

4 files changed

Lines changed: 303 additions & 2 deletions

File tree

‎pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ dependencies = [
3838
"pyarrow>=20.0.0",
3939
"httpx>=0.28.1",
4040
"setuptools>=65.5.0",
41-
"backoff>=2.1.0"
41+
"backoff>=2.1.0",
42+
"pyyaml>=6.0.3"
4243
]
4344
description = ""
4445
license = "GPL-3.0+"

‎src/datasync/main.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
import typer
66

7-
from . import dms, grass, ninagen, nva, pit_registering_salmon, ubw
7+
from . import dms, grass, ninagen, nva, pit_registering_salmon, services, ubw
88

99
app = typer.Typer(
1010
help="Provide subcommands for synchronizing different resources, see subcommands"
@@ -15,6 +15,7 @@
1515
app.add_typer(ninagen.app, name="ninagen")
1616
app.add_typer(pit_registering_salmon.app, name="pit-registering-salmon")
1717
app.add_typer(grass.app, name="grass-gis")
18+
app.add_typer(services.app, name="services")
1819

1920
if __name__ == "__main__":
2021
app()

‎src/datasync/services.py‎

Lines changed: 297 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,297 @@
1+
import copy
2+
from collections import OrderedDict
3+
4+
import duckdb
5+
import fsspec
6+
import pyarrow as pa
7+
import s3fs
8+
import typer
9+
import yaml
10+
11+
from .settings import env
12+
13+
app = typer.Typer(help="Miljødata Infrastructure as Code pipelines")
14+
15+
16+
AWS_BUCKET = env("SERVICES_AWS_BUCKET", default=None)
17+
AWS_ENDPOINT = env("SERVICES_AWS_ENDPOINT", default=None)
18+
AWS_ACCESS_KEY = env("SERVICES_AWS_ACCESS_KEY", default=None)
19+
AWS_SECRET_KEY = env("SERVICES_AWS_SECRET_KEY", default=None)
20+
21+
SERVICES_REPO = env("SERVICES_REPO", None)
22+
SERVICES_ORG = env("SERVICES_ORG", default="ninanor")
23+
24+
PARQUET_PREFIX = env("SERVICES_STORAGE_PREFIX", default="/dms/tables")
25+
26+
27+
GIT_USERNAME = env("GIT_USERNAME", default="")
28+
GIT_TOKEN = env("GIT_TOKEN", default="")
29+
30+
31+
@app.command(
32+
help="Convert metadata.yml definitions to a set of parquet "
33+
"that can be imported in the DMS"
34+
)
35+
def services_to_parquet(
36+
org=SERVICES_ORG,
37+
repo: str | None = SERVICES_REPO,
38+
bucket: str | None = AWS_BUCKET,
39+
endpoint: str | None = AWS_ENDPOINT,
40+
access_key: str | None = AWS_ACCESS_KEY,
41+
secret_key: str | None = AWS_SECRET_KEY,
42+
prefix=PARQUET_PREFIX,
43+
git_username=GIT_USERNAME,
44+
git_token=GIT_TOKEN,
45+
):
46+
fs = fsspec.filesystem(
47+
"github",
48+
org=org,
49+
repo=repo,
50+
username=git_username,
51+
token=git_token,
52+
)
53+
conn = duckdb.connect()
54+
55+
services_list = []
56+
57+
services_paths = fs.glob(path="services/*/*/metadata.yml")
58+
for spath in services_paths:
59+
with fs.open(spath, "r") as f:
60+
data = yaml.load(f, Loader=yaml.SafeLoader)
61+
_, project_name, module_name, _ = spath.split("/")
62+
services_list.append(
63+
{
64+
**data,
65+
"project_name": project_name,
66+
"module_name": module_name,
67+
"projects": list(map(str, data.get("projects") or [])),
68+
"technologies": list(map(str, data.get("technologies") or [])),
69+
"related_projects": list(
70+
map(str, data.get("related_projects") or [])
71+
),
72+
}
73+
)
74+
75+
df = pa.Table.from_pylist(services_list)
76+
77+
conn.sql(f"""
78+
create secret (
79+
type s3,
80+
endpoint '{endpoint}',
81+
KEY_ID '{access_key}',
82+
SECRET '{secret_key}',
83+
REGION 'us-east-1',
84+
URL_STYLE path
85+
);
86+
""")
87+
88+
conn.from_arrow(df).write_parquet(
89+
f"s3://{bucket}{prefix}/services_raw.parquet", overwrite=True
90+
)
91+
92+
# Projects
93+
conn.from_arrow(df).select("""
94+
project_name || '__' || module_name as service_id,
95+
unnest(projects) as project_id
96+
""").write_parquet(
97+
f"s3://{bucket}{prefix}/services_projectservice.parquet", overwrite=True
98+
)
99+
100+
# Service
101+
conn.from_arrow(df).select("""
102+
project_name || '__' || module_name as id,
103+
title,
104+
description,
105+
'{' || array_to_string(list_transform(keywords, x -> '"'||x||'"' ), ',') || '}' as keywords,
106+
'{' || array_to_string(list_transform(technologies, x -> '"'||x||'"' ), ',') || '}' as technologies,
107+
""").write_parquet( # noqa: E501
108+
f"s3://{bucket}{prefix}/services_service.parquet", overwrite=True
109+
)
110+
111+
# Contributors
112+
conn.sql("""
113+
from (
114+
from df
115+
select
116+
project_name || '__' || module_name as service_id,
117+
unnest(contributors) as contributor
118+
)
119+
select
120+
service_id,
121+
contributor.email as email,
122+
contributor.role as role,
123+
""").write_parquet(
124+
f"s3://{bucket}{prefix}/services_contributor.parquet", overwrite=True
125+
)
126+
127+
# Resources
128+
conn.sql("""
129+
from (
130+
from df
131+
select
132+
project_name || '__' || module_name as service_id,
133+
to_json(unnest(resources)) as resource,
134+
generate_subscripts(resources, 1) AS index
135+
)
136+
select
137+
service_id || '_' || "index" as id,
138+
service_id,
139+
nullif(trim(resource->'title', '""'), 'null') as title,
140+
nullif(trim(resource->'type', '""'), 'null') as type,
141+
nullif(trim(resource->'uri', '""'), 'null') as uri,
142+
nullif(trim(resource->'description', '""'), 'null') as description,
143+
nullif(trim(resource->'access', '""'), 'null') as access,
144+
nullif(trim(resource->'internal_ref', '""'), 'null') as internal_ref,
145+
coalesce(try_cast(trim(resource->'external', '""') as bool), false) as external
146+
""").write_parquet(
147+
f"s3://{bucket}{prefix}/services_resource.parquet", overwrite=True
148+
)
149+
150+
# Related_projects
151+
conn.from_arrow(df).select("""
152+
project_name || '__' || module_name as from_service_id,
153+
unnest(related_projects) as to_service_id
154+
;
155+
""").write_parquet(
156+
f"s3://{bucket}{prefix}/services_servicerelated.parquet", overwrite=True
157+
)
158+
159+
160+
DASHBOARD_PREFIX = env("SERVICES_DASHBOARD_PREFIX", default="/dms/services")
161+
162+
163+
DASHBOARD_REPO = env("SERVICES_DASHBOARD_REPO", None)
164+
DASHBOARD_ORG = env("SERVICES_DASHBOARD_ORG", default="ninanor")
165+
166+
167+
@app.command(
168+
help="Produce a Homer Dashbord using the Miljødata Infrastructure as Code "
169+
"repository as data source"
170+
)
171+
def dashboard(
172+
org=SERVICES_ORG,
173+
repo: str | None = SERVICES_REPO,
174+
config_org=DASHBOARD_ORG,
175+
config_repo=DASHBOARD_REPO,
176+
bucket: str | None = AWS_BUCKET,
177+
endpoint: str | None = AWS_ENDPOINT,
178+
access_key: str | None = AWS_ACCESS_KEY,
179+
secret_key: str | None = AWS_SECRET_KEY,
180+
git_username=GIT_USERNAME,
181+
git_token=GIT_TOKEN,
182+
prefix=DASHBOARD_PREFIX,
183+
):
184+
s3 = s3fs.S3FileSystem(
185+
endpoint_url=f"https://{endpoint}",
186+
secret=secret_key,
187+
key=access_key,
188+
)
189+
190+
infrastucture_repo = fsspec.filesystem(
191+
"github",
192+
org=org,
193+
repo=repo,
194+
username=git_username,
195+
token=git_token,
196+
)
197+
198+
dashboard_repo = fsspec.filesystem(
199+
"github",
200+
org=config_org,
201+
repo=config_repo,
202+
username=git_username,
203+
token=git_token,
204+
)
205+
206+
with dashboard_repo.open("base-config.yml", "r") as base_conf_file:
207+
base_conf = yaml.load(base_conf_file, Loader=yaml.SafeLoader)
208+
209+
curated = OrderedDict()
210+
all_services = OrderedDict()
211+
212+
ICON = {
213+
"web": "fas fa-globe",
214+
"code": "fab fa-github",
215+
"data": "fas fa-database",
216+
"docs": "fas fa-book",
217+
"mobileApp": "fas fa-mobile",
218+
"template": "fas fa-copy",
219+
"scripts": "fas fa-wrench",
220+
"other": "",
221+
}
222+
TAG = {
223+
"permit": "is-danger",
224+
"private": "is-info",
225+
"public": "is-success",
226+
}
227+
228+
services_paths = infrastucture_repo.glob(path="services/*/*/metadata.yml")
229+
for spath in services_paths:
230+
with infrastucture_repo.open(spath, "r") as f:
231+
data = yaml.load(f, Loader=yaml.SafeLoader)
232+
_, project_name, module_name, _ = spath.split("/")
233+
for resource in data.get("resources"):
234+
if not resource["uri"].startswith("http") or resource.get(
235+
"external", False
236+
):
237+
print(resource)
238+
# skip non http resources and external resources
239+
continue
240+
241+
if group := resource.get("group"):
242+
if group not in curated:
243+
curated[group] = {
244+
"name": group,
245+
"items": [],
246+
}
247+
248+
curated[group]["items"].append(
249+
{
250+
"name": resource.get("title", data.get("title", "")),
251+
"subtitle": resource.get("description", ""),
252+
"url": resource["uri"],
253+
"tag": resource["access"],
254+
"tagstyle": TAG[resource["access"]],
255+
"target": "_blank",
256+
"icon": ICON[resource.get("type", "web")],
257+
}
258+
)
259+
260+
group_2 = f"{project_name}"
261+
if group_2 not in all_services:
262+
all_services[group_2] = {
263+
"name": group_2.capitalize(),
264+
"items": [],
265+
}
266+
267+
all_services[group_2]["items"].append(
268+
{
269+
"name": resource.get("title", data.get("title", "")),
270+
"subtitle": resource.get("description", ""),
271+
"url": resource["uri"],
272+
"tag": resource["access"],
273+
"tagstyle": TAG[resource["access"]],
274+
"target": "_blank",
275+
"icon": ICON[resource.get("type", "web")],
276+
}
277+
)
278+
279+
curated_conf = copy.deepcopy(base_conf)
280+
curated_conf["services"] = [group for _, group in curated.items()]
281+
curated_conf["links"].append({"name": "All services", "url": "#all"})
282+
283+
with s3.open(f"{bucket}{prefix}/curated.yml", "w") as f:
284+
yaml.dump(
285+
curated_conf,
286+
f,
287+
)
288+
289+
all_conf = copy.deepcopy(base_conf)
290+
all_conf["services"] = [group for _, group in all_services.items()]
291+
all_conf["links"].append({"name": "Curated services", "url": "/"})
292+
293+
with s3.open(f"{bucket}{prefix}/all.yml", "w") as f:
294+
yaml.dump(
295+
all_conf,
296+
f,
297+
)

‎uv.lock‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)