|
| 1 | +import requests |
| 2 | +import pandas as pd |
| 3 | +import geopandas as gpd |
| 4 | +import os |
| 5 | +import zipfile |
| 6 | + |
| 7 | +# === CONFIGURATION === |
| 8 | +SERVICE_URL = "https://kort.nunagis.gl/refserver/rest/services/Grunddataregistre/Adresseregister_offentlig/MapServer" |
| 9 | +BASE_LAYER_ID = 0 |
| 10 | +JOIN_LAYERS = { |
| 11 | + 1: {"join_field": "Vejkode", "return_field": "Vejnavn"}, |
| 12 | + 4: { |
| 13 | + "join_field": "KommuneKode", |
| 14 | + "return_field": "Kommunenavn", |
| 15 | + }, |
| 16 | + 3: { |
| 17 | + "join_field": "Lokalitetskode", |
| 18 | + "return_field": "Lokalitetsnavn", |
| 19 | + }, |
| 20 | +} |
| 21 | +OUT_FIELDS = "*" |
| 22 | +OUTPUT_CSV = "gl_countrywide.csv" |
| 23 | +PAGE_LIMIT = 1000 |
| 24 | + |
| 25 | + |
| 26 | +# === Helper Functions === |
| 27 | +def fetch_data(layer_id, out_fields=OUT_FIELDS): |
| 28 | + """Fetch all features from a layer with pagination, including SHAPE data.""" |
| 29 | + url = f"{SERVICE_URL}/{layer_id}/query" |
| 30 | + all_features = [] |
| 31 | + offset = 0 |
| 32 | + |
| 33 | + while True: |
| 34 | + params = { |
| 35 | + "where": "1=1", |
| 36 | + "outFields": out_fields, |
| 37 | + "f": "json", |
| 38 | + "returnGeometry": "true", # Ensure geometry (SHAPE) is included |
| 39 | + "resultOffset": offset, |
| 40 | + "resultRecordCount": PAGE_LIMIT, |
| 41 | + } |
| 42 | + r = requests.get(url, params=params) |
| 43 | + r.raise_for_status() |
| 44 | + response = r.json() |
| 45 | + features = response.get("features", []) |
| 46 | + if not features: |
| 47 | + break |
| 48 | + all_features.extend(features) |
| 49 | + offset += PAGE_LIMIT |
| 50 | + |
| 51 | + return pd.DataFrame([f["attributes"] for f in all_features]), all_features |
| 52 | + |
| 53 | + |
| 54 | +def fetch_join_data(layer_id, join_fields, return_field): |
| 55 | + """Fetch join data from a layer with pagination.""" |
| 56 | + out_fields = ",".join(join_fields + [return_field]) |
| 57 | + df, _ = fetch_data(layer_id, out_fields=out_fields) |
| 58 | + print(f"Join data columns for layer {layer_id}: {df.columns.tolist()}") |
| 59 | + return df |
| 60 | + |
| 61 | + |
| 62 | +def extract_geometry(features): |
| 63 | + """Extract geometry from the SHAPE column and convert to x, y columns.""" |
| 64 | + x_coords = [] |
| 65 | + y_coords = [] |
| 66 | + for feature in features: |
| 67 | + geometry = feature.get("geometry", {}) |
| 68 | + x_coords.append(geometry.get("x")) |
| 69 | + y_coords.append(geometry.get("y")) |
| 70 | + return x_coords, y_coords |
| 71 | + |
| 72 | + |
| 73 | +# === Main Script === |
| 74 | +if __name__ == "__main__": |
| 75 | + # Step 1: Fetch base layer data |
| 76 | + print("Fetching base layer data...") |
| 77 | + base_df, base_features = fetch_data(BASE_LAYER_ID) |
| 78 | + print(f"Base data columns: {base_df.columns.tolist()}") |
| 79 | + |
| 80 | + # Step 2: Extract geometry from SHAPE column and add x, y columns |
| 81 | + print("Extracting geometry from SHAPE column...") |
| 82 | + x_coords, y_coords = extract_geometry(base_features) |
| 83 | + base_df["x"] = x_coords |
| 84 | + base_df["y"] = y_coords |
| 85 | + |
| 86 | + # Step 3: Perform joins with other layers |
| 87 | + for layer_id, join_info in JOIN_LAYERS.items(): |
| 88 | + print(f"Joining with layer {layer_id} on {join_info['join_field']}...") |
| 89 | + |
| 90 | + # Handle special case for layer 1 |
| 91 | + if layer_id == 1: |
| 92 | + print("Layer 1 requires joining on both Vejkode and KommuneKode...") |
| 93 | + join_df = fetch_join_data( |
| 94 | + layer_id, ["Vejkode", "Kommunekode"], join_info["return_field"] |
| 95 | + ) |
| 96 | + # Rename Kommunekode to KommuneKode for consistency |
| 97 | + if "Kommunekode" in join_df.columns: |
| 98 | + print("Renaming Kommunekode to KommuneKode for merge...") |
| 99 | + join_df = join_df.rename(columns={"Kommunekode": "KommuneKode"}) |
| 100 | + # Drop duplicates to ensure one-to-one join |
| 101 | + join_df = join_df.drop_duplicates(subset=["Vejkode", "KommuneKode"]) |
| 102 | + base_df = base_df.merge( |
| 103 | + join_df, |
| 104 | + left_on=["Vejkode", "KommuneKode"], |
| 105 | + right_on=["Vejkode", "KommuneKode"], |
| 106 | + how="left", |
| 107 | + ) |
| 108 | + else: |
| 109 | + join_df = fetch_join_data( |
| 110 | + layer_id, [join_info["join_field"]], join_info["return_field"] |
| 111 | + ) |
| 112 | + if join_df.empty: |
| 113 | + print(f"No data returned for layer {layer_id}, skipping join...") |
| 114 | + continue |
| 115 | + |
| 116 | + print(f"Before merge - looking for '{join_info['join_field']}' in join_df") |
| 117 | + print(f"Base columns: {base_df.columns.tolist()}") |
| 118 | + print(f"Join columns: {join_df.columns.tolist()}") |
| 119 | + |
| 120 | + # Create a mapping of the original column name to the join field name |
| 121 | + if ( |
| 122 | + join_info["join_field"] == "Lokalitetskode" |
| 123 | + and "LokalitetsKode" in base_df.columns |
| 124 | + ): |
| 125 | + print("Renaming LokalitetsKode to Lokalitetskode for merge") |
| 126 | + base_df = base_df.rename(columns={"LokalitetsKode": "Lokalitetskode"}) |
| 127 | + elif ( |
| 128 | + join_info["join_field"] == "KommuneKode" |
| 129 | + and "Kommunekode" in join_df.columns |
| 130 | + ): |
| 131 | + print("Renaming Kommunekode to KommuneKode for merge") |
| 132 | + join_df = join_df.rename(columns={"Kommunekode": "KommuneKode"}) |
| 133 | + |
| 134 | + base_df = base_df.merge( |
| 135 | + join_df, |
| 136 | + left_on=join_info["join_field"], |
| 137 | + right_on=join_info["join_field"], |
| 138 | + how="left", |
| 139 | + ) |
| 140 | + |
| 141 | + # Step 4: Save as CSV |
| 142 | + print(f"Saving output to {OUTPUT_CSV}...") |
| 143 | + base_df.to_csv(OUTPUT_CSV, index=False) |
| 144 | + |
| 145 | + # Step 5: Zip the CSV file |
| 146 | + zip_output = f"{OUTPUT_CSV}.zip" |
| 147 | + print(f"Zipping output to {zip_output}...") |
| 148 | + with zipfile.ZipFile(zip_output, "w", zipfile.ZIP_DEFLATED) as zipf: |
| 149 | + zipf.write(OUTPUT_CSV, arcname=os.path.basename(OUTPUT_CSV)) |
| 150 | + print("✅ Zipping complete!") |
| 151 | + |
| 152 | + # Optional: Remove the original CSV file after zipping |
| 153 | + os.remove(OUTPUT_CSV) |
| 154 | + print(f"Removed original CSV file: {OUTPUT_CSV}") |
| 155 | + print("✅ Done!") |
0 commit comments