Skip to content

Commit 3a4fd83

Browse files
mbergmberg
committed
Add selected pixels toggle, fix buildings colors, improve Overture import
- Add "Selected Pixels Only" toggle to map (defaults to on) with sampled_only filter in pixels_by_campaign SQL function - Fix buildings layer colors from blue to white with low opacity - Fix green buildings bug: handle NULL rounds in Mapbox expressions - Fix orphaned replacement pixels: check primary building count before creating replacements, filter orphans in SQL display - Overture import: skip fetch when buildings exist, populate coverage via spatial boundary query instead of passing IDs through Temporal (avoids gRPC 4MB limit) - Prevent duplicate import workflows with running workflow check - Include quadkey in coverage records created during import
1 parent e3b149a commit 3a4fd83

7 files changed

Lines changed: 357 additions & 50 deletions

File tree

‎truecover-app/src/components/MapView.tsx‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
101101
const [visibleAdminLevels, setVisibleAdminLevels] = useState<number[]>([0, 1, 2, 3, 4]);
102102
const [hoveredAdminId, setHoveredAdminId] = useState<string | null>(null);
103103
const [showBuildings, setShowBuildings] = useState<boolean>(true);
104+
const [showSelectedPixelsOnly, setShowSelectedPixelsOnly] = useState<boolean>(true);
104105
const [adminBoundariesCollapsed, setAdminBoundariesCollapsed] = useState<boolean>(true);
105106
const [showPixelGenerateModal, setShowPixelGenerateModal] = useState<boolean>(false);
106107
const [pendingAdminPixelGen, setPendingAdminPixelGen] = useState<{
@@ -127,7 +128,7 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
127128
// Update tile version when filter states change to bust cache
128129
React.useEffect(() => {
129130
setTileVersion(Date.now());
130-
}, [showSampled, interpolationMode, highlightRounds, selectedMetadataField, metadataVisualizationMode, visibleAdminLevels]);
131+
}, [showSampled, interpolationMode, highlightRounds, selectedMetadataField, metadataVisualizationMode, visibleAdminLevels, showSelectedPixelsOnly]);
131132

132133
// Use locations data if in locations mode, otherwise use regular data
133134
const primaryData = mode === 'locations' && locations ? locations : data;
@@ -955,8 +956,11 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
955956
return true; // Not filtering by rounds
956957
}
957958

958-
// No rounds = empty string "{}"
959-
const hasAnyRounds = ['!=', ['get', 'rounds'], '{}'];
959+
// rounds is NULL when no coverage record exists, or "{}" when unsampled
960+
const hasAnyRounds = ['all',
961+
['has', 'rounds'],
962+
['!=', ['get', 'rounds'], '{}']
963+
];
960964

961965
// If no specific rounds selected, show all locations with any rounds
962966
if (!highlightRounds || highlightRounds.length === 0) {
@@ -1320,8 +1324,8 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
13201324
source-layer="building"
13211325
minzoom={12}
13221326
paint={{
1323-
'fill-color': '#4393c3',
1324-
'fill-opacity': 0.2
1327+
'fill-color': '#ffffff',
1328+
'fill-opacity': 0.1
13251329
}}
13261330
/>
13271331
<Layer
@@ -1330,7 +1334,7 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
13301334
source-layer="building"
13311335
minzoom={12}
13321336
paint={{
1333-
'line-color': '#2166ac',
1337+
'line-color': '#ffffff',
13341338
'line-width': 1
13351339
}}
13361340
/>
@@ -1451,7 +1455,7 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
14511455
<Source
14521456
id="pixels-source"
14531457
type="vector"
1454-
tiles={[`${MARTIN_URL}/pixels_by_campaign/{z}/{x}/{y}?campaign_id=${campaignId}&indicator_id=${indicatorId || ''}&metadata_field=${selectedMetadataField || ''}&v=${pixelVersion || '0'}&t=${tileVersion}`]}
1458+
tiles={[`${MARTIN_URL}/pixels_by_campaign/{z}/{x}/{y}?campaign_id=${campaignId}&indicator_id=${indicatorId || ''}&metadata_field=${selectedMetadataField || ''}&sampled_only=${showSelectedPixelsOnly}&v=${pixelVersion || '0'}&t=${tileVersion}`]}
14551459
minzoom={0}
14561460
maxzoom={24}
14571461
>
@@ -2316,6 +2320,17 @@ const MapView: React.FC<MapViewProps> = ({ data, selectedData, locations, mode =
23162320
Buildings
23172321
</span>
23182322
</label>
2323+
<label className="flex items-center mb-1 cursor-pointer">
2324+
<input
2325+
type="checkbox"
2326+
checked={showSelectedPixelsOnly}
2327+
onChange={(e) => setShowSelectedPixelsOnly(e.target.checked)}
2328+
className="mr-2"
2329+
/>
2330+
<span className="font-mono text-xs text-tactical-text-primary">
2331+
Selected Pixels Only
2332+
</span>
2333+
</label>
23192334
</div>
23202335

23212336
<div className="mt-3 pt-3 border-t border-tactical-border-medium">

‎truecover-backend/db/migrations/add_sampled_only_filter.sql‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
-- ABOUTME: Adds sampled_only query parameter to pixels_by_campaign tile function.
2+
-- ABOUTME: When sampled_only=true, only pixels with assigned rounds are returned.
3+
CREATE OR REPLACE FUNCTION pixels_by_campaign(z integer, x integer, y integer, query_params json)
4+
RETURNS bytea AS $$
5+
DECLARE
6+
mvt_polygons bytea;
7+
mvt_points bytea;
8+
mvt_labels bytea;
9+
target_campaign_id uuid;
10+
target_indicator_id uuid;
11+
metadata_field text;
12+
sampled_only boolean;
13+
BEGIN
14+
target_campaign_id := (query_params->>'campaign_id')::uuid;
15+
target_indicator_id := NULLIF(query_params->>'indicator_id', '')::uuid;
16+
metadata_field := query_params->>'metadata_field';
17+
sampled_only := COALESCE((query_params->>'sampled_only')::boolean, false);
18+
19+
IF target_campaign_id IS NULL THEN
20+
RETURN NULL;
21+
END IF;
22+
23+
-- Generate MVT tile for pixel polygons
24+
SELECT INTO mvt_polygons ST_AsMVT(tile, 'pixels', 4096, 'geom')
25+
FROM (
26+
SELECT
27+
ST_AsMVTGeom(
28+
ST_Transform(p.geometry, 3857),
29+
ST_TileEnvelope(z, x, y),
30+
4096, 64, true
31+
) AS geom,
32+
p.quadkey,
33+
p.level,
34+
p.latitude,
35+
p.longitude,
36+
cp.prevalence_prediction,
37+
cp.prevalence_bci_width,
38+
cp.n_trials,
39+
cp.n_covered,
40+
cp.rounds,
41+
(cp.replacement_for IS NOT NULL) AS is_replacement,
42+
pm.metadata,
43+
CASE
44+
WHEN metadata_field IS NOT NULL AND pm.metadata IS NOT NULL THEN
45+
(pm.metadata->>metadata_field)::numeric
46+
ELSE NULL
47+
END AS metadata_value
48+
FROM pixels p
49+
JOIN pixel_area pa ON p.quadkey = pa.quadkey
50+
JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
51+
LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey
52+
AND cp.campaign_id = target_campaign_id
53+
AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id)
54+
LEFT JOIN coverage_pixel primary_cp ON cp.replacement_for = primary_cp.id
55+
LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
56+
WHERE ca.campaign_id = target_campaign_id
57+
AND p.geometry && ST_Transform(ST_TileEnvelope(z, x, y), 4326)
58+
AND (NOT sampled_only OR (cp.rounds IS NOT NULL AND cp.rounds != '{}'))
59+
-- Hide replacement pixels whose primary has no rounds
60+
AND (cp.replacement_for IS NULL
61+
OR (primary_cp.rounds IS NOT NULL AND primary_cp.rounds != '{}'))
62+
) as tile
63+
WHERE geom IS NOT NULL;
64+
65+
-- Generate label points at pixel corners for zoom 16+ display.
66+
-- Uses a CTE to scan the pixels table once, then generates both corner points.
67+
IF z >= 16 THEN
68+
SELECT INTO mvt_labels ST_AsMVT(tile, 'pixels_labels', 4096, 'geom')
69+
FROM (
70+
WITH pixel_data AS (
71+
SELECT
72+
p.quadkey,
73+
p.geometry,
74+
(pm.metadata->>'population')::numeric AS population,
75+
lc.building_count
76+
FROM pixels p
77+
JOIN pixel_area pa ON p.quadkey = pa.quadkey
78+
JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
79+
LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey
80+
AND cp.campaign_id = target_campaign_id
81+
AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id)
82+
LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
83+
LEFT JOIN LATERAL (
84+
SELECT COUNT(*)::integer AS building_count
85+
FROM locations l
86+
WHERE l.quadkey = p.quadkey
87+
) lc ON true
88+
WHERE ca.campaign_id = target_campaign_id
89+
AND p.geometry && ST_Transform(ST_TileEnvelope(z, x, y), 4326)
90+
AND (NOT sampled_only OR (cp.rounds IS NOT NULL AND cp.rounds != '{}'))
91+
)
92+
-- Top-left corner (quadkey label)
93+
SELECT
94+
ST_AsMVTGeom(
95+
ST_Transform(ST_SetSRID(ST_MakePoint(ST_XMin(pd.geometry), ST_YMax(pd.geometry)), 4326), 3857),
96+
ST_TileEnvelope(z, x, y),
97+
4096, 64, true
98+
) AS geom,
99+
'quadkey' AS label_type,
100+
pd.quadkey,
101+
NULL::numeric AS population,
102+
NULL::integer AS building_count
103+
FROM pixel_data pd
104+
UNION ALL
105+
-- Bottom-right corner (stats label)
106+
SELECT
107+
ST_AsMVTGeom(
108+
ST_Transform(ST_SetSRID(ST_MakePoint(ST_XMax(pd.geometry), ST_YMin(pd.geometry)), 4326), 3857),
109+
ST_TileEnvelope(z, x, y),
110+
4096, 64, true
111+
) AS geom,
112+
'stats' AS label_type,
113+
pd.quadkey,
114+
pd.population,
115+
pd.building_count
116+
FROM pixel_data pd
117+
) as tile
118+
WHERE geom IS NOT NULL;
119+
END IF;
120+
121+
-- Only generate centroids when metadata circle visualization is needed
122+
IF metadata_field IS NOT NULL AND metadata_field != '' THEN
123+
SELECT INTO mvt_points ST_AsMVT(tile, 'pixels_centroids', 4096, 'geom')
124+
FROM (
125+
SELECT
126+
ST_AsMVTGeom(
127+
ST_Transform(ST_SetSRID(ST_MakePoint(p.longitude, p.latitude), 4326), 3857),
128+
ST_TileEnvelope(z, x, y),
129+
4096, 64, true
130+
) AS geom,
131+
p.quadkey,
132+
p.level,
133+
CASE
134+
WHEN pm.metadata IS NOT NULL THEN
135+
(pm.metadata->>metadata_field)::numeric
136+
ELSE NULL
137+
END AS metadata_value
138+
FROM pixels p
139+
JOIN pixel_area pa ON p.quadkey = pa.quadkey
140+
JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
141+
LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey
142+
AND cp.campaign_id = target_campaign_id
143+
AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id)
144+
LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
145+
WHERE ca.campaign_id = target_campaign_id
146+
AND p.geometry && ST_Transform(ST_TileEnvelope(z, x, y), 4326)
147+
AND (NOT sampled_only OR (cp.rounds IS NOT NULL AND cp.rounds != '{}'))
148+
) as tile
149+
WHERE geom IS NOT NULL;
150+
END IF;
151+
152+
RETURN COALESCE(mvt_polygons, ''::bytea) || COALESCE(mvt_points, ''::bytea) || COALESCE(mvt_labels, ''::bytea);
153+
END
154+
$$ LANGUAGE plpgsql STABLE STRICT PARALLEL SAFE;

‎truecover-backend/routes/admin_boundaries.py‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -685,24 +685,42 @@ def import_overture_buildings_async(user, pcode):
685685
check_campaign_access(user['id'], campaign_id)
686686

687687
try:
688-
# Generate workflow ID
689-
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
690-
workflow_id = f"overture-import-{pcode}-{campaign_id}-{timestamp}"
691-
692-
# Start the workflow
693-
async def start_workflow():
688+
# Check for already-running import for this pcode
689+
async def check_and_start():
694690
client = await get_temporal_client()
695-
handle = await client.start_workflow(
691+
692+
# Search for running workflows matching this pcode
693+
prefix = f"overture-import-{pcode}-"
694+
running = client.list_workflows(
695+
f'WorkflowType = "OvertureImportWorkflow" '
696+
f'AND ExecutionStatus = "Running"'
697+
)
698+
async for wf in running:
699+
if wf.id.startswith(prefix):
700+
return None, wf.id # Already running
701+
702+
# No running workflow — start one
703+
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
704+
workflow_id = f"overture-import-{pcode}-{campaign_id}-{timestamp}"
705+
await client.start_workflow(
696706
OvertureImportWorkflow.run,
697707
args=[pcode, campaign_id, geometry],
698708
id=workflow_id,
699709
task_queue="truecover-tasks"
700710
)
711+
return workflow_id, None
701712

702-
run_async(start_workflow())
713+
new_id, existing_id = run_async(check_and_start())
714+
715+
if existing_id:
716+
return jsonify({
717+
'error': f'Import already running for {pcode}',
718+
'workflow_id': existing_id,
719+
'status': 'already_running'
720+
}), 409
703721

704722
return jsonify({
705-
'workflow_id': workflow_id,
723+
'workflow_id': new_id,
706724
'status': 'started'
707725
}), 202
708726

‎truecover-backend/temporal/activities/cluster_sampling.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1145,7 +1145,8 @@ async def sample_buildings_within_pixels(
11451145
""", (all_quadkeys,))
11461146
quadkey_building_count = {row[0]: row[1] for row in cursor.fetchall()}
11471147

1148-
# Phase 3: Filter pixels — walk in priority order, keep those meeting threshold
1148+
# Phase 3: Filter pixels — walk in priority order, keep those meeting threshold.
1149+
# buildings_per_pixel of 0 means no minimum — accept all pixels.
11491150
qualified_pixel_ids = []
11501151
qualified_quadkeys = []
11511152
skipped = 0
@@ -1158,7 +1159,7 @@ async def sample_buildings_within_pixels(
11581159
skipped += 1
11591160
continue
11601161
bcount = quadkey_building_count.get(qk, 0)
1161-
if bcount >= buildings_per_pixel:
1162+
if buildings_per_pixel == 0 or bcount >= buildings_per_pixel:
11621163
qualified_pixel_ids.append(pixel_id)
11631164
qualified_quadkeys.append(qk)
11641165
else:
@@ -1338,7 +1339,8 @@ async def sample_buildings_within_pixels(
13381339
'pixels_assigned': len(qualified_pixel_ids),
13391340
'pixels_skipped': skipped,
13401341
'buildings_selected': total_selected,
1341-
'pixels_with_buildings': pixels_with_buildings
1342+
'pixels_with_buildings': pixels_with_buildings,
1343+
'qualified_pixel_ids': qualified_pixel_ids
13421344
}
13431345

13441346
finally:
@@ -1536,13 +1538,27 @@ async def create_replacement_pixels(
15361538
""", (campaign_id,))
15371539
already_sampled_quadkeys = {row[0] for row in cursor.fetchall()}
15381540

1541+
# Count buildings in primary pixels to skip those below threshold
1542+
primary_quadkeys = list(pixel_id_to_quadkey.values())
1543+
primary_building_counts = {}
1544+
if min_building_count > 0 and primary_quadkeys:
1545+
cursor.execute("""
1546+
SELECT quadkey, COUNT(*) FROM locations
1547+
WHERE quadkey = ANY(%s)
1548+
GROUP BY quadkey
1549+
""", (primary_quadkeys,))
1550+
primary_building_counts = {row[0]: row[1] for row in cursor.fetchall()}
1551+
15391552
# Compute all neighbor quadkeys for all primary pixels
15401553
neighbor_candidates = {} # primary_pixel_id -> list of neighbor quadkeys
15411554
all_neighbor_quadkeys = set()
15421555
for pixel_id in primary_pixel_ids:
15431556
qk = pixel_id_to_quadkey.get(pixel_id)
15441557
if not qk:
15451558
continue
1559+
# Skip primaries that don't meet the building count threshold
1560+
if min_building_count > 0 and primary_building_counts.get(qk, 0) < min_building_count:
1561+
continue
15461562
tile = mercantile.quadkey_to_tile(qk)
15471563
neighbors = []
15481564
for dx in (-1, 0, 1):

0 commit comments

Comments
 (0)