Skip to content
Merged

Odk #32

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions truecover-app/src/components/ExportLocationsModal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
// Entity export workflow state
const [exportWorkflowId, setExportWorkflowId] = useState<string | null>(null);
const [exportProgress, setExportProgress] = useState<{
total_pixels: number;
created_pixels: number;
current_quadkey: string;
total_entities: number;
created_entities: number;
current_label: string;
} | null>(null);
const [isExporting, setIsExporting] = useState(false);
const [pixelGeometryType, setPixelGeometryType] = useState<'centroid' | 'boundary'>('centroid');
Expand Down Expand Up @@ -748,8 +748,11 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
)}
</div>

{/* Pixel Geometry Type Selection */}
<div>
{/* Pixel Geometry Type Selection - only shown when pixel rounds are selected */}
{selectedRoundIds.some(id => {
const round = rounds.find(r => r.id === id);
return round?.sampling_target === 'pixels';
}) && <div>
<label className="block text-sm font-mono font-bold text-tactical-text-primary uppercase tracking-wider mb-2">
Pixel Geometry Type
</label>
Expand Down Expand Up @@ -787,7 +790,7 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
</div>
</label>
</div>
</div>
</div>}
</>
)}

Expand Down Expand Up @@ -826,12 +829,12 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
<div className="flex justify-between">
<span className="text-tactical-text-dim">Progress:</span>
<span className="text-tactical-text-primary font-bold">
{exportProgress.created_pixels} / {exportProgress.total_pixels} pixels
{exportProgress.created_entities} / {exportProgress.total_entities} entities
</span>
</div>
{exportProgress.current_quadkey && (
{exportProgress.current_label && (
<div className="text-xs text-tactical-text-dim">
Current: {exportProgress.current_quadkey}
Current: {exportProgress.current_label}
</div>
)}
</div>
Expand Down
5 changes: 5 additions & 0 deletions truecover-app/src/components/ProjectSettings.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ const ProjectSettings: React.FC<ProjectSettingsProps> = ({
setError(null);
setOnaProjects([]);
setOnaEntityLists([]);

// Auto-load entity lists when project is configured but no entity list is set
if (project.ona_project_id && !project.ona_entity_list_id) {
loadEntityListsForProject(project.ona_project_id);
}
}
}, [project, isOpen]);

Expand Down
10 changes: 5 additions & 5 deletions truecover-app/src/services/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -962,15 +962,15 @@ export const entityExportApi = {
workflow_id: string;
status: string;
progress?: {
total_pixels: number;
created_pixels: number;
current_quadkey: string;
total_entities: number;
created_entities: number;
current_label: string;
error_message: string | null;
};
result?: {
success: boolean;
total_pixels: number;
created_pixels: number;
total_entities: number;
created_entities: number;
message: string;
};
error?: string;
Expand Down
3 changes: 1 addition & 2 deletions truecover-backend/db/migrations/add_sampled_only_filter.sql
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,14 @@ BEGIN
SELECT
p.quadkey,
p.geometry,
(pm.metadata->>'population')::numeric AS population,
p.population,
lc.building_count
FROM pixels p
JOIN pixel_area pa ON p.quadkey = pa.quadkey
JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey
AND cp.campaign_id = target_campaign_id
AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id)
LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
LEFT JOIN LATERAL (
SELECT COUNT(*)::integer AS building_count
FROM locations l
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,11 @@ BEGIN
SELECT
p.quadkey,
p.geometry,
(pm.metadata->>'population')::numeric AS population,
p.population,
lc.building_count
FROM pixels p
JOIN pixel_area pa ON p.quadkey = pa.quadkey
JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
LEFT JOIN LATERAL (
SELECT COUNT(*)::integer AS building_count
FROM locations l
Expand Down
6 changes: 2 additions & 4 deletions truecover-backend/routes/campaigns.py
Original file line number Diff line number Diff line change
Expand Up @@ -581,10 +581,9 @@ def compute_pixels_for_area(user, area_id):
WITH pixel_stats AS (
SELECT
COUNT(*) as pixel_count,
COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population
COALESCE(SUM(p.population), 0) as total_population
FROM pixel_area pa
JOIN pixels p ON pa.quadkey = p.quadkey
LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
WHERE pa.campaign_area_id = %s
),
location_counts AS (
Expand Down Expand Up @@ -696,10 +695,9 @@ def compute_all_pixels_for_campaign(user, campaign_id):
WITH pixel_stats AS (
SELECT
COUNT(*) as pixel_count,
COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population
COALESCE(SUM(p.population), 0) as total_population
FROM pixel_area pa
JOIN pixels p ON pa.quadkey = p.quadkey
LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
WHERE pa.campaign_area_id = %s
),
location_counts AS (
Expand Down
28 changes: 26 additions & 2 deletions truecover-backend/routes/entity_export.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
# ABOUTME: Routes for exporting pixel coverage data to ODK entity lists
# ABOUTME: Routes for exporting coverage data (pixels and locations) to ODK entity lists
# ABOUTME: Handles starting and monitoring Temporal workflows for entity creation

from flask import Blueprint, jsonify, request
from auth.middleware import require_auth
from auth.helpers import check_campaign_access
from db.connection import get_db_connection, return_db_connection

entity_export_bp = Blueprint('entity_export', __name__)

Expand Down Expand Up @@ -39,6 +40,29 @@ def start_entity_export_workflow(user, campaign_id):
if not project_id:
return jsonify({'error': 'project_id is required'}), 400

# Split rounds by sampling_target
conn = get_db_connection()
cursor = conn.cursor()
try:
placeholders = ','.join(['%s'] * len(round_ids))
cursor.execute(f"""
SELECT id, COALESCE(sampling_target, 'locations') as sampling_target
FROM rounds
WHERE id IN ({placeholders})
""", tuple(round_ids))

pixel_round_ids = []
location_round_ids = []
for row in cursor.fetchall():
rid, target = str(row[0]), row[1]
if target == 'pixels':
pixel_round_ids.append(rid)
else:
location_round_ids.append(rid)
finally:
cursor.close()
return_db_connection(conn)

# Generate workflow ID
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
workflow_id = f"entity-export-{campaign_id}-{timestamp}"
Expand All @@ -48,7 +72,7 @@ async def start_workflow():
client = await get_temporal_client()
handle = await client.start_workflow(
EntityExportWorkflow.run,
args=[campaign_id, indicator_id, round_ids, project_id, geometry_type],
args=[campaign_id, indicator_id, pixel_round_ids, location_round_ids, project_id, geometry_type],
id=workflow_id,
task_queue="truecover-tasks"
)
Expand Down
17 changes: 14 additions & 3 deletions truecover-backend/temporal/activities/enrichment.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,17 @@ async def enrich_area_pixels(
metadata = pixel_metadata.metadata || EXCLUDED.metadata,
updated_at = NOW()
""", updates)

# Also write to pixels.population column for population data
if metadata_field_name == 'population':
pop_updates = [
(json.loads(metadata_json)[metadata_field_name], quadkey)
for quadkey, metadata_json in updates
]
cursor.executemany("""
UPDATE pixels SET population = %s WHERE quadkey = %s
""", pop_updates)

conn.commit()
total_updated += len(updates)

Expand All @@ -354,11 +365,11 @@ async def enrich_area_pixels(
SET cached_population = sub.total_pop
FROM (
SELECT pa.campaign_area_id,
COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_pop
COALESCE(SUM(p.population), 0) as total_pop
FROM pixel_area pa
JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
JOIN pixels p ON pa.quadkey = p.quadkey
JOIN campaign_areas ca2 ON pa.campaign_area_id = ca2.id
WHERE ca2.campaign_id = %s AND pm.metadata ? 'population'
WHERE ca2.campaign_id = %s
GROUP BY pa.campaign_area_id
) sub
WHERE ca.id = sub.campaign_area_id
Expand Down
101 changes: 86 additions & 15 deletions truecover-backend/temporal/activities/entity_export.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# ABOUTME: Temporal activities for exporting pixel coverage data to ODK entity lists
# ABOUTME: Handles fetching pixel data and creating entities via Ona API
# ABOUTME: Temporal activities for exporting coverage data to ODK entity lists
# ABOUTME: Handles fetching pixel and location data and creating entities via Ona API

from temporalio import activity
from typing import List, Dict, Any
Expand Down Expand Up @@ -85,7 +85,7 @@ async def fetch_pixel_coverage_activity(
p.adm4_pcode,
pc.rounds
FROM coverage_pixel pc
JOIN pixels p ON p.quadkey = pc.quadkey AND p.campaign_id = pc.campaign_id
JOIN pixels p ON p.quadkey = pc.quadkey
WHERE pc.campaign_id = %s
AND pc.indicator_id = %s
AND pc.rounds && %s::integer[]
Expand All @@ -111,19 +111,84 @@ async def fetch_pixel_coverage_activity(
return_db_connection(conn)


@activity.defn
async def fetch_location_coverage_activity(
campaign_id: str,
indicator_id: str,
round_ids: List[str]
) -> List[Dict[str, Any]]:
"""
Fetch location coverage data filtered by selected rounds.

Returns list of location coverage records with location details.
"""
conn = get_db_connection()
cursor = conn.cursor()

try:
# Get round numbers from round IDs
placeholders = ','.join(['%s'] * len(round_ids))
cursor.execute(f"""
SELECT round_number
FROM rounds
WHERE id IN ({placeholders})
""", tuple(round_ids))

round_numbers = [row[0] for row in cursor.fetchall()]

if not round_numbers:
return []

cursor.execute("""
SELECT DISTINCT
c.id,
l.external_id,
l.latitude,
l.longitude,
l.quadkey,
c.rounds
FROM coverage c
JOIN locations l ON l.id = c.location_id
WHERE c.campaign_id = %s
AND c.indicator_id = %s
AND c.rounds && %s::integer[]
ORDER BY l.external_id
""", (campaign_id, indicator_id, round_numbers))

locations = []
for row in cursor.fetchall():
cov_id, external_id, latitude, longitude, quadkey, rounds = row
locations.append({
'id': str(cov_id),
'external_id': external_id or '',
'latitude': float(latitude) if latitude else None,
'longitude': float(longitude) if longitude else None,
'quadkey': quadkey or '',
'rounds': rounds or []
})

return locations

finally:
cursor.close()
return_db_connection(conn)


@activity.defn
async def create_odk_entity_activity(
project_id: str,
pixel_data: Dict[str, Any],
geometry_type: str = 'centroid'
entity_data: Dict[str, Any],
geometry_type: str = 'centroid',
entity_type: str = 'pixel'
) -> Dict[str, Any]:
"""
Create a single ODK entity via Ona API.

Args:
project_id: Project ID to get ODK credentials
pixel_data: Pixel data dict with id, quadkey, lat, lng, adm4_pcode
entity_data: Entity data dict (pixel or location fields)
geometry_type: 'centroid' or 'boundary' - how to represent pixel geometry
entity_type: 'pixel' or 'location'

Returns:
Dict with success status and created entity info
Expand Down Expand Up @@ -154,21 +219,27 @@ async def create_odk_entity_activity(
# Remove trailing slash from host_url
host_url = host_url.rstrip('/')

# Format geometry based on project setting
if geometry_type == 'boundary':
# Use pixel boundary (geoshape polygon)
geometry = get_pixel_boundary_coords(pixel_data['quadkey'])
if entity_type == 'location':
# Locations always use point geometry
label = entity_data['external_id']
geometry = f"{entity_data['latitude']} {entity_data['longitude']} 0 0"
details = entity_data['quadkey']
else:
# Use pixel centroid (geopoint)
geometry = f"{pixel_data['latitude']} {pixel_data['longitude']} 0 0"
# Pixels use configurable geometry
label = entity_data['quadkey']
if geometry_type == 'boundary':
geometry = get_pixel_boundary_coords(entity_data['quadkey'])
else:
geometry = f"{entity_data['latitude']} {entity_data['longitude']} 0 0"
details = entity_data['adm4_pcode']

# Build entity payload
entity_payload = {
'label': pixel_data['quadkey'],
'label': label,
'data': {
'geometry': geometry,
'status': 'not_visited',
'details': pixel_data['adm4_pcode']
'details': details
}
}

Expand Down Expand Up @@ -199,7 +270,7 @@ async def create_odk_entity_activity(

return {
'success': True,
'quadkey': pixel_data['quadkey'],
'label': label,
'entity_uuid': entity_result.get('uuid')
}

Expand Down
8 changes: 2 additions & 6 deletions truecover-backend/temporal/activities/rounds.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,8 @@ async def fetch_coverage_for_sampling(
pop_filter = ""
pop_params = []

if min_population is not None and population_field:
import re
if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', population_field):
raise ValueError(f"Invalid population field name: {population_field}")
pop_join = "LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey"
pop_filter = f"AND (pm.metadata->>'{population_field}')::float >= %s"
if min_population is not None:
pop_filter = "AND p.population >= %s"
pop_params = [min_population]

if allow_revisit:
Expand Down
Loading
Loading