Merge pull request #32 from onaio/odk

Odk

Author: mberg

truecover-app/src/components/ExportLocationsModal.tsx

@@ -42,9 +42,9 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
   // Entity export workflow state
   const [exportWorkflowId, setExportWorkflowId] = useState<string | null>(null);
   const [exportProgress, setExportProgress] = useState<{
-    total_pixels: number;
-    created_pixels: number;
-    current_quadkey: string;
+    total_entities: number;
+    created_entities: number;
+    current_label: string;
   } | null>(null);
   const [isExporting, setIsExporting] = useState(false);
   const [pixelGeometryType, setPixelGeometryType] = useState<'centroid' | 'boundary'>('centroid');
@@ -748,8 +748,11 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
               )}
             </div>
 
-            {/* Pixel Geometry Type Selection */}
-            <div>
+            {/* Pixel Geometry Type Selection - only shown when pixel rounds are selected */}
+            {selectedRoundIds.some(id => {
+              const round = rounds.find(r => r.id === id);
+              return round?.sampling_target === 'pixels';
+            }) && <div>
               <label className="block text-sm font-mono font-bold text-tactical-text-primary uppercase tracking-wider mb-2">
                 Pixel Geometry Type
               </label>
@@ -787,7 +790,7 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
                   </div>
                 </label>
               </div>
-            </div>
+            </div>}
           </>
         )}
 
@@ -826,12 +829,12 @@ const ExportLocationsModal: React.FC<ExportLocationsModalProps> = ({
               <div className="flex justify-between">
                 <span className="text-tactical-text-dim">Progress:</span>
                 <span className="text-tactical-text-primary font-bold">
-                  {exportProgress.created_pixels} / {exportProgress.total_pixels} pixels
+                  {exportProgress.created_entities} / {exportProgress.total_entities} entities
                 </span>
               </div>
-              {exportProgress.current_quadkey && (
+              {exportProgress.current_label && (
                 <div className="text-xs text-tactical-text-dim">
-                  Current: {exportProgress.current_quadkey}
+                  Current: {exportProgress.current_label}
                 </div>
               )}
             </div>

truecover-app/src/components/ProjectSettings.tsx

@@ -63,6 +63,11 @@ const ProjectSettings: React.FC<ProjectSettingsProps> = ({
       setError(null);
       setOnaProjects([]);
       setOnaEntityLists([]);
+
+      // Auto-load entity lists when project is configured but no entity list is set
+      if (project.ona_project_id && !project.ona_entity_list_id) {
+        loadEntityListsForProject(project.ona_project_id);
+      }
     }
   }, [project, isOpen]);
 

truecover-app/src/services/api.ts

@@ -962,15 +962,15 @@ export const entityExportApi = {
     workflow_id: string;
     status: string;
     progress?: {
-      total_pixels: number;
-      created_pixels: number;
-      current_quadkey: string;
+      total_entities: number;
+      created_entities: number;
+      current_label: string;
       error_message: string | null;
     };
     result?: {
       success: boolean;
-      total_pixels: number;
-      created_pixels: number;
+      total_entities: number;
+      created_entities: number;
       message: string;
     };
     error?: string;

truecover-backend/db/migrations/add_sampled_only_filter.sql

@@ -71,15 +71,14 @@ BEGIN
                 SELECT
                     p.quadkey,
                     p.geometry,
-                    (pm.metadata->>'population')::numeric AS population,
+                    p.population,
                     lc.building_count
                 FROM pixels p
                 JOIN pixel_area pa ON p.quadkey = pa.quadkey
                 JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
                 LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey
                     AND cp.campaign_id = target_campaign_id
                     AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id)
-                LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
                 LEFT JOIN LATERAL (
                     SELECT COUNT(*)::integer AS building_count
                     FROM locations l

truecover-backend/db/migrations/optimize_pixels_by_campaign.sql

@@ -64,12 +64,11 @@ BEGIN
                 SELECT
                     p.quadkey,
                     p.geometry,
-                    (pm.metadata->>'population')::numeric AS population,
+                    p.population,
                     lc.building_count
                 FROM pixels p
                 JOIN pixel_area pa ON p.quadkey = pa.quadkey
                 JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
-                LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
                 LEFT JOIN LATERAL (
                     SELECT COUNT(*)::integer AS building_count
                     FROM locations l

truecover-backend/routes/campaigns.py

@@ -581,10 +581,9 @@ def compute_pixels_for_area(user, area_id):
             WITH pixel_stats AS (
                 SELECT
                     COUNT(*) as pixel_count,
-                    COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population
+                    COALESCE(SUM(p.population), 0) as total_population
                 FROM pixel_area pa
                 JOIN pixels p ON pa.quadkey = p.quadkey
-                LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
                 WHERE pa.campaign_area_id = %s
             ),
             location_counts AS (
@@ -696,10 +695,9 @@ def compute_all_pixels_for_campaign(user, campaign_id):
                 WITH pixel_stats AS (
                     SELECT
                         COUNT(*) as pixel_count,
-                        COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population
+                        COALESCE(SUM(p.population), 0) as total_population
                     FROM pixel_area pa
                     JOIN pixels p ON pa.quadkey = p.quadkey
-                    LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
                     WHERE pa.campaign_area_id = %s
                 ),
                 location_counts AS (

truecover-backend/routes/entity_export.py

@@ -1,9 +1,10 @@
-# ABOUTME: Routes for exporting pixel coverage data to ODK entity lists
+# ABOUTME: Routes for exporting coverage data (pixels and locations) to ODK entity lists
 # ABOUTME: Handles starting and monitoring Temporal workflows for entity creation
 
 from flask import Blueprint, jsonify, request
 from auth.middleware import require_auth
 from auth.helpers import check_campaign_access
+from db.connection import get_db_connection, return_db_connection
 
 entity_export_bp = Blueprint('entity_export', __name__)
 
@@ -39,6 +40,29 @@ def start_entity_export_workflow(user, campaign_id):
         if not project_id:
             return jsonify({'error': 'project_id is required'}), 400
 
+        # Split rounds by sampling_target
+        conn = get_db_connection()
+        cursor = conn.cursor()
+        try:
+            placeholders = ','.join(['%s'] * len(round_ids))
+            cursor.execute(f"""
+                SELECT id, COALESCE(sampling_target, 'locations') as sampling_target
+                FROM rounds
+                WHERE id IN ({placeholders})
+            """, tuple(round_ids))
+
+            pixel_round_ids = []
+            location_round_ids = []
+            for row in cursor.fetchall():
+                rid, target = str(row[0]), row[1]
+                if target == 'pixels':
+                    pixel_round_ids.append(rid)
+                else:
+                    location_round_ids.append(rid)
+        finally:
+            cursor.close()
+            return_db_connection(conn)
+
         # Generate workflow ID
         timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
         workflow_id = f"entity-export-{campaign_id}-{timestamp}"
@@ -48,7 +72,7 @@ async def start_workflow():
             client = await get_temporal_client()
             handle = await client.start_workflow(
                 EntityExportWorkflow.run,
-                args=[campaign_id, indicator_id, round_ids, project_id, geometry_type],
+                args=[campaign_id, indicator_id, pixel_round_ids, location_round_ids, project_id, geometry_type],
                 id=workflow_id,
                 task_queue="truecover-tasks"
             )

truecover-backend/temporal/activities/enrichment.py

@@ -341,6 +341,17 @@ async def enrich_area_pixels(
                         metadata = pixel_metadata.metadata || EXCLUDED.metadata,
                         updated_at = NOW()
                 """, updates)
+
+                # Also write to pixels.population column for population data
+                if metadata_field_name == 'population':
+                    pop_updates = [
+                        (json.loads(metadata_json)[metadata_field_name], quadkey)
+                        for quadkey, metadata_json in updates
+                    ]
+                    cursor.executemany("""
+                        UPDATE pixels SET population = %s WHERE quadkey = %s
+                    """, pop_updates)
+
                 conn.commit()
                 total_updated += len(updates)
 
@@ -354,11 +365,11 @@ async def enrich_area_pixels(
                 SET cached_population = sub.total_pop
                 FROM (
                     SELECT pa.campaign_area_id,
-                           COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_pop
+                           COALESCE(SUM(p.population), 0) as total_pop
                     FROM pixel_area pa
-                    JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
+                    JOIN pixels p ON pa.quadkey = p.quadkey
                     JOIN campaign_areas ca2 ON pa.campaign_area_id = ca2.id
-                    WHERE ca2.campaign_id = %s AND pm.metadata ? 'population'
+                    WHERE ca2.campaign_id = %s
                     GROUP BY pa.campaign_area_id
                 ) sub
                 WHERE ca.id = sub.campaign_area_id

truecover-backend/temporal/activities/entity_export.py

@@ -1,5 +1,5 @@
-# ABOUTME: Temporal activities for exporting pixel coverage data to ODK entity lists
-# ABOUTME: Handles fetching pixel data and creating entities via Ona API
+# ABOUTME: Temporal activities for exporting coverage data to ODK entity lists
+# ABOUTME: Handles fetching pixel and location data and creating entities via Ona API
 
 from temporalio import activity
 from typing import List, Dict, Any
@@ -85,7 +85,7 @@ async def fetch_pixel_coverage_activity(
                 p.adm4_pcode,
                 pc.rounds
             FROM coverage_pixel pc
-            JOIN pixels p ON p.quadkey = pc.quadkey AND p.campaign_id = pc.campaign_id
+            JOIN pixels p ON p.quadkey = pc.quadkey
             WHERE pc.campaign_id = %s
                 AND pc.indicator_id = %s
                 AND pc.rounds && %s::integer[]
@@ -111,19 +111,84 @@ async def fetch_pixel_coverage_activity(
         return_db_connection(conn)
 
 
+@activity.defn
+async def fetch_location_coverage_activity(
+    campaign_id: str,
+    indicator_id: str,
+    round_ids: List[str]
+) -> List[Dict[str, Any]]:
+    """
+    Fetch location coverage data filtered by selected rounds.
+
+    Returns list of location coverage records with location details.
+    """
+    conn = get_db_connection()
+    cursor = conn.cursor()
+
+    try:
+        # Get round numbers from round IDs
+        placeholders = ','.join(['%s'] * len(round_ids))
+        cursor.execute(f"""
+            SELECT round_number
+            FROM rounds
+            WHERE id IN ({placeholders})
+        """, tuple(round_ids))
+
+        round_numbers = [row[0] for row in cursor.fetchall()]
+
+        if not round_numbers:
+            return []
+
+        cursor.execute("""
+            SELECT DISTINCT
+                c.id,
+                l.external_id,
+                l.latitude,
+                l.longitude,
+                l.quadkey,
+                c.rounds
+            FROM coverage c
+            JOIN locations l ON l.id = c.location_id
+            WHERE c.campaign_id = %s
+                AND c.indicator_id = %s
+                AND c.rounds && %s::integer[]
+            ORDER BY l.external_id
+        """, (campaign_id, indicator_id, round_numbers))
+
+        locations = []
+        for row in cursor.fetchall():
+            cov_id, external_id, latitude, longitude, quadkey, rounds = row
+            locations.append({
+                'id': str(cov_id),
+                'external_id': external_id or '',
+                'latitude': float(latitude) if latitude else None,
+                'longitude': float(longitude) if longitude else None,
+                'quadkey': quadkey or '',
+                'rounds': rounds or []
+            })
+
+        return locations
+
+    finally:
+        cursor.close()
+        return_db_connection(conn)
+
+
 @activity.defn
 async def create_odk_entity_activity(
     project_id: str,
-    pixel_data: Dict[str, Any],
-    geometry_type: str = 'centroid'
+    entity_data: Dict[str, Any],
+    geometry_type: str = 'centroid',
+    entity_type: str = 'pixel'
 ) -> Dict[str, Any]:
     """
     Create a single ODK entity via Ona API.
 
     Args:
         project_id: Project ID to get ODK credentials
-        pixel_data: Pixel data dict with id, quadkey, lat, lng, adm4_pcode
+        entity_data: Entity data dict (pixel or location fields)
         geometry_type: 'centroid' or 'boundary' - how to represent pixel geometry
+        entity_type: 'pixel' or 'location'
 
     Returns:
         Dict with success status and created entity info
@@ -154,21 +219,27 @@ async def create_odk_entity_activity(
         # Remove trailing slash from host_url
         host_url = host_url.rstrip('/')
 
-        # Format geometry based on project setting
-        if geometry_type == 'boundary':
-            # Use pixel boundary (geoshape polygon)
-            geometry = get_pixel_boundary_coords(pixel_data['quadkey'])
+        if entity_type == 'location':
+            # Locations always use point geometry
+            label = entity_data['external_id']
+            geometry = f"{entity_data['latitude']} {entity_data['longitude']} 0 0"
+            details = entity_data['quadkey']
         else:
-            # Use pixel centroid (geopoint)
-            geometry = f"{pixel_data['latitude']} {pixel_data['longitude']} 0 0"
+            # Pixels use configurable geometry
+            label = entity_data['quadkey']
+            if geometry_type == 'boundary':
+                geometry = get_pixel_boundary_coords(entity_data['quadkey'])
+            else:
+                geometry = f"{entity_data['latitude']} {entity_data['longitude']} 0 0"
+            details = entity_data['adm4_pcode']
 
         # Build entity payload
         entity_payload = {
-            'label': pixel_data['quadkey'],
+            'label': label,
             'data': {
                 'geometry': geometry,
                 'status': 'not_visited',
-                'details': pixel_data['adm4_pcode']
+                'details': details
             }
         }
 
@@ -199,7 +270,7 @@ async def create_odk_entity_activity(
 
         return {
             'success': True,
-            'quadkey': pixel_data['quadkey'],
+            'label': label,
             'entity_uuid': entity_result.get('uuid')
         }
 

truecover-backend/temporal/activities/rounds.py

@@ -116,12 +116,8 @@ async def fetch_coverage_for_sampling(
             pop_filter = ""
             pop_params = []
 
-            if min_population is not None and population_field:
-                import re
-                if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', population_field):
-                    raise ValueError(f"Invalid population field name: {population_field}")
-                pop_join = "LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey"
-                pop_filter = f"AND (pm.metadata->>'{population_field}')::float >= %s"
+            if min_population is not None:
+                pop_filter = "AND p.population >= %s"
                 pop_params = [min_population]
 
             if allow_revisit: