Merge pull request #23 from onaio/jan26-cleanup

Add sampled population tracking to campaign areas and rounds

Author: mberg

.gitignore

@@ -11,3 +11,7 @@ geoparquet/
 .playwright-mcp/
 
 pixeldata/
+
+# Database dumps
+*.dump
+*.sql.gz

truecover-app/src/components/CampaignAreasManager.tsx

@@ -28,6 +28,7 @@ interface CampaignArea {
   pixel_count: number;
   sampled_count: number;
   total_population: number;
+  sampled_population: number;
   building_count: number;
   division_name: string | null;
   district_name: string | null;
@@ -344,6 +345,7 @@ const CampaignAreasManager: React.FC<CampaignAreasManagerProps> = ({
   const totalPixels = areas.reduce((sum, area) => sum + (area.pixel_count || 0), 0);
   const totalSampled = areas.reduce((sum, area) => sum + (area.sampled_count || 0), 0);
   const totalPopulation = areas.reduce((sum, area) => sum + (area.total_population || 0), 0);
+  const totalSampledPopulation = areas.reduce((sum, area) => sum + (area.sampled_population || 0), 0);
   const totalBuildings = areas.reduce((sum, area) => sum + (area.building_count || 0), 0);
 
   return (
@@ -472,7 +474,22 @@ const CampaignAreasManager: React.FC<CampaignAreasManagerProps> = ({
                           )}
                         </td>
                         <td className="text-right font-mono">
-                          {area.total_population.toLocaleString()}
+                          {area.total_population === 0 ? (
+                            <span className="text-tactical-text-muted">—</span>
+                          ) : (
+                            <span>
+                              <span className="text-tactical-accent-green">
+                                {area.sampled_population.toLocaleString()}
+                              </span>
+                              <span className="text-tactical-text-muted"> / </span>
+                              <span className="text-tactical-text-primary">
+                                {area.total_population.toLocaleString()}
+                              </span>
+                              <span className="text-tactical-accent-green ml-1">
+                                ({Math.round(area.sampled_population / area.total_population * 100)}%)
+                              </span>
+                            </span>
+                          )}
                         </td>
                         <td className="text-right font-mono">
                           {buildingExtractionWorkflows.has(area.id) ? (
@@ -537,8 +554,19 @@ const CampaignAreasManager: React.FC<CampaignAreasManagerProps> = ({
                           {totalPixels.toLocaleString()}
                         </span>
                       </td>
-                      <td className="text-right font-mono font-bold text-tactical-text-primary">
-                        {totalPopulation.toLocaleString()}
+                      <td className="text-right font-mono font-bold">
+                        <span className="text-tactical-accent-green">
+                          {totalSampledPopulation.toLocaleString()}
+                        </span>
+                        <span className="text-tactical-text-muted"> / </span>
+                        <span className="text-tactical-text-primary">
+                          {totalPopulation.toLocaleString()}
+                        </span>
+                        {totalPopulation > 0 && (
+                          <span className="text-tactical-accent-green ml-1">
+                            ({Math.round(totalSampledPopulation / totalPopulation * 100)}%)
+                          </span>
+                        )}
                       </td>
                       <td className="text-right font-mono font-bold text-tactical-text-primary">
                         {totalBuildings.toLocaleString()}

truecover-app/src/components/RoundsManager.tsx

@@ -21,6 +21,7 @@ interface Round {
   updated_at: string;
   location_count: number;
   pixel_count: number;
+  sampled_population: number;
 }
 
 interface RoundsManagerProps {
@@ -39,6 +40,7 @@ const RoundsManager: React.FC<RoundsManagerProps> = ({ campaignId, areaName, pro
   const { getToken } = useAuth();
   const { data: indicators } = useIndicators(projectId);
   const [rounds, setRounds] = useState<Round[]>([]);
+  const [totalPopulation, setTotalPopulation] = useState<number>(0);
   const [isLoading, setIsLoading] = useState(true);
   const [isCreateModalOpen, setIsCreateModalOpen] = useState(false);
   const [isExportModalOpen, setIsExportModalOpen] = useState(false);
@@ -67,6 +69,7 @@ const RoundsManager: React.FC<RoundsManagerProps> = ({ campaignId, areaName, pro
       );
 
       setRounds(response.data.rounds || []);
+      setTotalPopulation(response.data.total_population || 0);
     } catch (err: any) {
       console.error('Error loading rounds:', err);
       // Don't set error if it's just an empty result or table doesn't exist
@@ -193,6 +196,9 @@ const RoundsManager: React.FC<RoundsManagerProps> = ({ campaignId, areaName, pro
                   <th className="px-4 py-3 text-center text-xs font-mono font-bold text-tactical-text-primary uppercase tracking-wider">
                     Pixels
                   </th>
+                  <th className="px-4 py-3 text-right text-xs font-mono font-bold text-tactical-text-primary uppercase tracking-wider">
+                    Sampled Population
+                  </th>
                 </tr>
               </thead>
               <tbody>
@@ -246,6 +252,27 @@ const RoundsManager: React.FC<RoundsManagerProps> = ({ campaignId, areaName, pro
                     >
                       {round.pixel_count}
                     </td>
+                    <td
+                      className="px-4 py-3 text-right text-sm font-mono cursor-pointer"
+                      onClick={() => handleRoundClick(round.round_number)}
+                    >
+                      {totalPopulation === 0 ? (
+                        <span className="text-tactical-text-muted">—</span>
+                      ) : (
+                        <span>
+                          <span className="text-tactical-accent-green">
+                            {round.sampled_population.toLocaleString()}
+                          </span>
+                          <span className="text-tactical-text-muted"> / </span>
+                          <span className="text-tactical-text-primary">
+                            {totalPopulation.toLocaleString()}
+                          </span>
+                          <span className="text-tactical-accent-green ml-1">
+                            ({Math.round(round.sampled_population / totalPopulation * 100)}%)
+                          </span>
+                        </span>
+                      )}
+                    </td>
                   </tr>
                 ))}
               </tbody>

truecover-backend/db/migrations/add_cached_stats_to_campaign_areas.sql

@@ -4,8 +4,10 @@
 ALTER TABLE campaign_areas
 ADD COLUMN IF NOT EXISTS cached_pixel_count INTEGER DEFAULT 0,
 ADD COLUMN IF NOT EXISTS cached_population NUMERIC DEFAULT 0,
-ADD COLUMN IF NOT EXISTS cached_building_count INTEGER DEFAULT 0;
+ADD COLUMN IF NOT EXISTS cached_building_count INTEGER DEFAULT 0,
+ADD COLUMN IF NOT EXISTS cached_sampled_population NUMERIC DEFAULT 0;
 
 COMMENT ON COLUMN campaign_areas.cached_pixel_count IS 'Cached count of pixels in this area, updated when pixels are computed';
 COMMENT ON COLUMN campaign_areas.cached_population IS 'Cached sum of population from all pixels in this area';
 COMMENT ON COLUMN campaign_areas.cached_building_count IS 'Cached count of buildings/locations in this area';
+COMMENT ON COLUMN campaign_areas.cached_sampled_population IS 'Cached sum of population from sampled pixels in this area';

truecover-backend/routes/campaigns.py

@@ -290,7 +290,8 @@ def list_campaign_areas(user, campaign_id):
                 dist.name as district_name,
                 upz.name as upazila_name,
                 uni.name as union_name,
-                COALESCE(ca.cached_sampled_count, 0) as sampled_count
+                COALESCE(ca.cached_sampled_count, 0) as sampled_count,
+                COALESCE(ca.cached_sampled_population, 0) as sampled_population
             FROM campaign_areas ca
             LEFT JOIN admin_boundaries ab ON ca.admin_boundary_id = ab.id
             -- Get parent boundary names using the pcode hierarchy
@@ -327,7 +328,8 @@ def list_campaign_areas(user, campaign_id):
                 'district_name': row[17],
                 'upazila_name': row[18],
                 'union_name': row[19],
-                'sampled_count': row[20] or 0
+                'sampled_count': row[20] or 0,
+                'sampled_population': int(row[21]) if row[21] else 0
             }
             areas.append(area)
 

truecover-backend/routes/rounds.py

@@ -869,6 +869,15 @@ def list_rounds(user, campaign_id):
             """, (campaign_id, row[1]))
             pixel_count = cursor.fetchone()[0]
 
+            # Sum population from pixels in this round
+            cursor.execute("""
+                SELECT COALESCE(SUM(p.population), 0)
+                FROM coverage_pixel cp
+                JOIN pixels p ON cp.quadkey = p.quadkey
+                WHERE cp.campaign_id = %s AND %s = ANY(cp.rounds)
+            """, (campaign_id, row[1]))
+            sampled_population = cursor.fetchone()[0] or 0
+
             rounds.append({
                 'id': str(row[0]),
                 'round_number': row[1],
@@ -880,11 +889,23 @@ def list_rounds(user, campaign_id):
                 'updated_at': row[7].isoformat() if row[7] else None,
                 'sampling_target': row[8] if len(row) > 8 else 'locations',
                 'location_count': location_count,
-                'pixel_count': pixel_count
+                'pixel_count': pixel_count,
+                'sampled_population': int(sampled_population)
             })
 
+        # Get total population from all campaign areas
+        cursor.execute("""
+            SELECT COALESCE(SUM(cached_population), 0)
+            FROM campaign_areas
+            WHERE campaign_id = %s
+        """, (campaign_id,))
+        total_population = cursor.fetchone()[0] or 0
+
         cursor.close()
-        return jsonify({'rounds': rounds}), 200
+        return jsonify({
+            'rounds': rounds,
+            'total_population': int(total_population)
+        }), 200
 
     except Exception as e:
         print(f"Error listing rounds: {e}")

truecover-backend/temporal/activities/cluster_sampling.py

@@ -348,12 +348,32 @@ async def compute_pixels_for_campaign_areas(
             area_pixel_count = cursor.rowcount
             total_pixels += area_pixel_count
 
-            # Update cached_pixel_count on the campaign_area
+            # Update cached_pixel_count, cached_population, and cached_building_count
             cursor.execute("""
+                WITH pixel_stats AS (
+                    SELECT
+                        COUNT(*) as pixel_count,
+                        COALESCE(SUM(p.population), 0) as total_population
+                    FROM pixel_area pa
+                    JOIN pixels p ON pa.quadkey = p.quadkey
+                    WHERE pa.campaign_area_id = %s
+                ),
+                location_counts AS (
+                    SELECT COUNT(l.id) as building_count
+                    FROM campaign_areas ca
+                    LEFT JOIN locations l ON l.campaign_id = ca.campaign_id
+                        AND l.latitude BETWEEN ca.bbox_min_lat AND ca.bbox_max_lat
+                        AND l.longitude BETWEEN ca.bbox_min_lng AND ca.bbox_max_lng
+                        AND ST_Intersects(l.geometry, ca.geometry)
+                    WHERE ca.id = %s
+                )
                 UPDATE campaign_areas
-                SET cached_pixel_count = %s, updated_at = NOW()
+                SET cached_pixel_count = (SELECT pixel_count FROM pixel_stats),
+                    cached_population = (SELECT total_population FROM pixel_stats),
+                    cached_building_count = (SELECT building_count FROM location_counts),
+                    updated_at = NOW()
                 WHERE id = %s
-            """, (area_pixel_count, area_id))
+            """, (area_id, area_id, area_id))
 
         conn.commit()
         activity.logger.info(f"Computed {total_pixels} pixel associations for {len(campaign_area_ids)} areas")
@@ -688,18 +708,22 @@ async def assign_pixels_to_round(
                 WHERE id = %s AND NOT (%s = ANY(COALESCE(rounds, '{}')))
             """, (round_number, coverage_id, round_number))
 
-        # Update cached_sampled_count on campaign_area (count pixels with any round)
+        # Update cached_sampled_count and cached_sampled_population on campaign_area
         cursor.execute("""
             WITH sampled AS (
-                SELECT COUNT(DISTINCT cp.quadkey) as cnt
+                SELECT
+                    COUNT(DISTINCT cp.quadkey) as cnt,
+                    COALESCE(SUM(p.population), 0) as sampled_pop
                 FROM coverage_pixel cp
                 JOIN pixel_area pa ON cp.quadkey = pa.quadkey
+                JOIN pixels p ON cp.quadkey = p.quadkey
                 WHERE pa.campaign_area_id = %s
                   AND cp.rounds IS NOT NULL
                   AND array_length(cp.rounds, 1) > 0
             )
             UPDATE campaign_areas
             SET cached_sampled_count = (SELECT cnt FROM sampled),
+                cached_sampled_population = (SELECT sampled_pop FROM sampled),
                 updated_at = NOW()
             WHERE id = %s
         """, (campaign_area_id, campaign_area_id))
@@ -756,12 +780,15 @@ async def update_campaign_area_sampled_count_for_union(
 
         area_id = str(row[0])
 
-        # Count pixels with rounds assigned
+        # Count pixels and population with rounds assigned
         cursor.execute("""
             WITH sampled AS (
-                SELECT COUNT(DISTINCT cp.quadkey) as cnt
+                SELECT
+                    COUNT(DISTINCT cp.quadkey) as cnt,
+                    COALESCE(SUM(p.population), 0) as sampled_pop
                 FROM coverage_pixel cp
                 JOIN pixel_area pa ON cp.quadkey = pa.quadkey
+                JOIN pixels p ON cp.quadkey = p.quadkey
                 WHERE pa.campaign_area_id = %s
                   AND cp.campaign_id = %s
                   AND cp.indicator_id = %s
@@ -770,6 +797,7 @@ async def update_campaign_area_sampled_count_for_union(
             )
             UPDATE campaign_areas
             SET cached_sampled_count = (SELECT cnt FROM sampled),
+                cached_sampled_population = (SELECT sampled_pop FROM sampled),
                 updated_at = NOW()
             WHERE id = %s
             RETURNING cached_sampled_count
@@ -814,28 +842,34 @@ async def update_campaign_area_sampled_counts(
         results = {}
 
         for area_id in campaign_area_ids:
-            # Count pixels that have been sampled (have any round assigned)
+            # Count pixels and population that have been sampled (have any round assigned)
             cursor.execute("""
-                SELECT COUNT(DISTINCT cp.quadkey)
+                SELECT
+                    COUNT(DISTINCT cp.quadkey),
+                    COALESCE(SUM(p.population), 0)
                 FROM coverage_pixel cp
                 JOIN pixel_area pa ON cp.quadkey = pa.quadkey
+                JOIN pixels p ON cp.quadkey = p.quadkey
                 WHERE pa.campaign_area_id = %s
                   AND cp.campaign_id = %s
                   AND cp.indicator_id = %s
                   AND cp.rounds IS NOT NULL
                   AND array_length(cp.rounds, 1) > 0
             """, (area_id, campaign_id, indicator_id))
 
-            sampled_count = cursor.fetchone()[0] or 0
+            row = cursor.fetchone()
+            sampled_count = row[0] or 0
+            sampled_population = row[1] or 0
             results[area_id] = sampled_count
 
-            # Update cached count
+            # Update cached counts
             cursor.execute("""
                 UPDATE campaign_areas
                 SET cached_sampled_count = %s,
+                    cached_sampled_population = %s,
                     updated_at = NOW()
                 WHERE id = %s
-            """, (sampled_count, area_id))
+            """, (sampled_count, sampled_population, area_id))
 
         conn.commit()
 
@@ -1192,18 +1226,22 @@ async def clear_round_from_pixels(
 
         cleared_count = cursor.rowcount
 
-        # Update cached_sampled_count (count pixels still having rounds)
+        # Update cached_sampled_count and cached_sampled_population (count pixels still having rounds)
         cursor.execute("""
             WITH sampled AS (
-                SELECT COUNT(DISTINCT cp.quadkey) as cnt
+                SELECT
+                    COUNT(DISTINCT cp.quadkey) as cnt,
+                    COALESCE(SUM(p.population), 0) as sampled_pop
                 FROM coverage_pixel cp
                 JOIN pixel_area pa ON cp.quadkey = pa.quadkey
+                JOIN pixels p ON cp.quadkey = p.quadkey
                 WHERE pa.campaign_area_id = %s
                   AND cp.rounds IS NOT NULL
                   AND array_length(cp.rounds, 1) > 0
             )
             UPDATE campaign_areas
             SET cached_sampled_count = (SELECT cnt FROM sampled),
+                cached_sampled_population = (SELECT sampled_pop FROM sampled),
                 updated_at = NOW()
             WHERE id = %s
         """, (campaign_area_id, campaign_area_id))