Read population from pixels.population instead of pixel_metadata

All population reads now use the pixels.population column directly,
removing the JOIN to pixel_metadata for population data. The
enrichment pipeline also writes to pixels.population when enriching
population. Truncated the pixel_metadata table (data was duplicated).
The pixel_metadata table is retained for non-population covariates.

Author: mberg

truecover-backend/db/migrations/add_sampled_only_filter.sql

@@ -71,15 +71,14 @@ BEGIN
                 SELECT
                     p.quadkey,
                     p.geometry,
-                    (pm.metadata->>'population')::numeric AS population,
+                    p.population,
                     lc.building_count
                 FROM pixels p
                 JOIN pixel_area pa ON p.quadkey = pa.quadkey
                 JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
                 LEFT JOIN coverage_pixel cp ON p.quadkey = cp.quadkey
                     AND cp.campaign_id = target_campaign_id
                     AND (target_indicator_id IS NULL OR cp.indicator_id = target_indicator_id)
-                LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
                 LEFT JOIN LATERAL (
                     SELECT COUNT(*)::integer AS building_count
                     FROM locations l

truecover-backend/db/migrations/optimize_pixels_by_campaign.sql

@@ -64,12 +64,11 @@ BEGIN
                 SELECT
                     p.quadkey,
                     p.geometry,
-                    (pm.metadata->>'population')::numeric AS population,
+                    p.population,
                     lc.building_count
                 FROM pixels p
                 JOIN pixel_area pa ON p.quadkey = pa.quadkey
                 JOIN campaign_areas ca ON pa.campaign_area_id = ca.id
-                LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey
                 LEFT JOIN LATERAL (
                     SELECT COUNT(*)::integer AS building_count
                     FROM locations l

truecover-backend/routes/campaigns.py

@@ -581,10 +581,9 @@ def compute_pixels_for_area(user, area_id):
             WITH pixel_stats AS (
                 SELECT
                     COUNT(*) as pixel_count,
-                    COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population
+                    COALESCE(SUM(p.population), 0) as total_population
                 FROM pixel_area pa
                 JOIN pixels p ON pa.quadkey = p.quadkey
-                LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
                 WHERE pa.campaign_area_id = %s
             ),
             location_counts AS (
@@ -696,10 +695,9 @@ def compute_all_pixels_for_campaign(user, campaign_id):
                 WITH pixel_stats AS (
                     SELECT
                         COUNT(*) as pixel_count,
-                        COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_population
+                        COALESCE(SUM(p.population), 0) as total_population
                     FROM pixel_area pa
                     JOIN pixels p ON pa.quadkey = p.quadkey
-                    LEFT JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
                     WHERE pa.campaign_area_id = %s
                 ),
                 location_counts AS (

truecover-backend/temporal/activities/enrichment.py

@@ -341,6 +341,17 @@ async def enrich_area_pixels(
                         metadata = pixel_metadata.metadata || EXCLUDED.metadata,
                         updated_at = NOW()
                 """, updates)
+
+                # Also write to pixels.population column for population data
+                if metadata_field_name == 'population':
+                    pop_updates = [
+                        (json.loads(metadata_json)[metadata_field_name], quadkey)
+                        for quadkey, metadata_json in updates
+                    ]
+                    cursor.executemany("""
+                        UPDATE pixels SET population = %s WHERE quadkey = %s
+                    """, pop_updates)
+
                 conn.commit()
                 total_updated += len(updates)
 
@@ -354,11 +365,11 @@ async def enrich_area_pixels(
                 SET cached_population = sub.total_pop
                 FROM (
                     SELECT pa.campaign_area_id,
-                           COALESCE(SUM((pm.metadata->>'population')::numeric), 0) as total_pop
+                           COALESCE(SUM(p.population), 0) as total_pop
                     FROM pixel_area pa
-                    JOIN pixel_metadata pm ON pa.quadkey = pm.quadkey
+                    JOIN pixels p ON pa.quadkey = p.quadkey
                     JOIN campaign_areas ca2 ON pa.campaign_area_id = ca2.id
-                    WHERE ca2.campaign_id = %s AND pm.metadata ? 'population'
+                    WHERE ca2.campaign_id = %s
                     GROUP BY pa.campaign_area_id
                 ) sub
                 WHERE ca.id = sub.campaign_area_id

truecover-backend/temporal/activities/rounds.py

@@ -116,12 +116,8 @@ async def fetch_coverage_for_sampling(
             pop_filter = ""
             pop_params = []
 
-            if min_population is not None and population_field:
-                import re
-                if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', population_field):
-                    raise ValueError(f"Invalid population field name: {population_field}")
-                pop_join = "LEFT JOIN pixel_metadata pm ON p.quadkey = pm.quadkey"
-                pop_filter = f"AND (pm.metadata->>'{population_field}')::float >= %s"
+            if min_population is not None:
+                pop_filter = "AND p.population >= %s"
                 pop_params = [min_population]
 
             if allow_revisit: