Merge pull request #116 from topcoder-platform/PM-5703

PM-5703: correct dashboard payment and participation data

Author: jmgasper

README.md

@@ -34,17 +34,19 @@ Dashboard figures use these shared definitions:
 
 - Signups come from `identity.user.create_date`. `status = 'A'` is activated;
   every other current status is not activated.
-- Paid-member activity requires a `PAID` finance payment for a `PAYMENT`
-  winning. Its event timestamp is `date_paid`, falling back to `created_at`.
-  Members are deduplicated within each payment bucket and month.
+- Paid-member activity uses the latest non-cancelled finance payment for a
+  `PAYMENT` winning. It is grouped by the payment creation month so projected
+  payments that are owed or on hold remain visible. Members are deduplicated
+  within each payment bucket and month.
 - Member-payment values use the latest payment version and sum `gross_amount`,
   falling back to `total_amount`. The payment-by-customer dashboard ranks the
   top five billing-account clients across the selected range and groups all
   unnamed or remaining clients under `Other Customers`.
-- Registrations are Submitter resource creation events. Submissions are
-  non-deleted review submission events, using `submittedDate` and falling
-  back to `createdAt`. Each category is deduplicated independently by member
-  and month.
+- Challenge participation uses the latest actual phase completion month for
+  Challenge, Marathon Match, and First2Finish cohorts. Registrants are
+  Submitter resources, and submitters have a non-deleted submission for the
+  same challenge and member. Each category is deduplicated by member and
+  cohort month.
 - Rates are percentages from 0 through 100.
 
 Human access is limited to Administrator and Talent Manager roles. Machine

sql/reports/dashboard/challenge-participation.sql

@@ -4,8 +4,10 @@
 --   $1 timestamptz - inclusive reporting range start
 --   $2 timestamptz - exclusive reporting range end
 --
--- Registration and submission are independent activity events. A member is
--- counted once in each category per month, regardless of challenge count.
+-- The month is the challenge cohort's latest actual phase completion month,
+-- matching the Challenge Registrants report. This avoids treating legacy
+-- resource import timestamps as registration dates. A member is counted once
+-- in each category per month, regardless of challenge count.
 WITH bounds AS (
   SELECT
     $1::timestamptz AT TIME ZONE 'UTC' AS start_at,
@@ -19,23 +21,48 @@ months AS (
   ) AS month_start
   FROM bounds b
 ),
-registration_events AS MATERIALIZED (
+eligible_challenges AS MATERIALIZED (
   SELECT
+    c.id AS challenge_id,
+    lp."actualEndDate" AS activity_at
+  FROM challenges."Challenge" c
+  JOIN challenges."ChallengeType" ct
+    ON ct.id = c."typeId"
+  JOIN LATERAL (
+    SELECT cp."actualEndDate"
+    FROM challenges."ChallengePhase" cp
+    WHERE cp."challengeId" = c.id
+    ORDER BY cp."scheduledEndDate" DESC
+    LIMIT 1
+  ) lp
+    ON lp."actualEndDate" IS NOT NULL
+  WHERE ct.name IN ('Challenge', 'Marathon Match', 'First2Finish')
+),
+registration_events AS MATERIALIZED (
+  SELECT DISTINCT
+    ec.challenge_id,
     NULLIF(TRIM(r."memberId"), '') AS member_id,
-    r."createdAt" AS activity_at
-  FROM resources."Resource" r
+    ec.activity_at
+  FROM eligible_challenges ec
+  JOIN resources."Resource" r
+    ON r."challengeId" = ec.challenge_id
   JOIN resources."ResourceRole" rr
     ON rr.id = r."roleId"
   WHERE COALESCE(NULLIF(TRIM(rr."nameLower"), ''), LOWER(rr.name)) = 'submitter'
     AND NULLIF(TRIM(r."memberId"), '') IS NOT NULL
 ),
 submission_events AS MATERIALIZED (
   SELECT
-    NULLIF(TRIM(s."memberId"), '') AS member_id,
-    COALESCE(s."submittedDate", s."createdAt") AS activity_at
-  FROM reviews.submission s
-  WHERE s.status <> 'DELETED'
-    AND NULLIF(TRIM(s."memberId"), '') IS NOT NULL
+    re.member_id,
+    re.activity_at
+  FROM registration_events re
+  WHERE EXISTS (
+    SELECT 1
+    FROM reviews.submission s
+    WHERE s."challengeId" = re.challenge_id
+      AND s."memberId" = re.member_id
+      AND s.status <> 'DELETED'
+  )
 ),
 selected_registrations AS (
   SELECT

sql/reports/dashboard/members-paid.sql

@@ -4,11 +4,10 @@
 --   $1 timestamptz - inclusive reporting range start
 --   $2 timestamptz - exclusive reporting range end
 --
--- A paid event is a PAYMENT winning whose current payment status is PAID.
--- date_paid is authoritative when present; created_at supports migrated paid rows
--- that do not have a paid timestamp. TOPGEAR_PAYMENT is intentionally excluded
--- because it is a separate canonical accrual bucket and is not one of the four
--- dashboard categories.
+-- The report is a financial projection, so the latest non-cancelled payment
+-- record is counted in its creation month even when payout is still on hold or
+-- owed. TOPGEAR_PAYMENT is intentionally excluded because it is a separate
+-- canonical accrual bucket and is not one of the four dashboard categories.
 WITH bounds AS (
   SELECT
     $1::timestamptz AT TIME ZONE 'UTC' AS start_at,
@@ -22,10 +21,17 @@ months AS (
   ) AS month_start
   FROM bounds b
 ),
-paid_events AS MATERIALIZED (
+latest_payment_versions AS MATERIALIZED (
+  SELECT
+    p.winnings_id,
+    MAX(p.version) AS max_version
+  FROM finance.payment p
+  GROUP BY p.winnings_id
+),
+payment_events AS MATERIALIZED (
   SELECT
     NULLIF(TRIM(w.winner_id), '') AS member_id,
-    COALESCE(p.date_paid, p.created_at) AS paid_at,
+    p.created_at AS activity_at,
     CASE
       WHEN w.category::text = 'TAAS_PAYMENT' THEN 'taas'
       WHEN w.category::text = 'ENGAGEMENT_PAYMENT' THEN 'engagement'
@@ -39,17 +45,20 @@ paid_events AS MATERIALIZED (
       ELSE 'challenge'
     END AS payment_type
   FROM finance.payment p
+  JOIN latest_payment_versions lpv
+    ON lpv.winnings_id = p.winnings_id
+   AND lpv.max_version = p.version
   JOIN finance.winnings w
     ON w.winning_id = p.winnings_id
-  WHERE p.payment_status = 'PAID'
+  WHERE p.payment_status IS DISTINCT FROM 'CANCELLED'
     AND w.type = 'PAYMENT'
-    AND COALESCE(p.date_paid, p.created_at) IS NOT NULL
+    AND p.created_at IS NOT NULL
     AND NULLIF(TRIM(w.winner_id), '') IS NOT NULL
     AND w.category::text IS DISTINCT FROM 'TOPGEAR_PAYMENT'
 ),
 selected_months AS (
   SELECT
-    DATE_TRUNC('month', pe.paid_at) AS month_start,
+    DATE_TRUNC('month', pe.activity_at) AS month_start,
     COUNT(DISTINCT pe.member_id) FILTER (
       WHERE pe.payment_type = 'taas'
     ) AS taas,
@@ -62,18 +71,18 @@ selected_months AS (
     COUNT(DISTINCT pe.member_id) FILTER (
       WHERE pe.payment_type = 'engagement'
     ) AS engagement
-  FROM paid_events pe
+  FROM payment_events pe
   CROSS JOIN bounds b
-  WHERE pe.paid_at >= b.start_at
-    AND pe.paid_at < b.end_at
-  GROUP BY DATE_TRUNC('month', pe.paid_at)
+  WHERE pe.activity_at >= b.start_at
+    AND pe.activity_at < b.end_at
+  GROUP BY DATE_TRUNC('month', pe.activity_at)
 ),
 all_time_months AS (
   SELECT
-    DATE_TRUNC('month', pe.paid_at) AS month_start,
+    DATE_TRUNC('month', pe.activity_at) AS month_start,
     COUNT(DISTINCT pe.member_id) AS unique_members
-  FROM paid_events pe
-  GROUP BY DATE_TRUNC('month', pe.paid_at)
+  FROM payment_events pe
+  GROUP BY DATE_TRUNC('month', pe.activity_at)
 ),
 peak_month AS (
   SELECT
@@ -98,7 +107,7 @@ all_time_summary AS (
     COUNT(DISTINCT pe.member_id) FILTER (
       WHERE pe.payment_type = 'engagement'
     ) AS engagement_unique_members
-  FROM paid_events pe
+  FROM payment_events pe
 )
 SELECT
   TO_CHAR(m.month_start, 'YYYY-MM-01') AS month,

src/reports/dashboard/dashboard-reports.sql.spec.ts

@@ -34,12 +34,16 @@ describe("Dashboard report SQL", () => {
     expect(sql).toContain("AS peak_month_signups");
   });
 
-  it("counts paid members once per canonical payment bucket and month", () => {
+  it("counts projected members once per canonical payment bucket and month", () => {
     const sql = sqlLoader.load("reports/dashboard/members-paid.sql");
 
-    expect(sql).toContain("p.payment_status = 'PAID'");
+    expect(sql).toContain("MAX(p.version) AS max_version");
+    expect(sql).toContain("lpv.max_version = p.version");
+    expect(sql).toContain("p.payment_status IS DISTINCT FROM 'CANCELLED'");
     expect(sql).toContain("w.type = 'PAYMENT'");
-    expect(sql).toContain("COALESCE(p.date_paid, p.created_at)");
+    expect(sql).toContain("p.created_at AS activity_at");
+    expect(sql).not.toContain("p.payment_status = 'PAID'");
+    expect(sql).not.toContain("p.date_paid");
     expect(sql).toContain("w.category::text = 'TAAS_PAYMENT'");
     expect(sql).toContain("w.category::text = 'ENGAGEMENT_PAYMENT'");
     expect(sql).toContain("'TASK_REVIEW_PAYMENT'");
@@ -104,15 +108,26 @@ describe("Dashboard report SQL", () => {
     expect(sql).toContain("COALESCE(ma.amount, 0) AS amount");
   });
 
-  it("counts registration and submission activity independently", () => {
+  it("counts registrants and linked submitters by challenge completion cohort", () => {
     const sql = sqlLoader.load("reports/dashboard/challenge-participation.sql");
 
-    expect(sql).toContain('FROM resources."Resource" r');
+    expect(sql).toContain('FROM challenges."Challenge" c');
+    expect(sql).toContain('JOIN challenges."ChallengeType" ct');
+    expect(sql).toContain(
+      "ct.name IN ('Challenge', 'Marathon Match', 'First2Finish')",
+    );
+    expect(sql).toContain('FROM challenges."ChallengePhase" cp');
+    expect(sql).toContain('lp."actualEndDate" AS activity_at');
+    expect(sql).toContain('ORDER BY cp."scheduledEndDate" DESC');
+    expect(sql).toContain('JOIN resources."Resource" r');
     expect(sql).toContain('JOIN resources."ResourceRole" rr');
     expect(sql).toContain("= 'submitter'");
     expect(sql).toContain("FROM reviews.submission s");
+    expect(sql).toContain('s."challengeId" = re.challenge_id');
+    expect(sql).toContain('s."memberId" = re.member_id');
     expect(sql).toContain("s.status <> 'DELETED'");
-    expect(sql).toContain('COALESCE(s."submittedDate", s."createdAt")');
+    expect(sql).not.toContain('r."createdAt" AS activity_at');
+    expect(sql).not.toContain('COALESCE(s."submittedDate", s."createdAt")');
     expect(sql).toContain("COUNT(DISTINCT re.member_id)");
     expect(sql).toContain("COUNT(DISTINCT se.member_id)");
     expect(sql).toContain("LEAST(");