Merge pull request #8760 from Couchers-org/backend/feature/feature-usage-logging

Log feature flag usage per user

Author: aapeliv

app/backend/src/couchers/experimentation.py

@@ -16,12 +16,12 @@
 
 import urllib3
 from growthbook import GrowthBook
-from growthbook.common_types import Experiment, Result
+from growthbook.common_types import Experiment, FeatureResult, Result
 from sqlalchemy.dialects.postgresql import insert
 
 from couchers.config import config
 from couchers.db import session_scope
-from couchers.models.logging import ExperimentExposure
+from couchers.models.logging import ExperimentExposure, FeatureUsage
 
 logger = logging.getLogger(__name__)
 
@@ -139,13 +139,18 @@ def _record_exposure(user_id: int, experiment: Experiment, result: Result, **_:
         session.execute(stmt)
 
 
+def _record_feature_usage(user_id: int, key: str, result: FeatureResult, **_: Any) -> None:
+    with session_scope() as session:
+        session.add(FeatureUsage(user_id=user_id, feature_key=key, value=result.value))
+
+
 def _create_evaluator(user_id: int | None) -> GrowthBook:
     """
     Build a per-request GrowthBook evaluator over the current feature snapshot.
 
     Pass user_id=None for an anonymous (logged-out) evaluation: with no `id` attribute GrowthBook
     can't bucket the user, so experiments and percentage rollouts are skipped and flags fall
-    through to their defaults. No exposure is recorded without a user.
+    through to their defaults. No exposure or usage is recorded without a user.
 
     Reads the in-memory snapshot maintained by the background refresh thread - never does HTTP
     from the request path. Constructing without `client_key` keeps the GrowthBook a pure
@@ -164,9 +169,14 @@ def on_experiment_viewed(experiment: Experiment, result: Result, **kwargs: Any)
         if user_id is not None:
             _record_exposure(user_id, experiment, result)
 
+    def on_feature_usage(key: str, result: FeatureResult, *args: Any, **kwargs: Any) -> None:
+        if user_id is not None:
+            _record_feature_usage(user_id, key, result)
+
     return GrowthBook(
         attributes={"id": str(user_id)} if user_id is not None else {},
         features=features,
         savedGroups=saved_groups,
         on_experiment_viewed=on_experiment_viewed,
+        on_feature_usage=on_feature_usage,
     )

app/backend/src/couchers/migrations/versions/0157_log_feature_flag_usage_per_user.py

@@ -0,0 +1,46 @@
+"""Log feature flag usage per user
+
+Revision ID: 0157
+Revises: 0156
+Create Date: 2026-05-23 05:13:52.193316
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "0157"
+down_revision = "0156"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "feature_usage",
+        sa.Column("id", sa.BigInteger(), nullable=False),
+        sa.Column("time", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("user_id", sa.BigInteger(), nullable=False),
+        sa.Column("feature_key", sa.String(), nullable=False),
+        sa.Column("value", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
+        sa.PrimaryKeyConstraint("id", name=op.f("pk_feature_usage")),
+        schema="logging",
+    )
+    op.create_index(
+        "ix_logging_feature_usage_feature_key_time",
+        "feature_usage",
+        ["feature_key", "time"],
+        unique=False,
+        schema="logging",
+    )
+    op.create_index(
+        "ix_logging_feature_usage_user_id_time", "feature_usage", ["user_id", "time"], unique=False, schema="logging"
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_logging_feature_usage_user_id_time", table_name="feature_usage", schema="logging")
+    op.drop_index("ix_logging_feature_usage_feature_key_time", table_name="feature_usage", schema="logging")
+    op.drop_table("feature_usage", schema="logging")

app/backend/src/couchers/models/logging.py

@@ -156,3 +156,33 @@ class ExperimentExposure(Base, kw_only=True):
         Index("ix_logging_experiment_exposures_user_id_created", "user_id", "created"),
         {"schema": "logging"},
     )
+
+
+class FeatureUsage(Base, kw_only=True):
+    """
+    Append-only log of feature flag evaluations.
+
+    Populated by GrowthBook's on_feature_usage callback - one row per check.
+    """
+
+    __tablename__ = "feature_usage"
+
+    id: Mapped[int] = mapped_column(BigInteger, primary_key=True, init=False)
+
+    # when the feature was checked
+    time: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), init=False)
+
+    # user the feature was checked for
+    user_id: Mapped[int] = mapped_column(BigInteger)
+
+    # feature identifier from GrowthBook
+    feature_key: Mapped[str] = mapped_column(String)
+
+    # the feature value the user received
+    value: Mapped[Any] = mapped_column(JSONB)
+
+    __table_args__ = (
+        Index("ix_logging_feature_usage_feature_key_time", "feature_key", "time"),
+        Index("ix_logging_feature_usage_user_id_time", "user_id", "time"),
+        {"schema": "logging"},
+    )

app/backend/src/tests/test_experimentation.py

@@ -1,9 +1,14 @@
 import pytest
+from growthbook.common_types import FeatureResult
+from sqlalchemy import select
 
 from couchers import experimentation
 from couchers.config import config
 from couchers.context import make_background_user_context, make_logged_out_context
+from couchers.db import session_scope
+from couchers.experimentation import _record_feature_usage
 from couchers.i18n import LocalizationContext
+from couchers.models.logging import FeatureUsage
 
 
 @pytest.fixture
@@ -22,7 +27,7 @@ def experimentation_snapshot(monkeypatch):
     monkeypatch.setitem(config, "EXPERIMENTATION_PASS_ALL_GATES", False)
 
 
-def test_logged_in_user_is_bucketed_into_rollout(experimentation_snapshot):
+def test_logged_in_user_is_bucketed_into_rollout(db, experimentation_snapshot):
     context = make_background_user_context(123)
     assert context.get_string_value("rollout_flag", "fallback") == "treatment"
 
@@ -41,3 +46,53 @@ def test_anonymous_user_still_gets_global_force_on_flag(experimentation_snapshot
 def test_unknown_feature_returns_in_code_default(experimentation_snapshot):
     context = make_logged_out_context(LocalizationContext.en_utc())
     assert context.get_string_value("does_not_exist", "my_default") == "my_default"
+
+
+def _get_usage(session, user_id):
+    return (
+        session.execute(select(FeatureUsage).where(FeatureUsage.user_id == user_id).order_by(FeatureUsage.id))
+        .scalars()
+        .all()
+    )
+
+
+def test_record_feature_usage_appends_a_row(db):
+    _record_feature_usage(1, "my_feature", FeatureResult(value=True, source="defaultValue"))
+
+    with session_scope() as session:
+        rows = _get_usage(session, 1)
+        assert len(rows) == 1
+        assert rows[0].feature_key == "my_feature"
+        assert rows[0].value is True
+        assert rows[0].time is not None
+
+
+def test_record_feature_usage_appends_a_row_per_check(db):
+    # every check appends - the log is append-only, not deduplicated per (user, feature)
+    _record_feature_usage(1, "my_feature", FeatureResult(value="first", source="force"))
+    _record_feature_usage(1, "my_feature", FeatureResult(value="second", source="force"))
+
+    with session_scope() as session:
+        rows = _get_usage(session, 1)
+        assert len(rows) == 2
+        assert [row.value for row in rows] == ["first", "second"]
+
+
+def test_record_feature_usage_records_each_user_and_feature(db):
+    _record_feature_usage(1, "feature_a", FeatureResult(value=1, source="force"))
+    _record_feature_usage(1, "feature_b", FeatureResult(value=2, source="force"))
+    _record_feature_usage(2, "feature_a", FeatureResult(value=3, source="force"))
+
+    with session_scope() as session:
+        assert {row.feature_key for row in _get_usage(session, 1)} == {"feature_a", "feature_b"}
+        assert len(_get_usage(session, 2)) == 1
+
+
+def test_record_feature_usage_none_value(db):
+    # unknown features evaluate to a None value - must persist without violating NOT NULL
+    _record_feature_usage(1, "unknown_feature", FeatureResult(value=None, source="unknownFeature"))
+
+    with session_scope() as session:
+        rows = _get_usage(session, 1)
+        assert len(rows) == 1
+        assert rows[0].value is None