Merge pull request #795 from kelly-musk/feat/issues-728-730-731-740

feat: notification digest batching, creators search, postgres backup drill, rate-limit tests

Author: aji70

.github/workflows/ci.yml

@@ -167,6 +167,59 @@ jobs:
         run: npm run build
         working-directory: frontend
 
+  db-backup-drill:
+    name: Postgres Backup / Restore Drill
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [backend-migrations]
+
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_USER: myfans_ci
+          POSTGRES_PASSWORD: myfans_ci
+          POSTGRES_DB: myfans_test
+        ports:
+          - 5432:5432
+        options: >
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+
+    env:
+      DB_HOST: localhost
+      DB_PORT: 5432
+      DB_USER: myfans_ci
+      DB_PASSWORD: myfans_ci
+      DB_NAME: myfans_test
+      JWT_SECRET: ci-test-secret-not-for-production
+      NODE_ENV: test
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: backend/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+        working-directory: backend
+
+      - name: Install postgres client
+        run: sudo apt-get install -y postgresql-client
+
+      - name: Run migrations (seed schema)
+        run: npm run migration:run
+        working-directory: backend
+
+      - name: Backup / restore drill
+        run: ./scripts/pg-backup-restore.sh drill
+
   contract:
     name: Contract (Rust)
     runs-on: ubuntu-latest

README.md

@@ -192,6 +192,7 @@ You will keep only these three folders and this README; other files can be remov
 - **[Security Policy](SECURITY.md)** - Security reporting, penetration testing tracker, and best practices
 - **[Bug Bash Checklist](docs/BUG_BASH_CHECKLIST.md)** - Comprehensive QA checklist before major releases
 - **[Changelog Guide](docs/CHANGELOG_GUIDE.md)** - How to use conventional commits for automatic changelog generation
+- **[Postgres Backup / Restore](docs/POSTGRES_BACKUP_RESTORE.md)** - Backup runbook, restore decision tree, and CI drill
 
 ### Development
 - **[Changelog](CHANGELOG.md)** - Automatically generated from conventional commits

backend/src/app.module.ts

@@ -40,9 +40,10 @@ const IDEMPOTENCY_ROUTES = [
 @Module({
   imports: [
     ThrottlerModule.forRoot([
-      { name: 'short', ttl: 60000, limit: 10 },
-      { name: 'medium', ttl: 60000, limit: 50 },
-      { name: 'long', ttl: 60000, limit: 100 },
+      { name: 'auth',   ttl: 60000, limit: 5   },
+      { name: 'short',  ttl: 60000, limit: 10  },
+      { name: 'medium', ttl: 60000, limit: 50  },
+      { name: 'long',   ttl: 60000, limit: 100 },
     ]),
     LoggingModule,
     MetricsModule,

backend/src/common/dto/paginated-response.dto.ts

@@ -16,11 +16,41 @@ export class PaginatedResponseDto<T> {
   @ApiProperty({ description: 'Whether there are more items' })
   hasMore: boolean;
 
-  constructor(data: T[], limit: number, nextCursor: string | null, hasMore: boolean) {
+  @ApiPropertyOptional({ description: 'Total number of matching items' })
+  total: number;
+
+  @ApiPropertyOptional({ description: 'Current page number (1-based)' })
+  page: number;
+
+  @ApiPropertyOptional({ description: 'Total number of pages' })
+  totalPages: number;
+
+  constructor(
+    data: T[],
+    limitOrTotal: number,
+    nextCursorOrPage: string | number | null,
+    hasmoreOrLimit: boolean | number,
+    page?: number,
+  ) {
     this.data = data;
     this.cursor = null;
-    this.limit = limit;
-    this.nextCursor = nextCursor;
-    this.hasMore = hasMore;
+
+    // Overload: (data, total, page, limit) — used by searchCreators
+    if (typeof nextCursorOrPage === 'number' && typeof hasmoreOrLimit === 'number') {
+      this.total = limitOrTotal;
+      this.page = nextCursorOrPage;
+      this.limit = hasmoreOrLimit;
+      this.nextCursor = null;
+      this.hasMore = this.page * this.limit < this.total;
+      this.totalPages = Math.ceil(this.total / this.limit);
+    } else {
+      // Original: (data, limit, nextCursor, hasMore)
+      this.limit = limitOrTotal;
+      this.nextCursor = nextCursorOrPage as string | null;
+      this.hasMore = hasmoreOrLimit as boolean;
+      this.total = data.length;
+      this.page = page ?? 1;
+      this.totalPages = 1;
+    }
   }
 }

backend/src/creators/creators.service.ts

@@ -120,37 +120,29 @@ export class CreatorsService {
   async searchCreators(
     searchDto: SearchCreatorsDto,
   ): Promise<PaginatedResponseDto<PublicCreatorDto>> {
-    const { cursor, limit = 20, q } = searchDto;
+    const { page = 1, limit = 20, q } = searchDto;
     const trimmed = q?.trim();
 
     const qb = this.userRepository
       .createQueryBuilder('user')
       .leftJoin('user.creator', 'creator')
       .addSelect('creator.bio', 'creator_bio')
       .where('user.is_creator = :isCreator', { isCreator: true })
-      .orderBy('user.id', 'ASC')
-      .take(limit + 1);
-
-    if (cursor) {
-      const cursorId = parseInt(cursor, 10);
-      if (!isNaN(cursorId)) {
-        qb.andWhere('user.id > :cursorId', { cursorId });
-      }
-    }
-
-    const { entities, raw } = await qb.getRawAndEntities();
-    const hasMore = entities.length > limit;
-    if (hasMore) {
-      entities.pop();
-    }
-
-    let nextCursor: string | null = null;
-    if (entities.length > 0) {
-      nextCursor = String(entities[entities.length - 1].id);
+      .orderBy('user.username', 'ASC')
+      .skip((page - 1) * limit)
+      .take(limit);
+
+    if (trimmed) {
+      qb.andWhere(
+        '(LOWER(user.display_name) LIKE :search OR LOWER(user.username) LIKE :search)',
+        { search: `${trimmed.toLowerCase()}%` },
+      );
     }
 
-    return new PaginatedResponseDto(entities, limit, nextCursor, hasMore);
-  }
+    const [{ entities, raw }, total] = await Promise.all([
+      qb.getRawAndEntities(),
+      qb.getCount(),
+    ]);
 
     const data = entities.map((user, index) => {
       const dto = new PublicCreatorDto(user, user.creator);
@@ -159,7 +151,7 @@ export class CreatorsService {
     });
 
     this.logger.debug(
-      `Creator search returned ${data.length} rows for query "${trimmed ?? ''}"`,
+      `Creator search returned ${data.length}/${total} rows for query "${trimmed ?? ''}"`,
     );
 
     return new PaginatedResponseDto(data, total, page, limit);

backend/src/migration.datasource.ts

@@ -8,6 +8,7 @@ import { CreateWalletChallenges1711554834000 } from './auth/1711554834000-Create
 import { CreateIdempotencyKeys1711554835000 } from './idempotency/1711554835000-CreateIdempotencyKeys';
 import { AddQueuedAtToModerationFlags1745000000000 } from './moderation/1745000000000-AddQueuedAtToModerationFlags';
 import { CreateReferralTables1745000000000 } from './referral/1745000000000-CreateReferralTables';
+import { AddDigestColumnsToNotifications1745100000000 } from './notifications/1745100000000-AddDigestColumnsToNotifications';
 
 export const migrationDataSource = new DataSource({
   type: 'postgres',
@@ -25,5 +26,6 @@ export const migrationDataSource = new DataSource({
     CreateIdempotencyKeys1711554835000,
     AddQueuedAtToModerationFlags1745000000000,
     CreateReferralTables1745000000000,
+    AddDigestColumnsToNotifications1745100000000,
   ],
 });

backend/src/notifications/1745100000000-AddDigestColumnsToNotifications.ts

@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddDigestColumnsToNotifications1745100000000
+  implements MigrationInterface
+{
+  async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "notifications" ADD COLUMN IF NOT EXISTS "digest_count" integer NOT NULL DEFAULT 1`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "notifications" ADD COLUMN IF NOT EXISTS "digest_event_times" jsonb`,
+    );
+  }
+
+  async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "notifications" DROP COLUMN IF EXISTS "digest_event_times"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "notifications" DROP COLUMN IF EXISTS "digest_count"`,
+    );
+  }
+}

backend/src/notifications/dto/notification.dto.ts

@@ -1,4 +1,4 @@
-import { IsBoolean, IsEnum, IsOptional, IsString } from 'class-validator';
+import { IsBoolean, IsEnum, IsInt, IsISO8601, IsOptional, IsString, Min } from 'class-validator';
 import { NotificationType } from '../entities/notification.entity';
 
 export class CreateNotificationDto {
@@ -16,6 +16,15 @@ export class CreateNotificationDto {
 
   @IsOptional()
   metadata?: Record<string, unknown>;
+
+  @IsOptional()
+  @IsInt()
+  @Min(1)
+  digest_count?: number;
+
+  @IsOptional()
+  @IsISO8601({}, { each: true })
+  digest_event_times?: string[];
 }
 
 export class MarkReadDto {

backend/src/notifications/entities/notification.entity.ts

@@ -49,6 +49,14 @@ export class Notification {
   @Column({ type: 'jsonb', nullable: true })
   metadata: Record<string, unknown> | null;
 
+  /** Number of individual events collapsed into this digest (1 = not a digest). */
+  @Column({ type: 'int', default: 1 })
+  digest_count: number;
+
+  /** ISO timestamps of the individual events batched into this digest. */
+  @Column({ type: 'jsonb', nullable: true })
+  digest_event_times: string[] | null;
+
   @CreateDateColumn()
   created_at: Date;
 }