feat: Add --include-non-partition-tables flag and fix scan error handling

- Add IncludeNonPartitionTables config option and --include-non-partition-tables flag
- Create nonPartitionTableListSQL query to find regular tables matching partition patterns
- Update partition discovery in archiver.go and progress.go to optionally include non-partition tables
- Fix scan error handling in progress.go to properly propagate errors instead of silently ignoring them
- Add partition validation to skip tables without columns during discovery
- Refactor partition discovery to use shared helper function for consistency

Fixes issue where tables matching partition naming patterns but not actually being
PostgreSQL partitions were causing errors. Now users can optionally include these
tables for processing.

Version 1.6.0

Author: kevinelliott

CHANGELOG.md

@@ -7,8 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.7.0] - 2025-11-24
+
 ### Added
 - **Archive Command:**
+  - New `--include-non-partition-tables` flag to include regular tables matching partition naming patterns (not just actual PostgreSQL partitions)
   - When a table lacks physical partitions, providing `--date-column`, `--start-date`, and `--end-date` now enables synthetic date-window processing so partitionless tables can be archived with the standard workflow
 - **pg_dump & dump-hybrid**
   - Date-window dumps now reuse the global cache so completed windows or partition groups are skipped immediately on reruns when the S3 object already exists and matches size/MD5 (or multipart ETag)
@@ -18,6 +21,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - Cache files are namespaced by subcommand plus the absolute S3 destination, preventing collisions when the same table is archived by different workflows or sent to different paths
   - Existing per-table caches are migrated automatically the first time a scoped cache is loaded
 
+### Fixed
+- **Archive & pg_dump Commands:**
+  - Partition discovery now only returns tables that inherit from the requested parent table, preventing lookups against similarly named tables that are not real partitions (and the resulting "table not found or has no columns" errors)
+  - Partition permission checks use the same metadata so missing privileges on actual partitions are reported accurately instead of being mistaken for missing tables
+  - Scan errors during partition discovery in TUI mode are now properly propagated instead of being silently ignored, matching the behavior in debug mode
+
 ## [1.5.9] - 2025-11-17
 
 ### Fixed

cmd/archiver.go

@@ -347,22 +347,61 @@ func (a *Archiver) runArchivalProcess(ctx context.Context, _ *tea.Program, _ *Ta
 	}
 
 	a.logger.Debug("Discovering partitions...")
-	// Inline partition discovery (same logic as progress.go doDiscover)
-	query := `
-		SELECT c.relname::text AS tablename
-		FROM pg_class c
-		JOIN pg_namespace n ON n.oid = c.relnamespace
-		WHERE n.nspname = 'public'
-			AND c.relname LIKE $1
-			AND c.relkind = 'r'
-			AND NOT EXISTS (
-				SELECT 1 FROM pg_inherits WHERE inhparent = c.oid
-			)
-		ORDER BY c.relname;
-	`
+	var partitions []PartitionInfo
+	seenTables := make(map[string]bool) // Track tables to avoid duplicates
 
-	pattern := a.config.Table + "_%"
-	rows, err := a.db.QueryContext(ctx, query, pattern)
+	// Helper function to process tables from a query result
+	processTableRows := func(rows *sql.Rows, sourceType string) error {
+		defer rows.Close()
+		for rows.Next() {
+			// Check for cancellation in the loop
+			select {
+			case <-ctx.Done():
+				a.logger.Info("⚠️  Cancellation detected during partition discovery")
+				return ctx.Err()
+			default:
+			}
+
+			var tableName string
+			if err := rows.Scan(&tableName); err != nil {
+				return fmt.Errorf("failed to scan %s name: %w", sourceType, err)
+			}
+
+			// Skip if we've already seen this table
+			if seenTables[tableName] {
+				continue
+			}
+			seenTables[tableName] = true
+
+			date, ok := a.extractDateFromTableName(tableName)
+			if !ok {
+				a.logger.Debug(fmt.Sprintf("Skipping table %s (no valid date)", tableName))
+				continue
+			}
+
+			// Validate that the table actually has columns before adding it
+			// This prevents errors later when trying to process tables that exist
+			// but have no columns or aren't valid tables
+			schema, schemaErr := a.getTableSchema(ctx, tableName)
+			if schemaErr != nil {
+				a.logger.Debug(fmt.Sprintf("Skipping table %s (schema validation failed: %v)", tableName, schemaErr))
+				continue
+			}
+			if schema == nil || len(schema.Columns) == 0 {
+				a.logger.Debug(fmt.Sprintf("Skipping table %s (no columns found)", tableName))
+				continue
+			}
+
+			partitions = append(partitions, PartitionInfo{
+				TableName: tableName,
+				Date:      date,
+			})
+		}
+		return rows.Err()
+	}
+
+	// Query actual partitions
+	rows, err := a.db.QueryContext(ctx, leafPartitionListSQL, defaultTableSchema, a.config.Table)
 	if err != nil {
 		// Check if error is due to cancellation or closed connection
 		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || isConnectionError(err) {
@@ -371,46 +410,25 @@ func (a *Archiver) runArchivalProcess(ctx context.Context, _ *tea.Program, _ *Ta
 		}
 		return fmt.Errorf("failed to query partitions: %w", err)
 	}
-	defer rows.Close()
-
-	// Check for cancellation immediately after query starts
-	select {
-	case <-ctx.Done():
-		a.logger.Info("⚠️  Cancellation detected after query started")
-		return ctx.Err()
-	default:
+	if err := processTableRows(rows, "partition"); err != nil {
+		return fmt.Errorf("error iterating over partition rows: %w", err)
 	}
 
-	var partitions []PartitionInfo
-	for rows.Next() {
-		// Check for cancellation in the loop
-		select {
-		case <-ctx.Done():
-			a.logger.Info("⚠️  Cancellation detected during partition discovery")
-			return ctx.Err()
-		default:
-		}
-
-		var tableName string
-		if err := rows.Scan(&tableName); err != nil {
-			return fmt.Errorf("failed to scan partition name: %w", err)
+	// If enabled, also query non-partition tables matching the pattern
+	if a.config.IncludeNonPartitionTables {
+		a.logger.Debug("Including non-partition tables matching pattern...")
+		nonPartitionRows, err := a.db.QueryContext(ctx, nonPartitionTableListSQL, defaultTableSchema, a.config.Table)
+		if err != nil {
+			// Check if error is due to cancellation or closed connection
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || isConnectionError(err) {
+				a.logger.Info("⚠️  Query cancelled or connection closed")
+				return context.Canceled
+			}
+			return fmt.Errorf("failed to query non-partition tables: %w", err)
 		}
-
-		date, ok := a.extractDateFromTableName(tableName)
-		if !ok {
-			a.logger.Debug(fmt.Sprintf("Skipping table %s (no valid date)", tableName))
-			continue
+		if err := processTableRows(nonPartitionRows, "non-partition table"); err != nil {
+			return fmt.Errorf("error iterating over non-partition table rows: %w", err)
 		}
-
-		partitions = append(partitions, PartitionInfo{
-			TableName: tableName,
-			Date:      date,
-		})
-	}
-
-	// Check for errors from iterating over rows
-	if err := rows.Err(); err != nil {
-		return fmt.Errorf("error iterating over partition rows: %w", err)
 	}
 
 	if len(partitions) == 0 {
@@ -601,18 +619,8 @@ func (a *Archiver) checkTablePermissions(ctx context.Context) error {
 	}
 
 	// Check if we can see and access partition tables
-	pattern := a.config.Table + "_%"
-	partitionCheckQuery := `
-		SELECT tablename
-		FROM pg_tables
-		WHERE schemaname = 'public'
-		AND tablename LIKE $1
-		AND has_table_privilege('public.' || tablename, 'SELECT')
-		LIMIT 1
-	`
-
 	var samplePartition string
-	err = a.db.QueryRowContext(ctx, partitionCheckQuery, pattern).Scan(&samplePartition)
+	err = a.db.QueryRowContext(ctx, leafPartitionPermissionSQL, defaultTableSchema, a.config.Table).Scan(&samplePartition)
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
 		// Only fail if it's not a "no rows" error
 		return fmt.Errorf("failed to check partition table permissions: %w", err)
@@ -622,14 +630,9 @@ func (a *Archiver) checkTablePermissions(ctx context.Context) error {
 	if errors.Is(err, sql.ErrNoRows) {
 		// Let's see if partitions exist but we can't access them
 		var partitionExists bool
-		existsQuery := `
-			SELECT EXISTS (
-				SELECT 1 FROM pg_tables
-				WHERE schemaname = 'public'
-				AND tablename LIKE $1
-			)
-		`
-		_ = a.db.QueryRowContext(ctx, existsQuery, pattern).Scan(&partitionExists)
+		if err := a.db.QueryRowContext(ctx, leafPartitionExistsSQL, defaultTableSchema, a.config.Table).Scan(&partitionExists); err != nil {
+			return fmt.Errorf("failed to check for partition existence: %w", err)
+		}
 
 		if partitionExists {
 			// Partitions exist but we can't access them

cmd/config.go

@@ -41,26 +41,27 @@ var (
 const regionAuto = "auto"
 
 type Config struct {
-	Debug            bool
-	LogFormat        string
-	DryRun           bool
-	Workers          int
-	SkipCount        bool
-	CacheViewer      bool
-	ViewerPort       int
-	ChunkSize        int // Number of rows to process in each chunk (streaming mode)
-	Database         DatabaseConfig
-	S3               S3Config
-	Table            string
-	StartDate        string
-	EndDate          string
-	OutputDuration   string
-	OutputFormat     string
-	Compression      string
-	CompressionLevel int
-	DateColumn       string
-	DumpMode         string // pg_dump mode: schema-only, data-only, schema-and-data
-	CacheScope       CacheScope
+	Debug                    bool
+	LogFormat                string
+	DryRun                   bool
+	Workers                  int
+	SkipCount                bool
+	CacheViewer              bool
+	ViewerPort               int
+	ChunkSize                int // Number of rows to process in each chunk (streaming mode)
+	IncludeNonPartitionTables bool // Include regular tables matching partition naming pattern
+	Database                 DatabaseConfig
+	S3                       S3Config
+	Table                    string
+	StartDate                string
+	EndDate                  string
+	OutputDuration           string
+	OutputFormat             string
+	Compression              string
+	CompressionLevel         int
+	DateColumn               string
+	DumpMode                 string // pg_dump mode: schema-only, data-only, schema-and-data
+	CacheScope               CacheScope
 }
 
 type DatabaseConfig struct {

cmd/partition_queries.go

@@ -0,0 +1,60 @@
+package cmd
+
+const defaultTableSchema = "public"
+
+const leafPartitionBaseCTE = `
+WITH leaf_partitions AS (
+	SELECT
+		child.relname::text AS tablename,
+		child.oid AS table_oid
+	FROM pg_inherits i
+		JOIN pg_class child ON child.oid = i.inhrelid
+		JOIN pg_namespace child_ns ON child_ns.oid = child.relnamespace
+		JOIN pg_class parent ON parent.oid = i.inhparent
+		JOIN pg_namespace parent_ns ON parent_ns.oid = parent.relnamespace
+	WHERE parent_ns.nspname = $1
+		AND child_ns.nspname = $1
+		AND parent.relname = $2
+		AND child.relkind = 'r'
+		AND NOT EXISTS (
+			SELECT 1 FROM pg_inherits WHERE inhparent = child.oid
+		)
+)`
+
+const leafPartitionListSQL = leafPartitionBaseCTE + `
+SELECT tablename
+FROM leaf_partitions
+ORDER BY tablename;
+`
+
+const leafPartitionExistsSQL = leafPartitionBaseCTE + `
+SELECT EXISTS (
+	SELECT 1 FROM leaf_partitions
+);
+`
+
+const leafPartitionPermissionSQL = leafPartitionBaseCTE + `
+SELECT tablename
+FROM leaf_partitions
+WHERE has_table_privilege(table_oid, 'SELECT')
+LIMIT 1;
+`
+
+// nonPartitionTableListSQL finds regular tables (not partitions) that match
+// the partition naming pattern. This is useful when tables follow partition
+// naming conventions but aren't actually PostgreSQL partitions.
+const nonPartitionTableListSQL = `
+SELECT t.tablename::text
+FROM pg_tables t
+	JOIN pg_class c ON c.relname = t.tablename
+	JOIN pg_namespace n ON n.oid = c.relnamespace
+WHERE t.schemaname = $1
+	AND n.nspname = $1
+	AND t.tablename LIKE $2 || '_%'
+	AND t.tablename != $2
+	AND c.relkind = 'r'
+	AND NOT EXISTS (
+		SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid
+	)
+ORDER BY t.tablename;
+`

cmd/pg_dump.go

@@ -649,17 +649,8 @@ func (e *PgDumpExecutor) dumpPartitionsByDateRange(ctx context.Context) error {
 		}
 	}
 
-	// Discover partitions matching the table pattern
-	pattern := e.config.Table + "_%"
-	query := `
-		SELECT tablename
-		FROM pg_tables
-		WHERE schemaname = 'public'
-			AND tablename LIKE $1
-		ORDER BY tablename
-	`
-
-	rows, err := e.db.QueryContext(ctx, query, pattern)
+	// Discover partitions that inherit from the target table
+	rows, err := e.db.QueryContext(ctx, leafPartitionListSQL, defaultTableSchema, e.config.Table)
 	if err != nil {
 		return fmt.Errorf("failed to query partitions: %w", err)
 	}