Fix pagination for larger buckets

Fixes #90

Author: cloudlena

CLAUDE.md

@@ -47,7 +47,7 @@ HTTP request
 
 ### Key handler categories
 
-- **Views** (`HandleBucketsView`, `HandleBucketView`): Render full-page HTML templates.
+- **Views** (`HandleBucketsView`, `HandleBucketView`): Render full-page HTML templates. `HandleBucketView` has two listing paths, picked by `cursorPagingPossible`: the default name-ascending view is listed one page at a time through S3's own `StartAfter`/`MaxKeys` paging (`listObjectPage`), which is why it has no total count; every other view (other sorts, search, `All`, versions) needs the whole prefix and falls back to a scan capped at `maxScanObjects` (`listAllObjects`).
 - **CRUD** (`HandleCreateBucket`, `HandleCreateObject`, `HandleDeleteBucket`, `HandleDeleteObject`): REST-ish JSON/form handlers.
 - **Bulk** (`HandleBulkDeleteObjects`, `HandleBulkDownloadObjects`): Batch delete or ZIP-stream multiple objects.
 - **URL** (`HandleGenerateURL`): Returns presigned S3 download URLs.

README.md

@@ -87,6 +87,21 @@ These variables apply to the whole app and are never prefixed:
 - `TIMEOUT`: The read and write timeout in seconds (default to `600` - 10 minutes)
 - `ROOT_URL`: A root URL prefix if running behind a reverse proxy (defaults to unset)
 
+### Browsing large buckets
+
+The default bucket view asks S3 for one page of objects at a time, so opening a
+bucket and stepping through it costs the same whether it holds ten objects or ten
+million. Because S3 can only list keys in ascending order and offers no search of
+its own, that page-at-a-time listing is possible only for the default view: it is
+sorted by name ascending and unsearched. Such a view has no total object count
+and no last page to jump to, since S3 never reports how much it did not return.
+
+Sorting by another column, sorting descending, searching or choosing `All` items
+per page needs the whole location in memory instead, and so does `SHOW_VERSIONS`.
+Those listings stop after 10,000 objects and say so on the page — their counting,
+sorting and searching then cover only that many. Narrow the listing down with a
+search or by opening a folder to reach the rest.
+
 ### Build and Run Locally
 
 1.  Run `make build`

internal/app/s3manager/bucket_view.go

@@ -25,6 +25,21 @@ type listingQuery struct {
 	PerPage   int
 	ShowAll   bool
 	Search    string
+	// Cursors is the trail of the cursors of the preceding pages, one per page,
+	// as collected by the pager of a cursor-paged view. Keeping the whole trail
+	// in the URL rather than just the current cursor is what lets such a view
+	// still number its pages and step back to the previous one.
+	Cursors []string
+}
+
+// Cursor is the key the current page's listing resumes after, empty on the first
+// page.
+func (q listingQuery) Cursor() string {
+	if len(q.Cursors) == 0 {
+		return ""
+	}
+
+	return q.Cursors[len(q.Cursors)-1]
 }
 
 // HandleBucketView shows the details page of a bucket.
@@ -47,6 +62,8 @@ func HandleBucketView(instances S3Instances, templates fs.FS, opts Options) http
 		ShowVersions        bool
 		VersionsUnavailable bool
 		ShowMetadata        bool
+		Truncated           bool
+		MaxScanObjects      int
 	}
 
 	renderer := newPageRenderer(templates, "bucket.html.tmpl")
@@ -65,21 +82,32 @@ func HandleBucketView(instances S3Instances, templates fs.FS, opts Options) http
 
 		query := parseListingQuery(r.URL.Query())
 		data := pageData{
-			RootURL:      opts.RootURL,
-			BucketName:   bucketName,
-			CurrentPath:  path,
-			Paths:        removeEmptyStrings(strings.Split(path, "/")),
-			Endpoint:     instance.Client.EndpointURL().String(),
-			AllowDelete:  opts.AllowDelete,
-			CurrentS3:    instance,
-			S3Instances:  instances,
-			SortBy:       query.SortBy,
-			SortOrder:    query.SortOrder,
-			Search:       query.Search,
-			ShowMetadata: opts.ShowMetadata,
+			RootURL:        opts.RootURL,
+			BucketName:     bucketName,
+			CurrentPath:    path,
+			Paths:          removeEmptyStrings(strings.Split(path, "/")),
+			Endpoint:       instance.Client.EndpointURL().String(),
+			AllowDelete:    opts.AllowDelete,
+			CurrentS3:      instance,
+			S3Instances:    instances,
+			SortBy:         query.SortBy,
+			SortOrder:      query.SortOrder,
+			Search:         query.Search,
+			ShowMetadata:   opts.ShowMetadata,
+			MaxScanObjects: maxScanObjects,
 		}
 
-		objs, versionsShown, err := listObjects(r.Context(), instance.Client, bucketName, path, opts.ListRecursive, opts.ShowVersions)
+		// The default view is served one page at a time straight from S3, which
+		// keeps its cost independent of how many objects the bucket holds. Every
+		// other view needs the whole prefix in memory to do its job.
+		cursorPaging := cursorPagingPossible(query, opts)
+
+		var listing objectListing
+		if cursorPaging {
+			listing, err = listObjectPage(r.Context(), instance.Client, bucketName, path, query.Cursor(), opts.ListRecursive, query.PerPage)
+		} else {
+			listing, err = listAllObjects(r.Context(), instance.Client, bucketName, path, opts.ListRecursive, opts.ShowVersions)
+		}
 		if err != nil {
 			// A failed listing is reported on the page itself so that the user
 			// can switch instances or go back instead of being stuck on an
@@ -90,23 +118,43 @@ func HandleBucketView(instances S3Instances, templates fs.FS, opts Options) http
 			return
 		}
 
-		if versionsShown {
-			annotateVersionGroups(objs)
-		}
-		if query.Search != "" {
-			objs = filterObjects(objs, query.Search)
+		if cursorPaging {
+			data.objectPage = cursorPage(listing, query)
+		} else {
+			if listing.VersionsShown {
+				annotateVersionGroups(listing.Objects)
+			}
+			if query.Search != "" {
+				listing.Objects = filterObjects(listing.Objects, query.Search)
+			}
+			data.objectPage = paginateObjects(listing.Objects, query, listing.VersionsShown)
 		}
 
-		data.objectPage = paginateObjects(objs, query, versionsShown)
-		data.ShowVersions = versionsShown
+		data.ShowVersions = listing.VersionsShown
+		data.Truncated = listing.Truncated
 		// Only warn about unavailable versions when there is content to show;
 		// an empty bucket legitimately produces an empty versioned listing.
-		data.VersionsUnavailable = opts.ShowVersions && !versionsShown && len(objs) > 0
+		data.VersionsUnavailable = opts.ShowVersions && !listing.VersionsShown && len(listing.Objects) > 0
 
 		renderer(w, data)
 	}
 }
 
+// cursorPagingPossible reports whether a request can be served by listing only
+// the page it shows. S3 lists keys in ascending lexicographic order and can
+// resume after a given key, but it cannot sort by anything else, cannot list
+// backwards and has no search — so sorting by another column, sorting
+// descending, searching and asking for every object at once all still need the
+// full listing. So does a versioned listing, whose version groups have to be
+// assembled before they can be split into pages.
+func cursorPagingPossible(query listingQuery, opts Options) bool {
+	return !opts.ShowVersions &&
+		!query.ShowAll &&
+		query.Search == "" &&
+		query.SortBy == "key" &&
+		query.SortOrder == "asc"
+}
+
 // parseBucketPath extracts the bucket name and the path within the bucket from
 // a bucket view URL.
 func parseBucketPath(urlPath string) (string, string, error) {
@@ -127,6 +175,7 @@ func parseListingQuery(params url.Values) listingQuery {
 		Page:      1,
 		PerPage:   defaultPerPage,
 		Search:    strings.TrimSpace(params.Get("search")),
+		Cursors:   removeEmptyStrings(params["cursor"]),
 	}
 
 	if query.SortBy == "" {