Auto-paginate the no-arg List* helpers (#285)

ListIPs, ListAccounts, ListApplications, ListDatabases, ListKubernetesClusters
and ListObjectStores previously sent a single GET /v2/... with no pagination
parameters and silently returned whatever fit on the server-side first page
(default 20 items). For accounts with more resources than that — typical in
production — page 2 onward was invisible to civogo callers including
civo-cli (`civo ip ls`) and terraform-provider-civo (`civo_reserved_ip`,
`civo_kubernetes_cluster`, `civo_object_store`, `civo_database`).

The customer ticket that surfaced this: an account with > 20 Reserved IPs
in fra1 / lon1 could not see one specific IP via the CLI or Terraform.
Tracked in api-go epic civo&598 (server-side off-by-one, fixed in
civo/api-go!1354) and civogo epic civo&599 (this change).

The fix introduces a shared `paginateAll` helper in `pagination.go` and
routes the six no-arg `List*` functions through it. The merged response
always reports `Page=1, Pages=1, PerPage=len(Items)`. Iteration is
sequential and capped (100 pages / 10,000 items) with an explicit
ErrPaginationCapExceeded rather than spinning forever.

Two existing magic-number workarounds were retired in the same change:
- ListAllInstances no longer uses ListInstances(1, 99999999).
- FindObjectStoreCredential no longer uses ListObjectStoreCredentials(1, 10000).

Both depended on api-go's paginator early-return path and would silently
re-truncate the day server-side adds a per_page cap. They now share the
same iterator.

Includes regression tests covering: single page, multi-page in order,
trailing-item-on-last-page (the customer's case), and hard-cap exhaustion.
Updated affected fixtures that asserted the old buggy semantics (Pages>1
on a single-item response, PerPage=20 on the merged result).

Co-authored-by: david.woolger@civo.com <david.woolger@civo.com>

Author: dwoolger

CHANGELOG.md

@@ -1,4 +1,21 @@
 
+Unreleased
+=============
+
+* **Behaviour change**: `ListIPs`, `ListAccounts`, `ListApplications`,
+  `ListDatabases`, `ListKubernetesClusters`, and `ListObjectStores` now
+  iterate server-side pagination internally and return every item the
+  account owns, not just the first 20. The merged response always reports
+  `Page=1, Pages=1, PerPage=len(Items)`. Callers iterating `Items` are
+  unaffected; callers inspecting `Page`/`Pages`/`PerPage` to drive their
+  own iteration should drop that logic.
+* `ListAllInstances` and `FindObjectStoreCredential` no longer use the
+  brittle `per_page=99999999` / `per_page=10000` workaround; both go
+  through the new shared iterator (`paginateAll` in `pagination.go`).
+* Hard cap: each `List*` call returns `ErrPaginationCapExceeded` rather
+  than spinning if the server reports more than 100 pages / 10,000 items.
+* Originating customer report: api-go epic civo&598 + civogo epic civo&599.
+
 0.2.57
 =============
 2021-11-02

account.go

@@ -3,6 +3,7 @@ package civogo
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 )
 
 // PaginatedAccounts returns a paginated list of Account object
@@ -15,17 +16,21 @@ type PaginatedAccounts struct {
 
 // ListAccounts lists all accounts
 func (c *Client) ListAccounts() (*PaginatedAccounts, error) {
-	resp, err := c.SendGetRequest("/v2/accounts")
+	items, err := paginateAll("accounts", func(page, perPage int) ([]Account, int, error) {
+		resp, err := c.SendGetRequest(fmt.Sprintf("/v2/accounts?page=%d&per_page=%d", page, perPage))
+		if err != nil {
+			return nil, 0, decodeError(err)
+		}
+		pg := PaginatedAccounts{}
+		if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&pg); err != nil {
+			return nil, 0, decodeError(err)
+		}
+		return pg.Items, pg.Pages, nil
+	})
 	if err != nil {
-		return nil, decodeError(err)
+		return nil, err
 	}
-
-	accounts := &PaginatedAccounts{}
-	if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&accounts); err != nil {
-		return nil, decodeError(err)
-	}
-
-	return accounts, nil
+	return &PaginatedAccounts{Page: 1, Pages: 1, PerPage: len(items), Items: items}, nil
 }
 
 // GetAccountID returns the account ID

application.go

@@ -75,17 +75,21 @@ var ErrAppDomainNotFound = fmt.Errorf("domain not found")
 
 // ListApplications returns all applications in that specific region
 func (c *Client) ListApplications() (*PaginatedApplications, error) {
-	resp, err := c.SendGetRequest("/v2/applications")
+	items, err := paginateAll("applications", func(page, perPage int) ([]Application, int, error) {
+		resp, err := c.SendGetRequest(fmt.Sprintf("/v2/applications?page=%d&per_page=%d", page, perPage))
+		if err != nil {
+			return nil, 0, decodeError(err)
+		}
+		pg := PaginatedApplications{}
+		if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&pg); err != nil {
+			return nil, 0, decodeError(err)
+		}
+		return pg.Items, pg.Pages, nil
+	})
 	if err != nil {
-		return nil, decodeError(err)
-	}
-
-	application := &PaginatedApplications{}
-	if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&application); err != nil {
-		return nil, decodeError(err)
+		return nil, err
 	}
-
-	return application, nil
+	return &PaginatedApplications{Page: 1, Pages: 1, PerPage: len(items), Items: items}, nil
 }
 
 // GetApplication returns an application by ID

application_test.go

@@ -40,9 +40,11 @@ func TestListApplications(t *testing.T) {
 		return
 	}
 
+	// The SDK auto-paginates: the merged response always reports
+	// Page=1, Pages=1, PerPage=len(Items) — see paginateAll in pagination.go.
 	expected := &PaginatedApplications{
 		Page:    1,
-		PerPage: 20,
+		PerPage: 1,
 		Pages:   1,
 		Items: []Application{
 			{

database.go

@@ -81,17 +81,21 @@ type RestoreDatabaseRequest struct {
 
 // ListDatabases returns a list of all databases
 func (c *Client) ListDatabases() (*PaginatedDatabases, error) {
-	resp, err := c.SendGetRequest("/v2/databases")
+	items, err := paginateAll("databases", func(page, perPage int) ([]Database, int, error) {
+		resp, err := c.SendGetRequest(fmt.Sprintf("/v2/databases?page=%d&per_page=%d", page, perPage))
+		if err != nil {
+			return nil, 0, decodeError(err)
+		}
+		pg := PaginatedDatabases{}
+		if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&pg); err != nil {
+			return nil, 0, err
+		}
+		return pg.Items, pg.Pages, nil
+	})
 	if err != nil {
-		return nil, decodeError(err)
-	}
-
-	databases := &PaginatedDatabases{}
-	if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&databases); err != nil {
 		return nil, err
 	}
-
-	return databases, nil
+	return &PaginatedDatabases{Page: 1, Pages: 1, PerPage: len(items), Items: items}, nil
 }
 
 // GetDatabase finds a database by the database UUID

database_test.go

@@ -7,7 +7,7 @@ import (
 
 func TestListDatabases(t *testing.T) {
 	client, server, _ := NewClientForTesting(map[string]string{
-		"/v2/databases": `{"page": 1, "per_page": 20, "pages": 2, "items":[{"id": "12345", "name": "test-db"}]}`,
+		"/v2/databases": `{"page": 1, "per_page": 20, "pages": 1, "items":[{"id": "12345", "name": "test-db"}]}`,
 	})
 	defer server.Close()
 
@@ -17,10 +17,12 @@ func TestListDatabases(t *testing.T) {
 		return
 	}
 
+	// The SDK auto-paginates: the merged response always reports
+	// Page=1, Pages=1, PerPage=len(Items) — see paginateAll in pagination.go.
 	expected := &PaginatedDatabases{
 		Page:    1,
-		PerPage: 20,
-		Pages:   2,
+		PerPage: 1,
+		Pages:   1,
 		Items: []Database{
 			{
 				ID:   "12345",

instance.go

@@ -152,14 +152,20 @@ func (c *Client) ListInstances(page int, perPage int) (*PaginatedInstanceList, e
 	return &PaginatedInstances, err
 }
 
-// ListAllInstances returns all (well, upto 99,999,999 instances) Instances owned by the calling API account
+// ListAllInstances returns every Instance owned by the calling API account
+// by iterating the paginated /v2/instances endpoint until exhausted.
 func (c *Client) ListAllInstances() ([]Instance, error) {
-	instances, err := c.ListInstances(1, 99999999)
+	items, err := paginateAll("instances", func(page, perPage int) ([]Instance, int, error) {
+		pg, err := c.ListInstances(page, perPage)
+		if err != nil {
+			return nil, 0, err
+		}
+		return pg.Items, pg.Pages, nil
+	})
 	if err != nil {
-		return []Instance{}, decodeError(err)
+		return []Instance{}, err
 	}
-
-	return instances.Items, nil
+	return items, nil
 }
 
 // FindInstance finds a instance by either part of the ID or part of the hostname

instance_test.go

@@ -6,7 +6,7 @@ import (
 
 func TestListInstances(t *testing.T) {
 	client, server, _ := NewClientForTesting(map[string]string{
-		"/v2/instances": `{"page": 1, "per_page": 20, "pages": 2, "items":[{"id": "12345", "hostname": "foo.example.com"}]}`,
+		"/v2/instances": `{"page": 1, "per_page": 20, "pages": 1, "items":[{"id": "12345", "hostname": "foo.example.com"}]}`,
 	})
 	defer server.Close()
 
@@ -26,7 +26,7 @@ func TestFindInstance(t *testing.T) {
 		{
 			"page": 1,
 			"per_page": 20,
-			"pages": 2,
+			"pages": 1,
 			"items": [
 				{
 					"id": "12345",

ip.go

@@ -61,17 +61,21 @@ type Actions struct {
 
 // ListIPs returns all reserved IPs in that specific region
 func (c *Client) ListIPs() (*PaginatedIPs, error) {
-	resp, err := c.SendGetRequest("/v2/ips")
+	items, err := paginateAll("ips", func(page, perPage int) ([]IP, int, error) {
+		resp, err := c.SendGetRequest(fmt.Sprintf("/v2/ips?page=%d&per_page=%d", page, perPage))
+		if err != nil {
+			return nil, 0, decodeError(err)
+		}
+		pg := PaginatedIPs{}
+		if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&pg); err != nil {
+			return nil, 0, err
+		}
+		return pg.Items, pg.Pages, nil
+	})
 	if err != nil {
-		return nil, decodeError(err)
-	}
-
-	ips := &PaginatedIPs{}
-	if err := json.NewDecoder(bytes.NewReader(resp)).Decode(&ips); err != nil {
 		return nil, err
 	}
-
-	return ips, nil
+	return &PaginatedIPs{Page: 1, Pages: 1, PerPage: len(items), Items: items}, nil
 }
 
 // GetIP finds an reserved IP by the full ID

ip_test.go

@@ -28,9 +28,11 @@ func TestListIPs(t *testing.T) {
 		return
 	}
 
+	// The SDK auto-paginates: the merged response always reports
+	// Page=1, Pages=1, PerPage=len(Items) — see paginateAll in pagination.go.
 	expected := &PaginatedIPs{
 		Page:    1,
-		PerPage: 20,
+		PerPage: 1,
 		Pages:   1,
 		Items: []IP{
 			{