TT-18143: fix zero pruned size in retention plans (#544)

TT-18143: fix zero pruned size in retention plans (#544)

Author: olamilekan000

.github/workflows/retention-plan.yml

@@ -71,8 +71,11 @@ jobs:
           # shellcheck disable=SC2086
           go run . pkgs plan --grace-days "$GRACE_DAYS" $REPOS > plan.txt
 
-      # Newly eligible = checksums in this plan that were not in the
-      # previous committed plan. Only these trigger the Slack notice.
+      # One released file is indexed once per distro version on
+      # Packagecloud, so entry counts run ~10x higher than file counts;
+      # checksums identify the underlying files. Newly eligible =
+      # checksums in this plan that were not in the previous committed
+      # plan. Only these trigger the Slack notice.
       - name: Diff against previous plan
         id: diff
         run: |
@@ -85,6 +88,7 @@ jobs:
           fi
           comm -23 current.shas previous.shas > new.shas
           echo "new_count=$(wc -l < new.shas | tr -d ' ')" >> "$GITHUB_OUTPUT"
+          echo "total_files=$(wc -l < current.shas | tr -d ' ')" >> "$GITHUB_OUTPUT"
           echo "total_pruned=$(jq '[.[].pruned] | add' plan.json)" >> "$GITHUB_OUTPUT"
           echo "total_gib=$(jq '[.[].pruned_bytes] | add / 1073741824 * 10 | round / 10' plan.json)" >> "$GITHUB_OUTPUT"
           echo "not_before=$(jq -r '.[0].not_before[:10]' plan.json)" >> "$GITHUB_OUTPUT"
@@ -121,7 +125,7 @@ jobs:
             echo "|------|----------|----------|--------|------------|------------|"
             jq -r '.[] | "| \(.repo) | \(.retained + .pruned) | \(.retained) | \(.pruned) | \(.pruned_bytes / 1073741824 * 10 | round / 10) | \(.not_before[:10]) |"' plan.json
             echo ""
-            echo "$NEW_COUNT newly eligible packages since the previous plan."
+            echo "$NEW_COUNT files newly eligible since the previous plan."
             echo "Committed as [\`$COMMIT\`](https://github.com/TykTechnologies/artifact-retention-plans/commit/$COMMIT)."
             echo ""
             echo "Nothing was deleted by this workflow."
@@ -141,7 +145,7 @@ jobs:
                   "type": "section",
                   "text": {
                     "type": "mrkdwn",
-                    "text": "*Artifact retention plan published* — @support-team\n${{ steps.diff.outputs.new_count }} packages are newly eligible for pruning (${{ steps.diff.outputs.total_pruned }} total eligible, ${{ steps.diff.outputs.total_gib }} GiB). Nothing will be deleted before *${{ steps.diff.outputs.not_before }}*.\n<https://github.com/TykTechnologies/artifact-retention-plans/commit/${{ steps.commit.outputs.sha }}|Review the committed plan>"
+                    "text": "*Artifact retention plan published* — @support-team\n${{ steps.diff.outputs.new_count }} files are newly eligible for pruning since the last plan. In total ${{ steps.diff.outputs.total_files }} files are eligible, which appear as ${{ steps.diff.outputs.total_pruned }} package listings on Packagecloud (one per distro version, ${{ steps.diff.outputs.total_gib }} GiB). Nothing will be deleted before *${{ steps.diff.outputs.not_before }}*.\n<https://github.com/TykTechnologies/artifact-retention-plans/commit/${{ steps.commit.outputs.sha }}|Review the committed plan>"
                   }
                 }
               ]

cmd/pkgs.go

@@ -119,6 +119,7 @@ checksums, which later execution stages verify before deleting.`,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		asJSON, _ := cmd.Flags().GetBool("json")
 		graceDays, _ := cmd.Flags().GetInt("grace-days")
+		concurrency, _ := cmd.Flags().GetInt("concurrency")
 		grace := time.Duration(graceDays) * 24 * time.Hour
 		tracks, err := pkgs.LoadTracks()
 		if err != nil {
@@ -138,6 +139,7 @@ checksums, which later execution stages verify before deleting.`,
 			if err != nil {
 				return fmt.Errorf("planning %s: %w", repoName, err)
 			}
+			pkgClient.FillPrunedBytes(&plan, items, concurrency)
 			plans = append(plans, plan)
 			if !asJSON {
 				fmt.Fprint(cmd.OutOrStdout(), plan.Render())
@@ -229,6 +231,7 @@ func init() {
 
 	planSubCmd.Flags().Bool("json", false, "Emit the plan as JSON, including the prune-eligible package list")
 	planSubCmd.Flags().Int("grace-days", 30, "Days until the plan's not_before deadline; override for the 90-day launch notice")
+	planSubCmd.Flags().Int("concurrency", 8, "Concurrent size lookups, bounded overall by rps/burst")
 
 	mirrorSubCmd.Flags().String("plan", "", "Plan file from 'pkgs plan --json'")
 	mirrorSubCmd.MarkFlagRequired("plan")

pkgs/plan.go

@@ -3,13 +3,16 @@ package pkgs
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"sort"
-	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	pc "github.com/tyklabs/packagecloud/api/v1"
 	"golang.org/x/mod/semver"
+	"golang.org/x/sync/errgroup"
 )
 
 // Plan is a dry-run report of what the retention policy would prune
@@ -29,8 +32,9 @@ type Plan struct {
 	Cutoff string   `json:"cutoff,omitempty"`
 	Series []string `json:"series"`
 
-	Retained    int   `json:"retained"`
-	Pruned      int   `json:"pruned"`
+	Retained int `json:"retained"`
+	Pruned   int `json:"pruned"`
+	// PrunedBytes counts each unique file once
 	PrunedBytes int64 `json:"pruned_bytes"`
 
 	PrunedSeries map[string]int `json:"pruned_series,omitempty"`
@@ -150,9 +154,6 @@ func BuildPlan(repoName string, cfg pkgConfig, tracks Tracks, items []pc.Package
 		if prune {
 			p.Pruned++
 			p.PrunedSeries[semver.MajorMinor(v)]++
-			if sz, err := strconv.ParseInt(item.Size, 10, 64); err == nil {
-				p.PrunedBytes += sz
-			}
 			p.Packages = append(p.Packages, PlanPackage{
 				Name:          item.Name,
 				Version:       item.Version,
@@ -169,6 +170,66 @@ func BuildPlan(repoName string, cfg pkgConfig, tracks Tracks, items []pc.Package
 	return p, nil
 }
 
+// FillPrunedBytes sets p.PrunedBytes from the Content-Length of each
+// unique prune-eligible file; the listing API does not return sizes.
+// The count is advisory, so failures are logged and skipped.
+func (c *Client) FillPrunedBytes(p *Plan, items []pc.PackageDetail, concurrency int) {
+	urlBySha := make(map[string]string, len(items))
+	for _, item := range items {
+		urlBySha[item.Sha256Sum] = item.DownloadURL
+	}
+	seen := make(map[string]bool, len(p.Packages))
+	var total atomic.Int64
+	g := new(errgroup.Group)
+	g.SetLimit(concurrency)
+	for _, pp := range p.Packages {
+		if seen[pp.Sha256Sum] {
+			continue
+		}
+		seen[pp.Sha256Sum] = true
+		url, found := urlBySha[pp.Sha256Sum]
+		if !found {
+			log.Warn().Str("sha256", pp.Sha256Sum).Msgf("sizing %s: not in the repo listing", pp.Filename)
+			continue
+		}
+		g.Go(func() error {
+			size, err := c.headSize(url)
+			if err != nil {
+				log.Warn().Err(err).Msgf("sizing %s", pp.Filename)
+				return nil
+			}
+			total.Add(size)
+			return nil
+		})
+	}
+	_ = g.Wait()
+	p.PrunedBytes = total.Load()
+}
+
+// headSize returns the Content-Length of a download URL
+func (c *Client) headSize(url string) (int64, error) {
+	req, err := http.NewRequestWithContext(c.ctx, "HEAD", url, nil)
+	if err != nil {
+		return 0, err
+	}
+	req.SetBasicAuth(c.token, "")
+	if err := c.limiter.Wait(c.ctx); err != nil {
+		return 0, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("%s: %s", url, resp.Status)
+	}
+	if resp.ContentLength < 0 {
+		return 0, fmt.Errorf("%s: no content length", url)
+	}
+	return resp.ContentLength, nil
+}
+
 // Render returns a human-readable summary of the plan
 func (p Plan) Render() string {
 	var b strings.Builder

pkgs/plan_test.go

@@ -1,6 +1,8 @@
 package pkgs
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -20,7 +22,6 @@ func pkg(version string, age time.Duration) pc.PackageDetail {
 		DistroVersion: "ubuntu/jammy",
 		Filename:      "tyk-test_" + version + "_amd64.deb",
 		Sha256Sum:     "sha-" + version,
-		Size:          "1000",
 		CreateTime:    planNow.Add(-age),
 	}
 }
@@ -60,7 +61,6 @@ func TestBuildPlanTrackDriven(t *testing.T) {
 	assert.Equal(t, 1, plan.NonSemver)
 	assert.Equal(t, map[string]int{"v3.0.9": 1}, plan.Protected)
 	assert.Equal(t, map[string]int{"v2.8": 1, "v2.9": 1}, plan.PrunedSeries)
-	assert.Equal(t, int64(2000), plan.PrunedBytes)
 
 	// the prune list carries the tamper-evident identity
 	require.Len(t, plan.Packages, 2)
@@ -94,6 +94,35 @@ func TestBuildPlanStatic(t *testing.T) {
 	assert.Equal(t, map[string]int{"v1.8.2": 1}, plan.Protected)
 }
 
+func TestFillPrunedBytes(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, http.MethodHead, r.Method)
+		w.Header().Set("Content-Length", "1000")
+	}))
+	defer srv.Close()
+
+	items := []pc.PackageDetail{
+		pkg("1.6.9", 0),
+		pkg("1.6.10", 0),
+		pkg("2.0.0", 0), // retained: must not be sized
+	}
+	// same file under a second distro version: counted once
+	dup := pkg("1.6.9", 0)
+	dup.DistroVersion = "debian/bookworm"
+	items = append(items, dup)
+	for i := range items {
+		items[i].DownloadURL = srv.URL + "/" + items[i].Filename
+	}
+
+	plan, err := BuildPlan("tyk-test", pkgConfig{VersionCutoff: "v1.7"}, testTracks, items, planNow, planGrace)
+	require.NoError(t, err)
+	require.Equal(t, 3, plan.Pruned)
+
+	c := NewClient("test-token", "tyk", 100, 100)
+	c.FillPrunedBytes(&plan, items, 4)
+	assert.Equal(t, int64(2000), plan.PrunedBytes)
+}
+
 func TestBuildPlanBadTrack(t *testing.T) {
 	_, err := BuildPlan("x", pkgConfig{Track: "nonesuch", Editions: []string{"ce"}}, testTracks, nil, planNow, planGrace)
 	require.Error(t, err)