main.go

package main

import (
	"bytes"
	"crypto"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/gocolly/colly"
)

const (
	userAgent          = "go-downloader v0.0.1 (johnweldon4@gmail.com)"
	dlURL              = "https://golang.org/dl/?mode=json&include=all"
	fileDownloadFmt    = "https://golang.org/dl/%s"
	bufferSize         = 1 << 20 // ~1MB
	maxDownloadBody    = 1 << 29 // ~530MB
	maxDownloadTimeout = 10 * time.Minute
	requestTimeout     = 30 * time.Second
	dirPerms           = 0o755
	filePerms          = 0o644
)

// nolint: gochecknoglobals
var (
	skipVersions = []string{ /* "go1.16.4", "go1.15" */ }
	skipWords    = []string{"beta", "rc"}

	// keepOS and keepArch form the allowlist of binary archives to mirror.
	// Anything not listed here (other operating systems, architectures, and
	// installer packages such as .pkg/.msi) is skipped. Source archives are
	// always kept regardless of these maps.
	keepOS   = map[string]bool{"linux": true, "darwin": true, "windows": true}
	keepArch = map[string]bool{"amd64": true, "arm64": true}

	// latestPatchOnly, when true, mirrors only the newest patch release within
	// each major.minor line (e.g. only the highest go1.25.x), instead of every
	// historical patch.
	latestPatchOnly = true

	staticBuffer = make([]byte, bufferSize)
	errOut       io.Writer
	statusOut    io.Writer
)

func main() {
	statusOut = os.Stdout
	errOut = os.Stderr

	c := colly.NewCollector(colly.UserAgent(userAgent))
	c.SetRequestTimeout(requestTimeout)

	c.OnResponse(func(r *colly.Response) {
		releases, err := Parse(bytes.NewReader(r.Body))
		if err != nil {
			fmt.Fprintf(errOut, "error parsing releases: %v\n", err)

			return
		}

		keep := latestPatches(releases)

		for _, release := range releases {
			releaseName := release.Version
			if skipRelease(releaseName) {
				fmt.Fprintf(errOut, "skipping %s\n", releaseName)

				continue
			}

			if latestPatchOnly && !keep[releaseName] {
				fmt.Fprintf(errOut, "skipping %s (superseded patch)\n", releaseName)

				continue
			}

			if !ensureDirectory(releaseName) {
				continue
			}

			for _, download := range release.Downloads {
				if skipFile(download) {
					fmt.Fprintf(errOut, "skipping %s/%s\n", releaseName, download.Filename)

					continue
				}

				if checkHash(releaseName, download) {
					fmt.Fprintf(errOut, "already downloaded %s/%s\n", releaseName, download.Filename)

					continue
				}

				fmt.Fprintf(statusOut, "downloading %s/%s [%s]\n", releaseName, download.Filename, download.SHA256Sum)
				downloadFile(c, releaseName, download)
			}
		}
	})

	if err := c.Visit(dlURL); err != nil {
		fmt.Fprintf(errOut, "error visiting %s: %v\n", dlURL, err)
		os.Exit(-1)
	}
}

func skipRelease(version string) bool {
	for _, ver := range skipVersions {
		if ver == version {
			return true
		}
	}

	for _, word := range skipWords {
		if strings.Contains(version, word) {
			return true
		}
	}

	return false
}

func skipFile(file File) bool {
	if file.Size == 0 {
		return true
	}

	if len(file.SHA256Sum) == 0 {
		return true
	}

	// Always mirror the source archive.
	if file.Kind == "source" {
		return false
	}

	// Mirror only allowlisted OS/arch binary archives; drop installers
	// (.pkg/.msi) and any other kinds.
	if file.Kind != "archive" {
		return true
	}

	return !keepOS[file.OS] || !keepArch[file.Architecture]
}

// parseVersion splits a release version such as "go1.25.6" into its numeric
// major, minor, and patch components. Versions without a patch component
// (e.g. "go1.4") default the patch to 0. ok is false for versions that do not
// parse as a plain numeric release (e.g. betas and release candidates).
func parseVersion(version string) (maj, min, patch int, ok bool) {
	parts := strings.SplitN(strings.TrimPrefix(version, "go"), ".", 3)
	if len(parts) < 2 {
		return 0, 0, 0, false
	}

	var err error
	if maj, err = strconv.Atoi(parts[0]); err != nil {
		return 0, 0, 0, false
	}

	if min, err = strconv.Atoi(parts[1]); err != nil {
		return 0, 0, 0, false
	}

	if len(parts) == 3 {
		if patch, err = strconv.Atoi(parts[2]); err != nil {
			return 0, 0, 0, false
		}
	}

	return maj, min, patch, true
}

// latestPatches returns the set of versions that are the newest patch within
// each major.minor line. Versions that do not parse as plain numeric releases
// are kept as-is (the beta/rc skip in skipRelease still excludes them).
func latestPatches(releases Releases) map[string]bool {
	bestVer := map[string]string{}
	bestPatch := map[string]int{}
	keep := map[string]bool{}

	for _, r := range releases {
		maj, min, patch, ok := parseVersion(r.Version)
		if !ok {
			keep[r.Version] = true

			continue
		}

		key := fmt.Sprintf("%d.%d", maj, min)
		if p, exists := bestPatch[key]; !exists || patch > p {
			bestPatch[key] = patch
			bestVer[key] = r.Version
		}
	}

	for _, v := range bestVer {
		keep[v] = true
	}

	return keep
}

func ensureDirectory(name string) bool {
	info, err := os.Stat(name)
	if err != nil {
		if os.IsNotExist(err) {
			if err = os.Mkdir(name, dirPerms); err != nil {
				fmt.Fprintf(errOut, "could not create %q: %v\n", name, err)

				return false
			}

			return true
		}

		fmt.Fprintf(errOut, "could not stat %q: %v\n", name, err)

		return false
	}

	if info == nil {
		fmt.Fprintf(errOut, "could not find %q: %v\n", name, err)

		return false
	}

	if !info.IsDir() {
		fmt.Fprintf(errOut, "%q is not a directory\n", name)

		return false
	}

	return true
}

func dlTarget(dir string, file File) string { return path.Join(dir, file.Filename) }
func dlSHA(target string) string            { return target + ".sha" }

func computeHash(target string) []byte {
	in, err := os.Open(target)
	if err != nil {
		fmt.Fprintf(errOut, "could not read %q: %v\n", target, err)

		return nil
	}

	defer in.Close()

	hasher := crypto.SHA256.New()
	if _, err = io.CopyBuffer(hasher, in, staticBuffer); err != nil {
		fmt.Fprintf(errOut, "could not hash %q: %v\n", target, err)

		return nil
	}

	hashBytes := hasher.Sum(nil)
	hash := hex.EncodeToString(hashBytes)

	hashFile := dlSHA(target)
	_ = writeHash(hashFile, hash)

	return hashBytes
}

func checkHash(dir string, file File) bool {
	target := dlTarget(dir, file)

	info, err := os.Stat(target)
	if err != nil || info == nil {
		return false
	}

	if info.Size() != int64(file.Size) {
		fmt.Fprintf(errOut, "size of %q is %d, should be %d\n", target, info.Size(), file.Size)

		return false
	}

	hashFile := dlSHA(target)

	hd, err := ioutil.ReadFile(hashFile)
	if err != nil {
		if os.IsNotExist(err) {
			hashBytes := computeHash(target)

			return file.SHA256Sum.Equal(hashBytes)
		}

		fmt.Fprintf(errOut, "could not read %q: %v\n", hashFile, err)

		return false
	}

	hb, err := hex.DecodeString(string(hd))
	if err != nil {
		fmt.Fprintf(errOut, "malformed hash file %q: %v\n", hashFile, err)

		return false
	}

	matches := file.SHA256Sum.Equal(hb)
	if !matches {
		fmt.Fprintf(errOut, "%q sha does not match; expected %s, got %s\n", target, file.SHA256Sum, hd)
	}

	return matches
}

func downloadFile(c *colly.Collector, releaseName string, download File) {
	d := c.Clone()
	d.MaxBodySize = maxDownloadBody
	d.SetRequestTimeout(maxDownloadTimeout)

	d.OnResponse(func(dr *colly.Response) {
		if dr.StatusCode == http.StatusOK {
			target := dlTarget(releaseName, download)
			if err := dr.Save(target); err != nil {
				fmt.Fprintf(errOut, "  [%d] could not save %q: %v\n", d.ID, target, err)

				return
			}

			fmt.Fprintf(statusOut, "  [%d] downloaded %q\n", d.ID, target)

			hashFile := dlSHA(target)

			if err := writeHash(hashFile, string(download.SHA256Sum)); err != nil {
				return
			}

			fmt.Fprintf(statusOut, "  [%d] saved hash %q\n", d.ID, hashFile)
		}
	})

	dl := fmt.Sprintf(fileDownloadFmt, download.Filename)
	fmt.Fprintf(errOut, "  [%d] getting %s\n", d.ID, dl)

	if err := d.Visit(dl); err != nil {
		fmt.Fprintf(errOut, "  [%d] could not visit %q: %v\n", d.ID, dl, err)
	}
}

func writeHash(hashFile, hash string) error {
	if err := ioutil.WriteFile(hashFile, []byte(hash), filePerms); err != nil {
		fmt.Fprintf(errOut, "could not write hash to %q: %v\n", hashFile, err)

		return err // nolint: wrapcheck
	}

	return nil
}

type Hash string

func (h Hash) Bytes() ([]byte, error) { return hex.DecodeString(string(h)) } // nolint: wrapcheck
func (h Hash) Equal(other []byte) bool {
	self, err := h.Bytes()
	if err != nil {
		return false
	}

	if len(self) != len(other) {
		return false
	}

	for ix := range self {
		if self[ix] != other[ix] {
			return false
		}
	}

	return true
}

type File struct {
	Filename     string `json:"filename"`
	OS           string `json:"os"`
	Architecture string `json:"arch"`
	Version      string `json:"version"`
	SHA256Sum    Hash   `json:"sha256"`
	Size         uint64 `json:"size"`
	Kind         string `json:"kind"`
}

type Release struct {
	Version   string `json:"version"`
	IsStable  bool   `json:"stable"`
	Downloads []File `json:"files"`
}

type Releases []Release

func Parse(data io.Reader) (Releases, error) {
	var r Releases
	if err := json.NewDecoder(data).Decode(&r); err != nil {
		return nil, err // nolint: wrapcheck
	}

	return r, nil
}