slackology.sh

#!/bin/bash

set -uo pipefail

die() {
	echo "Error: $*" >&2
	exit 1
}

scriptDir="${BASH_SOURCE[0]%/*}"
[ "$scriptDir" = "${BASH_SOURCE[0]}" ] && scriptDir="."
scriptDir=$(cd "$scriptDir" && pwd) || die "cannot determine the script's own directory."

fileListUrl="http://ftp.slackware.com/pub/slackware/slackware64-current/slackware64/FILE_LIST"
packagesFile=""
apiBase="https://repology.org/api/v1/project"
bulkBase="https://repology.org/api/v1/projects"
# The +URL has to be a real page so Repology can see who is calling, but it
# cannot name this project: repology.org answers 403 to any user agent
# containing the string "slackology" (verified -- swapping only that token in an
# otherwise identical header flips 200 to 403). Hence the owner's profile.
userAgent="SlackwareBot/1.0 (+https://github.com/fulalas; personal use script)"
jobs=""
apiConcurrency="3"
delay="1.1"
if [ -n "${XDG_CACHE_HOME:-}" ]; then
	cacheDir="$XDG_CACHE_HOME/slackology"
elif [ -n "${HOME:-}" ]; then
	cacheDir="$HOME/.cache/slackology"
else
	cacheDir=""
fi
cacheEntriesDir="$cacheDir/entries"
cacheTtl=$((24 * 3600))
cacheSchema=4
# Last line of every cache entry, so a half-written one is never mistaken for
# a complete one.
cacheEnd="end"
useCache=true
clearCache=false
buildEnabled=false
onlyPackages=()
installedOnly=false
pkgDbCandidates=(/var/lib/pkgtools/packages /var/log/packages)
pkgDbDir="${pkgDbCandidates[-1]}"
for candidate in "${pkgDbCandidates[@]}"; do
	if [ -d "$candidate" ]; then
		pkgDbDir="$candidate"
		break
	fi
done
sourceDir="$scriptDir/source"

# Separator for every internal record (worker token, result row, build token).
# Must be a character that cannot appear in a package name, version, status or
# URL, and must not be whitespace -- bash's read collapses runs of whitespace
# separators, which would silently shift the columns of a record whose version
# field is empty.
fieldSep='@'

reset=$'\033[0m'
bold=$'\033[1m'
cyan=$'\033[0;36m'
green=$'\033[0;32m'
red=$'\033[0;31m'
yellow=$'\033[0;33m'
brown=$'\033[1;33m'

tagWidth=12

# Packages are handed to the parallel workers in batches, so one bash start-up
# is amortized over several packages. The cap keeps batches small enough that
# every worker gets one -- a batch size larger than total/jobs would leave the
# run single-threaded and make -r/-d meaningless.
batchSizeMax=32

# Below this many uncached packages, per-package lookups are cheaper than
# paging through Repology's bulk listing.
bulkThreshold=200

# One row per status: name | colour | summary wording. Everything that consumes
# a status -- the colour of its line and the summary counters -- reads this
# table, so a new status cannot silently vanish from the report. Rows sharing a
# wording are summed into one summary figure.
statusRows=(
	"outdated|$red|outdated"
	"uptodate|$green|up to date"
	"newer|$yellow|ahead of tracker"
	"norelease|$yellow|without upstream release"
	"snapshot|$yellow|without upstream release"
	"nottracked|$brown|not tracked"
	"failed|$red|with a failed lookup"
)

# Every outgoing request goes through here so the flags -- and above all the
# user agent Repology requires -- are declared exactly once.
httpGet() {
	local timeout="$1"
	shift
	curl -fsSL --compressed --max-time "$timeout" -A "$userAgent" "$@"
}

# Fetch a JSON document and pull a single value out of it.
jsonField() {
	httpGet 30 "$1" 2>/dev/null | jq -r "$2" 2>/dev/null | head -n1
}

printStatus() {
	printf '  %s%-*s%s %s\n' "$1" "$tagWidth" "[$2]" "$reset" "$3"
}

# Read one line of stdin into each named variable. A variable with no line left
# to read is left empty; a value that was read is never discarded, even when the
# final line has no line break.
readLines() {
	local __name
	for __name in "$@"; do
		# shellcheck disable=SC2229 # indirect assignment is the point
		IFS= read -r "$__name" || :
	done
}

# Slackware and Repology disagree on case and on _ vs -; both sides of the
# name map have to be folded the same way, so the rule lives here only.
normalizeName() {
	local __value="${2,,}"
	printf -v "$1" '%s' "${__value//_/-}"
}

normalizeVersion() {
	local __value="${2//[_-]/.}"
	__value="${__value,,}"
	# Slackware spells patchlevels 5.3.015 where Repology spells them 5.3.p15
	# or 8.3p003; fold both spellings to the same shape.
	if [[ $__value == *p[0-9]* ]]; then
		local __head="${__value%%p[0-9]*}"
		__value="${__head%.}.${__value#"$__head"p}"
	fi
	printf -v "$1" '%s' "$__value"
}

# Order two normalized versions: sets the named variable to -1, 0 or 1 for
# a<b, a==b, a>b. Components are compared numerically when both are numeric, so
# 5.2.037 equals 5.2.37 and 3.0 equals 3.0.0, and a component carrying a letter
# suffix (a pre-release such as rc1) sorts below the bare number.
versionCompare() {
	local __out="$1"
	local -a fa fb
	IFS='.' read -r -a fa <<<"$2"
	IFS='.' read -r -a fb <<<"$3"
	local i n=${#fa[@]} result=0 x y xn yn xr yr
	[ "${#fb[@]}" -gt "$n" ] && n=${#fb[@]}
	for ((i = 0; i < n; i++)); do
		x="${fa[i]:-0}"
		y="${fb[i]:-0}"
		[ "$x" = "$y" ] && continue
		xn="${x%%[!0-9]*}"
		yn="${y%%[!0-9]*}"
		xr="${x#"$xn"}"
		yr="${y#"$yn"}"
		if [ "$((10#0$xn))" -ne "$((10#0$yn))" ]; then
			[ "$((10#0$xn))" -lt "$((10#0$yn))" ] && result=-1 || result=1
			break
		fi
		if [ -z "$xr" ] && [ -n "$yr" ]; then
			result=1
			break
		fi
		if [ -n "$xr" ] && [ -z "$yr" ]; then
			result=-1
			break
		fi
		if [ "$xr" != "$yr" ]; then
			[[ "$xr" < "$yr" ]] && result=-1 || result=1
			break
		fi
	done
	printf -v "$__out" '%d' "$result"
}

# Decimal seconds -> whole microseconds, for integer rate-limit arithmetic.
toMicros() {
	local __whole="${2%%.*}" __frac=""
	[ "$__whole" = "$2" ] || __frac="${2#*.}"
	__frac="${__frac}000000"
	printf -v "$1" '%d' "$((10#0$__whole * 1000000 + 10#${__frac:0:6}))"
}

urlBasename() {
	local __base="${2%/}"
	__base="${__base##*/}"
	printf -v "$1" '%s' "${__base%%\?*}"
}

urlHost() {
	local __host="${2#*://}"
	__host="${__host%%/*}"
	__host="${__host%%\?*}"
	printf -v "$1" '%s' "${__host,,}"
}

# Quote the characters a package name may contain that would otherwise act as
# regex operators.
reEscape() {
	local __value="$2" __out="" __i __c
	for ((__i = 0; __i < ${#__value}; __i++)); do
		__c="${__value:__i:1}"
		case "$__c" in
		[A-Za-z0-9_-]) __out+="$__c" ;;
		*) __out+="\\$__c" ;;
		esac
	done
	printf -v "$1" '%s' "$__out"
}

needValue() {
	[ "$2" -ge 2 ] || die "$1 requires a value."
}

requirePositiveInt() {
	[[ "$1" =~ ^[1-9][0-9]*$ ]] || die "$2 must be a positive integer."
}

# Optional data file: use it if given, auto-detect a sibling default if not,
# and treat an explicit empty value as "disabled".
resolveOptionalFile() {
	local -n __path="$1"
	if [[ ! -v $1 ]]; then
		__path=""
		[ -f "$2" ] && __path="$2"
	fi
	[ -z "$__path" ] || [ -f "$__path" ] || die "$3 not found: $__path"
}

printHelp() {
	local cacheTtlHours=$((cacheTtl / 3600))
	cat <<EOF
Usage: ${0##*/} [options]

Downloads a Slackware repository FILE_LIST and compares each package's
packaged version against its upstream project's latest known version via
the Repology aggregator (https://repology.org). Every package in the
listing is checked -- there's no dependency on what's installed locally.

The FILE_LIST itself is never cached; it's downloaded fresh on every run
(or read straight from -f/--file). What IS cached is each package's
Repology lookup result, for ${cacheTtlHours}h, under:
  ${cacheDir:-(set HOME or XDG_CACHE_HOME)}

When a run would need more than $bulkThreshold live lookups, the whole
repository's data is fetched from Repology's bulk endpoint in a handful of
requests instead of one request per package; anything the bulk listing
doesn't cover falls back to a per-package lookup.

Pass -b/--build to, for each outdated package, fetch that package's newest
upstream release (via its upstreamLinks.tsv URL) into the package's local
SlackBuild directory in the source tree (a source/ folder laid out like
source/<category>/<pkg>/), then run its <pkg>.SlackBuild with VERSION set to
the fetched version. Only a release matching the version Repology reported is
accepted -- if it can't be found, the fetch is reported as failed and the
existing sources are built instead. The fetch method is chosen from the URL
(git forge -> release tag, packed into a source tarball; PyPI/RubyGems ->
sdist/gem; SourceForge -> release file; plain http/ftp dir -> matching
tarball, one version subdirectory deep). Off by default.

Pass -i/--installed to check only the packages actually installed on this
machine, read from $pkgDbDir, instead of the FILE_LIST universe
(-u/-f are ignored in this mode).

The --build fetch step reads each package's upstream source location from
a TSV file (columns: package<TAB>upstream_url). Keys must match the Slackware
package name exactly. If a file named upstreamLinks.tsv sits next to this
script it is used automatically; override with -U/--upstream-file, disable
with -U ''.

Package names are translated to Repology's project names before querying.
Case and _/- differences are normalized automatically, so the map file
(lines: "slackware_name repology_name") only needs genuine renames -- e.g.
Slackware's mozilla-firefox is Repology's firefox. If a file named
repologyNames.map sits next to this script it is used automatically;
override with -R/--repology-map, disable with -R '' (normalization still
applies). Names with no map entry are queried in normalized form.

Options:
  -u, --url URL              FILE_LIST URL to parse for the package universe
                             (default: $fileListUrl)
  -f, --file PATH            Use a local repo listing instead of downloading
  -i, --installed            Check only packages installed on this machine
                             (reads $pkgDbDir, ignores -u/-f)
  -s, --source-dir DIR       Local Slackware source tree used by --build
                             (default: $sourceDir)
  -p, --package NAME         Check only this package (repeatable; case and
                             _/- differences don't matter)
  -j, --jobs N               Parallel workers (default: number of CPU cores)
  -r, --parallel-requests N  Concurrent live API requests allowed (default: $apiConcurrency)
  -d, --delay SECONDS        Minimum spacing between live API calls within
                             the same request slot (default: $delay)
  -n, --no-cache             Ignore cached Repology lookups (normally kept
                             for ${cacheTtlHours}h) and re-query every package
  -b, --build                Fetch newest release + build each outdated
                             package from its local SlackBuild (see -s)
  -U, --upstream-file PATH   TSV of package<TAB>upstream_url used by --build
                             to fetch sources (default: auto-detect
                             upstreamLinks.tsv beside the script; '' disables)
  -R, --repology-map PATH    Map of "slackware_name repology_name" used to
                             translate names before the Repology query
                             (default: auto-detect repologyNames.map; '' off)
  -c, --clear-cache          Delete all cached Repology lookups, then run
  -h, --help                 Show this help
EOF
}

while [ $# -gt 0 ]; do
	case "$1" in
	-u | --url)
		needValue "$1" $#
		fileListUrl="$2"
		shift 2
		;;
	-f | --file)
		needValue "$1" $#
		packagesFile="$2"
		shift 2
		;;
	-i | --installed)
		installedOnly=true
		shift
		;;
	-s | --source-dir)
		needValue "$1" $#
		sourceDir="$2"
		shift 2
		;;
	-p | --package)
		needValue "$1" $#
		onlyPackages+=("$2")
		shift 2
		;;
	-j | --jobs)
		needValue "$1" $#
		requirePositiveInt "$2" "-j/--jobs"
		jobs="$2"
		shift 2
		;;
	-r | --parallel-requests)
		needValue "$1" $#
		requirePositiveInt "$2" "-r/--parallel-requests"
		apiConcurrency="$2"
		shift 2
		;;
	-d | --delay)
		needValue "$1" $#
		[[ "$2" =~ ^([0-9]+(\.[0-9]*)?|\.[0-9]+)$ ]] || die "-d/--delay must be a non-negative number."
		delay="$2"
		shift 2
		;;
	-n | --no-cache)
		useCache=false
		shift
		;;
	-b | --build)
		buildEnabled=true
		shift
		;;
	-U | --upstream-file)
		needValue "$1" $#
		upstreamFile="$2"
		shift 2
		;;
	-R | --repology-map)
		needValue "$1" $#
		repologyMapFile="$2"
		shift 2
		;;
	-c | --clear-cache)
		clearCache=true
		shift
		;;
	-h | --help)
		printHelp
		exit 0
		;;
	*)
		die "Unknown option: $1"
		;;
	esac
done

requiredBins=(curl jq xargs flock mktemp sleep wc)
[ -n "$jobs" ] || requiredBins+=(nproc)
[ "$buildEnabled" = false ] || requiredBins+=(git timeout awk grep sort tail head find cp)
for bin in "${requiredBins[@]}"; do
	command -v "$bin" >/dev/null 2>&1 || die "'$bin' is required but not installed."
done

[ -n "$jobs" ] || jobs=$(nproc)
toMicros delayUs "$delay"

[ -n "$cacheDir" ] || die "cannot locate a cache directory: set HOME or XDG_CACHE_HOME."

resolveOptionalFile upstreamFile "$scriptDir/upstreamLinks.tsv" "upstream file"
resolveOptionalFile repologyMapFile "$scriptDir/repologyNames.map" "repology map"

declare -A repologyNameMap
if [ -n "$repologyMapFile" ]; then
	while read -r mapKey mapValue || [ -n "$mapKey" ]; do
		[ -n "$mapKey" ] && [ -n "$mapValue" ] || continue
		normalizeName mapKey "$mapKey"
		repologyNameMap["$mapKey"]="$mapValue"
	done <"$repologyMapFile"
fi

resultsDir=$(mktemp -d) || die "cannot create a temporary directory (check TMPDIR)."
trap 'rm -rf "$resultsDir"' EXIT
resultsFile="$resultsDir/results.tsv"
: >"$resultsFile" || die "cannot write to $resultsFile"

repoNames=()
declare -A repoVersion

addPackage() {
	local full="$1" nameVersion name version
	nameVersion="${full%-*}"
	nameVersion="${nameVersion%-*}"
	name="${nameVersion%-*}"
	if [ "$name" = "$nameVersion" ]; then
		version=""
	else
		version="${nameVersion##*-}"
	fi
	if [[ ! -v repoVersion[$name] ]]; then
		repoVersion[$name]="$version"
		repoNames+=("$name")
	fi
}

if [ "$installedOnly" = true ]; then
	if [ ! -d "$pkgDbDir" ]; then
		die "package database directory not found (tried ${pkgDbCandidates[*]})"
	fi

	for pkgFile in "$pkgDbDir"/*; do
		[ -f "$pkgFile" ] || continue
		addPackage "${pkgFile##*/}"
	done

	if [ ${#repoNames[@]} -eq 0 ]; then
		die "no installed packages found in $pkgDbDir"
	fi
else
	if [ -z "$packagesFile" ]; then
		packagesFile="$resultsDir/FILE_LIST"
		echo "Fetching repo listing from $fileListUrl..."
		httpGet 60 "$fileListUrl" -o "$packagesFile" || die "failed to download $fileListUrl"
	elif [ ! -f "$packagesFile" ]; then
		die "repo listing not found: $packagesFile"
	fi

	while IFS= read -r line || [ -n "$line" ]; do
		if [[ $line == -* && $line == *.txz ]]; then
			entry="${line##*/}"
			addPackage "${entry%.txz}"
		fi
	done <"$packagesFile"

	if [ ${#repoNames[@]} -eq 0 ]; then
		die "no *.txz entries found in $packagesFile -- is it a valid repo listing?"
	fi
fi

if [ "$clearCache" = true ]; then
	rm -rf "$cacheDir"
	echo "Cache cleared."
fi
mkdir -p "$cacheEntriesDir" || die "cannot create cache directory $cacheEntriesDir"
# The rate limiter needs these; failing to open one mid-run would silently kill
# a whole batch of packages, so prove they are writable up front.
for ((slot = 0; slot < apiConcurrency; slot++)); do
	: >>"$cacheDir/api-lock-$slot" || die "cannot write rate-limit lock $cacheDir/api-lock-$slot"
done

# -p/--package is matched through the same normalization as everything else, so
# -p cython finds Cython and -p mozilla_firefox finds mozilla-firefox.
if [ ${#onlyPackages[@]} -gt 0 ]; then
	declare -A actualName
	for pkgName in "${repoNames[@]}"; do
		normalizeName normName "$pkgName"
		# shellcheck disable=SC2154 # normalizeName assigns normName by name
		actualName["$normName"]="$pkgName"
	done
	filtered=()
	declare -A wanted
	for want in "${onlyPackages[@]}"; do
		normalizeName normName "$want"
		if [[ ! -v actualName[$normName] ]]; then
			echo "Warning: package not in listing, skipping: $want" >&2
		elif [[ ! -v wanted[$normName] ]]; then
			wanted["$normName"]=1
			filtered+=("${actualName[$normName]}")
		fi
	done
	[ ${#filtered[@]} -gt 0 ] || die "none of the requested packages are in the listing."
	repoNames=("${filtered[@]}")
fi

toProcess=()
for pkgName in "${repoNames[@]}"; do
	normalizeName normName "$pkgName"
	toProcess+=("${pkgName}${fieldSep}${repoVersion[$pkgName]}${fieldSep}${repologyNameMap[$normName]:-$normName}")
done

total=${#repoNames[@]}
runStartTs=$EPOCHSECONDS
batchSize=$(((total + jobs - 1) / jobs))
[ "$batchSize" -ge 1 ] || batchSize=1
[ "$batchSize" -le "$batchSizeMax" ] || batchSize="$batchSizeMax"

# The answer depends only on the project and the packaged version, so packages
# that share both (the nine gcc-* packages, the kernel-* packages, ...) share
# one cache entry and one request.
cacheFileOf() {
	local __key="${3}${fieldSep}${2}"
	printf -v "$1" '%s' "$cacheEntriesDir/${__key//[!A-Za-z0-9._+@-]/_}"
}

# Reads a cache entry into the caller's newest/tracked/slackStatus. Succeeds
# only for a complete, fresh entry. -n/--no-cache discards entries from earlier
# runs but keeps the ones this run's own bulk prefetch just wrote.
readCacheEntry() {
	local file="$1" schema ts sentinel
	[ -f "$file" ] || return 1
	readLines schema ts newest tracked slackStatus sentinel <"$file"
	[ "$sentinel" = "$cacheEnd" ] || return 1
	[ "$schema" = "$cacheSchema" ] || return 1
	[[ $ts =~ ^[0-9]+$ ]] || return 1
	[ $((runStartTs - ts)) -lt "$cacheTtl" ] || return 1
	[ "$useCache" = true ] || [ "$ts" -ge "$runStartTs" ] || return 1
	return 0
}

writeCacheEntry() {
	local file="$1" tmp="$1.$BASHPID"
	printf '%s\n%s\n%s\n%s\n%s\n%s\n' \
		"$cacheSchema" "$EPOCHSECONDS" "$2" "$3" "$4" "$cacheEnd" >"$tmp" &&
		mv -f "$tmp" "$file" && return 0
	rm -f "$tmp"
	return 1
}

echo -e "${bold}${cyan}Upstream package version checker${reset}"
if [ "$installedOnly" = true ]; then
	echo "Package source: installed packages in $pkgDbDir ($total unique packages)"
else
	echo "Repo listing:   $packagesFile ($total unique packages)"
fi
echo "Upstream data source: https://repology.org (API)"
if [ -n "$upstreamFile" ]; then
	echo "Upstream links: $upstreamFile"
fi
if [ -n "$repologyMapFile" ]; then
	echo "Repology name map: $repologyMapFile"
fi
echo "Workers: $jobs parallel ($apiConcurrency concurrent API requests, min ${delay}s apart per slot)"
if [ "$buildEnabled" = true ]; then
	echo "Build: enabled (--build) -- will fetch newest release into $sourceDir and build outdated packages"
else
	echo "Build: disabled (pass --build to enable)"
fi
echo

# Repology can return every project of a repository at once, 200 per page. For
# a whole-repository run that is a handful of requests instead of one per
# package, so prefetch into the cache and let the per-package path handle
# whatever the bulk listing doesn't cover.
prefetchProjects() {
	local repo="$1"
	local -A tokensOfProject=()
	local token pkgName pkgVersion repoName cacheFile uncached=0

	for token in "${toProcess[@]}"; do
		IFS="$fieldSep" read -r pkgName pkgVersion repoName <<<"$token"
		cacheFileOf cacheFile "$pkgVersion" "$repoName"
		local newest="" tracked="" slackStatus=""
		readCacheEntry "$cacheFile" && continue
		tokensOfProject["$repoName"]+="$pkgVersion"$'\n'
		uncached=$((uncached + 1))
	done

	[ "$uncached" -gt "$bulkThreshold" ] || return 0

	echo "Prefetching $repo project data from Repology (about $((uncached / 200 + 1)) pages)..."

	local cursor="" page=0 keys project newest tracked pairs pending version status
	while [ "$page" -lt 40 ]; do
		page=$((page + 1))
		keys=0
		while IFS="$fieldSep" read -r project newest tracked pairs || [ -n "$project" ]; do
			[ -n "$project" ] || continue
			keys=$((keys + 1))
			cursor="$project"
			pending="${tokensOfProject[$project]:-}"
			[ -n "$pending" ] || continue
			while IFS= read -r version || [ -n "$version" ]; do
				[ -n "$version" ] || continue
				status=""
				local haystack=",$pairs," rest
				rest="${haystack#*",$version:"}"
				[ "$rest" = "$haystack" ] || status="${rest%%,*}"
				cacheFileOf cacheFile "$version" "$project"
				writeCacheEntry "$cacheFile" "$newest" "$tracked" "$status" ||
					echo "Warning: cannot write cache entry $cacheFile" >&2
			done <<<"$pending"
		done < <(httpGet 120 "$bulkBase/${cursor:+$cursor/}?inrepo=$repo" 2>/dev/null |
			jq -r '
				to_entries[] |
				[
					.key,
					([.value[] | select(.status == "newest")][0].version // ""),
					(if (.value | length) > 0 then "yes" else "no" end),
					([.value[] | select(.repo | startswith("slackware")) |
						"\(.version):\(.status)"] | join(","))
				] | join("'"$fieldSep"'")' 2>/dev/null)
		[ "$keys" -gt 0 ] || break
		[ "$keys" -ge 200 ] || break
		sleep "$delay"
	done
}

if [ "$total" -gt "$bulkThreshold" ]; then
	bulkRepo=""
	if [[ $installedOnly = false && $fileListUrl =~ /(slackware(64|arm)?-[A-Za-z0-9.]+)/ ]]; then
		bulkRepo="${BASH_REMATCH[1]//[-.]/_}"
	fi
	prefetchProjects "${bulkRepo:-slackware64_current}"
fi

# Live API calls are spread over $apiConcurrency slots, each of which allows one
# request per --delay. A worker takes whichever slot is free rather than a
# random one, so no slot idles while another has a queue.
apiLockFd=-1
apiSlot=0

acquireApiSlot() {
	local slot
	for ((slot = 0; slot < apiConcurrency; slot++)); do
		exec {apiLockFd}>"$cacheDir/api-lock-$slot" 2>/dev/null || continue
		if flock -n "$apiLockFd"; then
			apiSlot="$slot"
			return 0
		fi
		exec {apiLockFd}>&-
		apiLockFd=-1
	done
	slot=$((RANDOM % apiConcurrency))
	exec {apiLockFd}>"$cacheDir/api-lock-$slot" 2>/dev/null || return 1
	flock -x "$apiLockFd" || return 1
	apiSlot="$slot"
	return 0
}

releaseApiSlot() {
	[ "$apiLockFd" -lt 0 ] || exec {apiLockFd}>&-
	apiLockFd=-1
}

# Wait out only the time still owed on this slot since its previous request, so
# a slot is occupied for max(delay, request) rather than delay + request.
awaitApiSlot() {
	local stampFile="$cacheDir/api-last-$apiSlot" lastUs=0 nowUs waitUs
	[ -f "$stampFile" ] && read -r lastUs <"$stampFile"
	[[ $lastUs =~ ^[0-9]+$ ]] || lastUs=0
	nowUs=$((10#${EPOCHREALTIME/[.,]/}))
	waitUs=$((delayUs - (nowUs - lastUs)))
	if [ "$waitUs" -gt 0 ] && [ "$waitUs" -le "$delayUs" ]; then
		sleep "$(printf '%d.%06d' $((waitUs / 1000000)) $((waitUs % 1000000)))"
		nowUs=$((10#${EPOCHREALTIME/[.,]/}))
	fi
	printf '%s\n' "$nowUs" >"$stampFile" 2>/dev/null || :
}

# Platform-specific artifacts that sit next to the real source archive.
binaryArtifactRe='win32|win64|windows|macos|darwin|solaris|android|\.exe'
# Unanchored on purpose: it is matched against filenames embedded in an HTML
# listing, where a trailing $ would never match. Anchor it at the use site.
archiveExtRe='\.(tar\.(gz|bz2|xz|lz|zst)|tgz|tbz2?|txz|zip)'

# Strip "<pkg>-" and the archive extension from a candidate filename.
archiveVersion() {
	local __file="$2" __pkgLen="$3" __value
	__value="${__file%.tar.*}"
	[ "$__value" = "$__file" ] && __value="${__file%.*}"
	printf -v "$1" '%s' "${__value:__pkgLen+1}"
}

# Pick, out of an HTML or FTP directory listing, the archive that belongs to
# this package AND carries the wanted version. Anchoring on both is what stops
# a shared directory handing back another project's (or a Windows) tarball.
pickArchive() {
	local __out="$1" listing="$2" pkg="$3" want="$4"
	local pkgRe cand candVer best=""
	reEscape pkgRe "$pkg"
	printf -v "$__out" ''
	while read -r cand; do
		[ -n "$cand" ] || continue
		[[ ${cand,,} =~ $binaryArtifactRe ]] && continue
		archiveVersion candVer "$cand" "${#pkg}"
		normalizeVersion candVer "$candVer"
		[ "$candVer" = "$want" ] || continue
		# a real source tarball beats a zip of the same version
		if [ -z "$best" ] || { [[ $best != *.tar.* ]] && [[ $cand == *.tar.* ]]; }; then
			best="$cand"
		fi
	done < <(printf '%s' "$listing" |
		grep -oiE "$pkgRe-[0-9][A-Za-z0-9._+-]*($archiveExtRe)" | sort -u)
	printf -v "$__out" '%s' "$best"
}

# Releases often live one level down, in a directory named after the version
# series (archive.xfce.org/.../4.20/, download.gnome.org/sources/x/2025/) or
# after the release itself (ftp.gnu.org/gnu/gcc/gcc-15.2.0/).
pickVersionDir() {
	local __out="$1" listing="$2" pkg="$3" want="$4"
	local pkgRe dir series best=""
	reEscape pkgRe "$pkg"
	printf -v "$__out" ''
	while read -r dir; do
		dir="${dir#*\"}"
		dir="${dir%%/*}"
		[ -n "$dir" ] || continue
		series="$dir"
		[[ ${series,,} == "${pkg,,}-"* ]] && series="${series:${#pkg}+1}"
		normalizeVersion series "$series"
		if [ "$want" = "$series" ] || [[ $want == "$series".* ]]; then
			best="$dir"
		fi
	done < <(printf '%s' "$listing" |
		grep -oiE "href=\"([0-9]|$pkgRe-)[A-Za-z0-9._-]*/\"")
	printf -v "$__out" '%s' "$best"
}

listingUrlOf() {
	local __url="$2"
	# Only append a slash when the last path segment looks like a directory --
	# doing it unconditionally turns every download-page URL into a 404.
	[[ ${__url##*/} == *.* ]] || __url="${__url%/}/"
	printf -v "$1" '%s' "$__url"
}

# Fetch the release tag matching the wanted version and pack it as a source
# tarball, which is what a SlackBuild expects to find. A development snapshot
# is never substituted for a release.
fetchGit() {
	local url="$1" dest="$2" pkg="$3" want="$4"
	url="${url%/}"
	url="${url%.git}"
	local pkgRe tag
	reEscape pkgRe "$pkg"
	tag=$(GIT_TERMINAL_PROMPT=0 timeout 60 git ls-remote --tags --refs "$url" 2>/dev/null |
		awk -v want="$want" -v pkgRe="$pkgRe" '
			{
				tag = $2
				sub(/^refs\/tags\//, "", tag)
				key = tolower(tag)
				sub("^" tolower(pkgRe) "[-_.]", "", key)
				sub(/^(v|ver|rel|release)[-_.]?/, "", key)
				gsub(/[_-]/, ".", key)
				if (key == want) { print tag; exit }
			}')
	[ -n "$tag" ] || return 1

	local archive="$pkg-$want.tar.gz"
	GIT_TERMINAL_PROMPT=0 timeout 600 git clone --quiet --depth 1 \
		--branch "$tag" "$url" "$dest/.checkout" >/dev/null 2>&1 || return 1
	if ! (cd "$dest/.checkout" &&
		git archive --format=tar.gz --prefix="$pkg-$want/" -o "../$archive" HEAD) 2>/dev/null; then
		rm -rf "$dest/.checkout"
		return 1
	fi
	rm -rf "$dest/.checkout"
	echo "$archive"
}

fetchTarball() {
	local fileUrl="$1" dest="$2" name="${3:-}"
	[ -n "$name" ] || urlBasename name "$fileUrl"
	[ -n "$name" ] || return 1
	httpGet 600 -o "$dest/$name" "$fileUrl" 2>/dev/null || return 1
	echo "$name"
}

fetchPypi() {
	local name sdist
	urlBasename name "$1"
	sdist=$(jsonField "https://pypi.org/pypi/$name/$4/json" \
		'.urls[]? | select(.packagetype=="sdist") | .url')
	[ -n "$sdist" ] || return 1
	fetchTarball "$sdist" "$2"
}

fetchGem() {
	local name
	urlBasename name "$1"
	fetchTarball "https://rubygems.org/downloads/$name-$4.gem" "$2"
}

fetchSourceForge() {
	local url="$1" host proj file name
	urlHost host "$url"
	if [[ $host == *.sourceforge.* ]] && [[ $host != downloads.sourceforge.* ]]; then
		proj="${host%%.*}"
	elif [[ $url == */projects/* ]]; then
		proj="${url#*/projects/}"
		proj="${proj%%/*}"
	else
		proj="${url#*://*/}"
		proj="${proj%%/*}"
	fi
	[ -n "$proj" ] || return 1
	file=$(jsonField "https://sourceforge.net/projects/$proj/best_release.json" \
		'.release.filename // empty')
	[ -n "$file" ] || return 1
	urlBasename name "$file"
	local fileVer
	archiveVersion fileVer "$name" "${#3}"
	normalizeVersion fileVer "$fileVer"
	[ "$fileVer" = "$4" ] || return 1
	fetchTarball "https://sourceforge.net/projects/$proj/files$file/download" "$2" "$name"
}

fetchFromDir() {
	local url="$1" dest="$2" pkg="$3" want="$4"
	local listingUrl listing file sub
	listingUrlOf listingUrl "$url"
	listing=$(httpGet 90 "$listingUrl" 2>/dev/null) || return 1
	[ -n "$listing" ] || return 1
	pickArchive file "$listing" "$pkg" "$want"
	if [ -z "$file" ]; then
		pickVersionDir sub "$listing" "$pkg" "$want"
		[ -n "$sub" ] || return 1
		listingUrl="${listingUrl%/}/$sub/"
		listing=$(httpGet 90 "$listingUrl" 2>/dev/null) || return 1
		pickArchive file "$listing" "$pkg" "$want"
	fi
	[ -n "$file" ] || return 1
	fetchTarball "${listingUrl%/}/$file" "$dest"
}

# Ordered url/host pattern -> handler; first match wins.
fetchInto() {
	local url="$1" dest="$2" pkg="$3" want="$4"
	local host
	urlHost host "$url"

	local handler
	case "$host" in
	*.sourceforge.net | *.sourceforge.io | sourceforge.net | sourceforge.io)
		handler=fetchSourceForge
		;;
	*github* | *gitlab* | *codeberg* | *invent.kde* | *salsa* | *code.videolan* | \
		*gitweb* | *lovelyhq* | *forge.slackware* | *adelielinux* | sr.ht | *.sr.ht | git.*)
		handler=fetchGit
		;;
	pypi.org | pypi.python.org) handler=fetchPypi ;;
	rubygems.org) handler=fetchGem ;;
	*)
		case "$url" in
		http://* | https://* | ftp://*) handler=fetchFromDir ;;
		*) return 1 ;;
		esac
		;;
	esac

	local tmp artifact
	tmp=$(mktemp -d "$resultsDir/fetch.XXXXXX") || return 1
	if artifact=$("$handler" "$url" "$tmp" "$pkg" "$want") &&
		[ -n "$artifact" ] && [ -s "$tmp/$artifact" ]; then
		# Drop this package's older source archives so the SlackBuild's version
		# glob resolves to the release we just fetched.
		local old
		for old in "$dest/$pkg"-[0-9]*; do
			[ -f "$old" ] || continue
			[ "${old##*/}" = "$artifact" ] && continue
			[[ ${old##*/} =~ ($archiveExtRe|\.gem)$ ]] && rm -f "$old"
		done
		if cp -a "$tmp/$artifact" "$dest/$artifact"; then
			rm -rf "$tmp"
			printf '%s' "$artifact"
			return 0
		fi
	fi
	rm -rf "$tmp"
	return 1
}

buildPackage() {
	local token="$1" pkg pkgVersion newest url
	IFS="$fieldSep" read -r pkg pkgVersion newest url <<<"$token"

	local -a matches=("$sourceDir"/*/"$pkg"/"$pkg.SlackBuild")
	local sb="${matches[0]-}"
	if [ ! -e "$sb" ]; then
		sb=$(find "$sourceDir" -type f -name "$pkg.SlackBuild" 2>/dev/null | head -n1)
	fi
	if [ -z "$sb" ] || [ ! -e "$sb" ]; then
		printStatus "$yellow" build "$pkg: no $pkg.SlackBuild found under $sourceDir"
		return 2
	fi
	local dir="${sb%/*}"

	local want artifact="" buildVersion=""
	normalizeVersion want "$newest"
	if [ -n "$url" ]; then
		if artifact=$(fetchInto "$url" "$dir" "$pkg" "$want"); then
			printStatus "$green" fetch "$pkg: fetched $artifact -> $dir"
			buildVersion="$newest"
		else
			printStatus "$yellow" fetch "$pkg: no $newest release found at $url (keeping existing sources)"
		fi
	else
		printStatus "$yellow" fetch "$pkg: no upstream link on file (keeping existing sources)"
	fi

	local rc
	if [ -n "$buildVersion" ]; then
		printStatus "$cyan" build "$pkg: building $pkgVersion -> $newest ($sb)"
		(cd "$dir" && VERSION="$buildVersion" bash "./$pkg.SlackBuild")
		rc=$?
	else
		printStatus "$cyan" build "$pkg: rebuilding existing sources, still $pkgVersion not $newest ($sb)"
		(cd "$dir" && bash "./$pkg.SlackBuild")
		rc=$?
	fi
	if [ "$rc" -eq 0 ]; then
		printStatus "$green" build "$pkg: build succeeded"
	else
		printStatus "$red" build "$pkg: build FAILED (exit $rc)"
		return 1
	fi
	return 0
}

statusColorOf() {
	local __rest="${statusColors#*"|$2="}"
	if [ "$__rest" = "$statusColors" ]; then
		printf -v "$1" '%s' "$yellow"
	else
		printf -v "$1" '%s' "${__rest%%|*}"
	fi
}

processOne() {
	local token="$1"
	local pkgName pkgVersion repoName
	IFS="$fieldSep" read -r pkgName pkgVersion repoName <<<"$token"
	local cacheEntryFile
	cacheFileOf cacheEntryFile "$pkgVersion" "$repoName"
	local newest="" tracked="" slackStatus="" status=""

	if ! readCacheEntry "$cacheEntryFile"; then
		local response="" httpCode="" curlRc=1 apiShape=""
		newest="" tracked="" slackStatus=""
		if acquireApiSlot; then
			awaitApiSlot
			response=$(httpGet 15 -w '\n%{http_code}' "$apiBase/$repoName" 2>/dev/null)
			curlRc=$?
			releaseApiSlot
		fi
		httpCode="${response##*$'\n'}"
		response="${response%$'\n'*}"
		if [ "$curlRc" -eq 0 ] && [ "$httpCode" = "200" ]; then
			readLines apiShape newest tracked slackStatus < <(jq -r --arg v "$pkgVersion" '
				if type != "array" then "invalid"
				else
					("array",
						([.[] | select(.status == "newest")][0].version // ""),
						(if length > 0 then "yes" else "no" end),
						([.[] | select((.repo | startswith("slackware")) and .version == $v)][0].status // ""))
				end' <<<"$response" 2>/dev/null)
		fi
		if [ "$apiShape" = "array" ]; then
			writeCacheEntry "$cacheEntryFile" "$newest" "$tracked" "$slackStatus" ||
				echo "Warning: cannot write cache entry $cacheEntryFile" >&2
		else
			status="failed"
			newest="?"
		fi
	fi

	if [ -z "$status" ]; then
		# Trust Repology's own verdict for our exact package version when it
		# has one (and, for outdated, a release to point at); fall back to
		# comparing version strings ourselves otherwise.
		case "$slackStatus" in
		newest) status="uptodate" ;;
		devel) status="newer" ;;
		ignored | rolling | untrusted | incorrect | noscheme) status="snapshot" ;;
		outdated | legacy) [ -n "$newest" ] && status="outdated" ;;
		esac
	fi
	if [ -z "$status" ]; then
		if [ -n "$newest" ]; then
			local normRepo normNewest order
			normalizeVersion normRepo "$pkgVersion"
			normalizeVersion normNewest "$newest"
			versionCompare order "$normRepo" "$normNewest"
			case "$order" in
			-1) status="outdated" ;;
			0) status="uptodate" ;;
			*) status="newer" ;;
			esac
		elif [ "$tracked" = "yes" ]; then
			status="norelease"
		else
			status="nottracked"
		fi
	fi
	[ -n "$newest" ] || newest="-"

	# One short line per package, appended atomically; the parent tallies the
	# summary and derives the build queue from this single file.
	printf '%s%s%s%s%s%s%s\n' \
		"$status" "$fieldSep" "$pkgName" "$fieldSep" "$pkgVersion" "$fieldSep" "$newest" >>"$resultsFile"

	local statusColor note=""
	statusColorOf statusColor "$status"
	if [ "$status" = "snapshot" ]; then
		note="  ($slackStatus on Repology)"
	fi
	printStatus "$statusColor" "$status" "$pkgName ($pkgVersion -> $newest)${note}"
}

statusColors="|"
for row in "${statusRows[@]}"; do
	IFS='|' read -r rowName rowColor _ <<<"$row"
	statusColors+="$rowName=$rowColor|"
done

# Everything the parallel workers touch, declared in one place.
export -f processOne acquireApiSlot releaseApiSlot awaitApiSlot printStatus readLines \
	normalizeVersion versionCompare statusColorOf cacheFileOf readCacheEntry writeCacheEntry httpGet
export apiBase userAgent delay delayUs useCache cacheTtl cacheSchema cacheEnd cacheDir cacheEntriesDir
export apiConcurrency apiLockFd apiSlot runStartTs resultsFile fieldSep tagWidth reset yellow statusColors

printf '%s\n' "${toProcess[@]}" |
	xargs -d '\n' -P "$jobs" -n "$batchSize" \
		bash -c 'set -uo pipefail; for token; do processOne "$token"; done' _
workersRc=$?

declare -A upstreamUrlOf
if [ "$buildEnabled" = true ] && [ -n "$upstreamFile" ]; then
	while IFS=$'\t' read -r pkg url || [ -n "$pkg" ]; do
		[ "$pkg" = "package" ] && continue
		[ -n "$pkg" ] && [ -n "$url" ] && upstreamUrlOf["$pkg"]="$url"
	done <"$upstreamFile"
fi

declare -A statusCount
buildTokens=()
recorded=0
while IFS="$fieldSep" read -r status pkgName pkgVersion newest || [ -n "$status" ]; do
	[ -n "$status" ] || continue
	recorded=$((recorded + 1))
	statusCount[$status]=$((${statusCount[$status]:-0} + 1))
	if [ "$buildEnabled" = true ] && [ "$status" = "outdated" ]; then
		buildTokens+=("$pkgName$fieldSep$pkgVersion$fieldSep$newest$fieldSep${upstreamUrlOf[$pkgName]:-}")
	fi
done <"$resultsFile"

buildFailures=0
notBuilt=0
if [ ${#buildTokens[@]} -gt 0 ]; then
	echo
	echo -e "${bold}${cyan}Fetching + building ${#buildTokens[@]} outdated package(s) from $sourceDir...${reset}"
	for token in "${buildTokens[@]}"; do
		buildPackage "$token"
		case $? in
		0) ;;
		2) notBuilt=$((notBuilt + 1)) ;;
		*) buildFailures=$((buildFailures + 1)) ;;
		esac
	done
fi

# Every status in the table contributes to exactly one summary figure, so the
# figures always account for every package that produced a result.
summaryLabels=()
declare -A labelCount labelColor
accounted=0
for row in "${statusRows[@]}"; do
	IFS='|' read -r rowName rowColor rowLabel <<<"$row"
	if [[ ! -v labelCount[$rowLabel] ]]; then
		summaryLabels+=("$rowLabel")
		labelCount[$rowLabel]=0
		labelColor[$rowLabel]="$rowColor"
	fi
	labelCount[$rowLabel]=$((labelCount[$rowLabel] + ${statusCount[$rowName]:-0}))
	accounted=$((accounted + ${statusCount[$rowName]:-0}))
done

summary=""
for label in "${summaryLabels[@]}"; do
	summary+=", ${labelColor[$label]}${labelCount[$label]} ${label}${reset}"
done

echo
echo -e "${bold}Summary:${reset} $recorded of $total packages checked${summary}."

incomplete=false
if [ "$recorded" -ne "$total" ]; then
	echo -e "${red}$((total - recorded)) package(s) produced no result at all -- a worker died. Rerun.${reset}" >&2
	incomplete=true
fi
if [ "$accounted" -ne "$recorded" ]; then
	echo -e "${red}$((recorded - accounted)) result(s) had a status missing from the summary table.${reset}" >&2
	incomplete=true
fi
if [ "$workersRc" -ne 0 ]; then
	echo -e "${red}The package workers exited with status $workersRc.${reset}" >&2
	incomplete=true
fi
if [ "${statusCount[failed]:-0}" -gt 0 ]; then
	echo -e "${red}Failed lookups are not cached; rerun to retry, or reduce -r / raise -d if rate-limited.${reset}"
fi
if [ "$notBuilt" -gt 0 ]; then
	echo -e "${yellow}${notBuilt} package(s) had no local SlackBuild and were not built.${reset}" >&2
fi
if [ "$buildFailures" -gt 0 ]; then
	echo -e "${red}${buildFailures} package(s) failed to build.${reset}" >&2
fi

if [ "$incomplete" = true ] || [ "$buildFailures" -gt 0 ] || [ "$notBuilt" -gt 0 ]; then
	exit 1
fi