feat: migrate content ingestion to Hugo Modules and Go 1.26

Author: TineoC

.github/workflows/update-content.yml

@@ -0,0 +1,57 @@
+name: Update Content Modules
+
+on:
+  schedule:
+    - cron: '0 0 * * 1' # Every Monday at midnight
+  workflow_dispatch: # Allow manual trigger
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-modules:
+    runs-on: ubuntu-latest
+    if: github.repository == 'kubernetes/contributor-site'
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Setup Go
+        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+        with:
+          go-version: '1.26.2'
+
+      - name: Setup Hugo
+        uses: peaceiris/actions-hugo@75d2e84710de30f6ff7268e08f310b60ef14033f # v3.0.0
+        with:
+          hugo-version: '0.133.0'
+          extended: true
+
+      - name: Update Hugo Modules
+        run: |
+          hugo mod get -u
+          hugo mod tidy
+
+      - name: Run Content & Image Tests (Container)
+        run: make container-test
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@70a41aba780001da0a30141984ae2a0c95d8704e # v6.0.2
+        with:
+          commit-message: "chore: update external content modules"
+          title: "Update external content modules"
+          body: |
+            This PR updates the Hugo modules to pull the latest content from upstream repositories.
+            
+            This is an automated update. Prow will merge it after CI passes.
+            
+            /sig contributor-experience
+            /kind feature
+            /lgtm
+            /approve
+            
+            Triggered by: ${{ github.event_name }}
+          branch: "automation/update-modules"
+          base: "master"
+          labels: "kind/feature, sig/contributor-experience, tide/merge-method-squash"

.github/workflows/verify.yml

@@ -0,0 +1,22 @@
+name: Verify PR
+
+on:
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Run Content & Image Tests (Container)
+        run: |
+          # The Makefile will automatically build the image using the local Dockerfile
+          # which contains all necessary dependencies (Go 1.26, Hugo, Python, etc.)
+          make container-test

.htmltest.yml

@@ -0,0 +1,13 @@
+DirectoryPath: public
+CheckInternalLinks: true
+CheckInternalHashes: true
+CheckExternalLinks: false # Disabled to keep CI fast and stable
+CheckImages: true
+CheckAlt: true
+IgnoreURLs:
+  - "https://slack.k8s.io" # Often rate-limited
+IgnoreDirs:
+  - google_analytics
+  - reveal
+LogFile: htmltest.log
+LogLevel: 2

Dockerfile

@@ -17,16 +17,27 @@ RUN apk add --no-cache \
     sed \
     npm
 
+# Install Go
+ARG GO_VERSION=1.26.2
+RUN curl -sSfL "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" -o /tmp/go.tgz \
+    && tar -xz -C /usr/local -f /tmp/go.tgz \
+    && rm /tmp/go.tgz
+ENV PATH="/usr/local/go/bin:${PATH}"
+
 WORKDIR /src
 
 COPY package*.json ./
 
 RUN npm ci --ignore-scripts
 
+RUN apk add --no-cache python3 py3-pip
+RUN pip3 install --break-system-packages pyyaml
+
 RUN mkdir -p /usr/local/src && \
     cd /usr/local/src && \
     curl -L https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-${TARGETARCH}.tar.gz | tar -xz && \
     mv hugo /usr/local/bin/hugo && \
+    curl -sL https://htmltest.wjdp.uk | bash -s -- -b /usr/local/bin && \
     addgroup -Sg 1000 hugo && \
     adduser -Sg hugo -u 1000 -h /src hugo
 

Makefile

@@ -17,30 +17,34 @@ STAGING_IMAGE_REGISTRY	:= us-central1-docker.pkg.dev/k8s-staging-images
 IMAGE_REGISTRY			?= ${STAGING_IMAGE_REGISTRY}/contributor-site
 IMAGE_NAME				:= k8s-contrib-site-hugo
 IMAGE_REPO				:= $(IMAGE_REGISTRY)/$(IMAGE_NAME)
-IMAGE_VERSION			:= $(shell scripts/hash-files.sh Dockerfile Makefile netlify.toml .dockerignore cloudbuild.yaml package.json package-lock.json | cut -c 1-12)
+IMAGE_VERSION			:= $(shell scripts/hash-files.sh Dockerfile Makefile netlify.toml .dockerignore cloudbuild.yaml package.json package-lock.json go.mod go.sum | cut -c 1-12)
 COMMIT					:= $(shell git rev-parse --short HEAD)
-CONTAINER_RUN			:= $(CONTAINER_ENGINE) run --rm -it -v "$(CURDIR):/src"
-CONTAINER_RUN_TTY		:= $(CONTAINER_ENGINE) run --rm -it
+CONTAINER_RUN			:= $(CONTAINER_ENGINE) run --rm -v "$(CURDIR):/src"
+CONTAINER_RUN_TTY		:= $(CONTAINER_ENGINE) run --rm
 HUGO_VERSION			:= $(shell grep ^HUGO_VERSION netlify.toml | tail -n 1 | cut -d '=' -f 2 | tr -d " \"\n")
+GO_VERSION				:= $(shell grep ^GO_VERSION netlify.toml | tail -n 1 | cut -d '=' -f 2 | tr -d " \"\n")
 GIT_TAG					?= v$(HUGO_VERSION)-$(IMAGE_VERSION)
 CONTAINER_IMAGE			:= $(IMAGE_REPO):$(GIT_TAG)
+GOMODCACHE				?= $(shell go env GOMODCACHE)
 
 # Docker buildx related settings for multi-arch images
 DOCKER_BUILDX ?= docker buildx
 
 CONTAINER_HUGO_MOUNTS = \
 	--read-only \
 	--mount type=bind,source=$(CURDIR)/.git,target=/src/.git,readonly \
+	--mount type=bind,source=$(CURDIR)/go.mod,target=/src/go.mod \
+	--mount type=bind,source=$(CURDIR)/go.sum,target=/src/go.sum \
+	--mount type=bind,source=$(GOMODCACHE),target=/tmp/gomod \
 	--mount type=bind,source=$(CURDIR)/assets,target=/src/assets,readonly \
 	--mount type=bind,source=$(CURDIR)/content,target=/src/content,readonly \
-	--mount type=bind,source=$(CURDIR)/external-sources,target=/src/external-sources,readonly \
-	--mount type=bind,source=$(CURDIR)/hack,target=/src/hack,readonly \
 	--mount type=bind,source=$(CURDIR)/layouts,target=/src/layouts,readonly \
 	--mount type=bind,source=$(CURDIR)/static,target=/src/static,readonly \
+	--mount type=bind,source=$(CURDIR)/scripts,target=/src/scripts,readonly \
 	--mount type=tmpfs,destination=/tmp,tmpfs-mode=01777 \
 	--mount type=bind,source=$(CURDIR)/hugo.yaml,target=/src/hugo.yaml,readonly
 
-# Fast NONBLOCKING IO to stdout caused by the hack/gen-content.sh script can
+# Fast NONBLOCKING IO to stdout caused by Hugo module operations can
 # cause Netlify builds to terminate unexpectedly. This forces stdout to block.
 BLOCK_STDOUT_CMD	:= python -c "import os,sys,fcntl; \
 					flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); \
@@ -49,21 +53,36 @@ BLOCK_STDOUT_CMD	:= python -c "import os,sys,fcntl; \
 .DEFAULT_GOAL	:= help
 
 .PHONY: targets container-targets
-targets: help gen-content render server clean clean-all production-build preview-build
-container-targets: container-image container-push container-gen-content container-render container-server
+targets: help modules-get render server test clean clean-all production-build preview-build
+container-targets: container-image container-push container-modules-get container-render container-server container-test
 
 help: ## Show this help text.
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
 dependencies:
 	npm ci
+	pip3 install pyyaml || echo "Warning: pip3 install failed, verify-mounts.py may fail."
 
-gen-content: ## Generates content from external sources.
-	hack/gen-content.sh
+modules-get: ## Pulls latest Hugo module content.
+	hugo mod get -u
+	hugo mod tidy
+
+modules-tidy: ## Clean and tidy Hugo modules.
+	hugo mod tidy
 
 render: dependencies ## Build the site using Hugo on the host.
 	hugo --logLevel info --ignoreCache --minify
 
+test: render ## Run content verification and link/image checks.
+	python3 scripts/verify-mounts.py --config hugo.yaml --public public
+	if ! command -v htmltest >/dev/null; then curl -sL https://htmltest.wjdp.uk | bash; ./bin/htmltest; else htmltest; fi
+
+container-test: container-image ## Run tests within a container using pre-installed dependencies.
+	$(CONTAINER_RUN_TTY) $(CONTAINER_HUGO_MOUNTS) -e GOMODCACHE=/tmp/gomod $(CONTAINER_IMAGE) \
+		bash -c "hugo --ignoreCache --minify --destination /tmp/public && \
+		         python3 scripts/verify-mounts.py --config hugo.yaml --public /tmp/public && \
+		         htmltest --conf .htmltest.yml /tmp/public"
+
 server: dependencies ## Run Hugo locally (if Hugo "extended" is installed locally)
 	hugo server \
 		--logLevel info \
@@ -77,7 +96,7 @@ docker-image:
 	$(MAKE) container-image
 
 container-image: ## Build container image for use with container-* targets.
-	$(CONTAINER_ENGINE) build . -t $(CONTAINER_IMAGE) --label git_commit=$(COMMIT) --build-arg HUGO_VERSION=$(HUGO_VERSION)
+	$(CONTAINER_ENGINE) build . -t $(CONTAINER_IMAGE) --label git_commit=$(COMMIT) --build-arg HUGO_VERSION=$(HUGO_VERSION) --build-arg GO_VERSION=$(GO_VERSION)
 
 container-push: container-image ## Push container image for the preview of the website
 	$(CONTAINER_ENGINE) push $(CONTAINER_IMAGE)
@@ -94,24 +113,25 @@ docker-push: ## Build a multi-architecture image and push that into the registry
 		--push \
 		--platform=$(PLATFORMS) \
 		--build-arg HUGO_VERSION=$(HUGO_VERSION) \
+		--build-arg GO_VERSION=$(GO_VERSION) \
 		--tag $(CONTAINER_IMAGE) \
 		-f Dockerfile.cross .
 	$(DOCKER_BUILDX) stop image-builder
 	rm Dockerfile.cross
 
 docker-gen-content:
 	@echo -e "**** The use of docker-gen-content is deprecated. Use container-gen-content instead. ****" 1>&2
-	$(MAKE) container-gen-content
+	$(MAKE) container-modules-get
 
-container-gen-content: ## Generates content from external sources within a container (equiv to gen-content).
-	$(CONTAINER_RUN) $(CONTAINER_IMAGE) hack/gen-content.sh
+container-modules-get: ## Pulls latest Hugo module content within a container.
+	$(CONTAINER_RUN) -e GOMODCACHE=/tmp/gomod $(CONTAINER_IMAGE) hugo mod get -u
 
 docker-render:
 	@echo -e "**** The use of docker-render is deprecated. Use container-render instead. ****" 1>&2
 	$(MAKE) container-render
 
 container-render: ## Build the site using Hugo within a container (equiv to render).
-	$(CONTAINER_RUN_TTY) $(CONTAINER_HUGO_MOUNTS) $(CONTAINER_IMAGE) hugo --logLevel info --ignoreCache --minify
+	$(CONTAINER_RUN_TTY) $(CONTAINER_HUGO_MOUNTS) -e GOMODCACHE=/tmp/gomod $(CONTAINER_IMAGE) hugo --logLevel info --ignoreCache --minify
 
 docker-server:
 	@echo -e "**** The use of docker-server is deprecated. Use container-server instead. ****" 1>&2
@@ -123,9 +143,9 @@ container-server: ## Run Hugo locally within a container, available at http://lo
 		$(CONTAINER_HUGO_MOUNTS) \
 		--cap-drop=ALL \
 		--cap-drop=AUDIT_WRITE \
+		-e GOMODCACHE=/tmp/gomod \
 		$(CONTAINER_IMAGE) \
-	bash -c 'cd /src && hack/gen-content.sh --in-container && \
-		 cd /tmp/src && \
+	bash -c 'cd /src && hugo mod get -u && \
 		hugo server \
 		--environment preview \
 		--logLevel info \
@@ -141,40 +161,12 @@ container-server: ## Run Hugo locally within a container, available at http://lo
 clean: ## Cleans build artifacts.
 	rm -rf public/ resources/ _tmp/
 
-clean-all: ## Cleans both build artifacts and files synced to content directory
+clean-all: ## Cleans both build artifacts and Hugo cache.
 	rm -rf public/ resources/ _tmp/
-	rm -f content/en/events/community-meeting.md
-	rm -f content/en/events/meet-our-contributors.md
-	rm -f content/en/events/office-hours.md
-	rm -f content/en/docs/cheatsheet.md
-	rm -f content/en/resources/rename.md
-	find content/en/docs/guide -maxdepth 1 \
-		-not -path content/en/docs/guide \
-		-not -name ".gitignore" \
-		-exec rm -rf {} \;
-	find content/en/docs/comms -maxdepth 1 \
-		-not -path content/en/docs/comms \
-		-not -name ".gitignore" \
-		-not -name "_index.md" \
-		-exec rm -rf {} \;
-	find content/en/resources/release -maxdepth 1  \
-		-not -path content/en/resources/release \
-		-not -name ".gitignore" \
-		-exec rm -rf {} \;
-	find content/en/docs/orientation -maxdepth 1 \
-		-not -path content/en/docs/orientation \
-		-not -name ".gitignore" \
-		-exec rm -rf {} \;
-	find content/en/community -maxdepth 1 \
-		-not -path content/en/community \
-		-not -name ".gitignore" \
-		-not -name "_index.md" \
-		-not -name "code-of-conduct.md" \
-		-exec rm -rf {} \;
+	hugo mod clean
 
 production-build: ## Builds the production site (this command used only by Netlify).
 	$(BLOCK_STDOUT_CMD)
-	hack/gen-content.sh
 	hugo \
 		--environment production \
 		--logLevel info \
@@ -183,7 +175,6 @@ production-build: ## Builds the production site (this command used only by Netli
 
 preview-build: ## Builds a deploy preview of the site (this command used only by Netlify).
 	$(BLOCK_STDOUT_CMD)
-	hack/gen-content.sh
 	hugo \
 		--environment preview \
 		--logLevel info \

README.md

@@ -43,7 +43,7 @@ Site is to use [Docker][docker]. To begin, create the docker image to be used
 with generating the site by executing `make container-image`.
 
 To ensure you can view the site with externally sourced content, run
-`make container-gen-content` before previewing the site by with
+`make container-modules-get` before previewing the site with
 `make container-server`.
 
 **NOTE to Apple Silicon Mac Users**
@@ -73,22 +73,7 @@ git submodule update --init --recursive --depth 1
 ```
 
 To ensure you can view the site with externally sourced content, run
-`make gen-content` before previewing the site by with `make server`.
-
-**NOTE to MacOS Users**
-
-The `hack/gen-content.sh` script requires the gnu version
-of base packages such as `find`, `grep`, and `sed`. 
-
-```
-brew install coreutils findutils grep gnu-sed gnu-tar make readlink
-```
-
-You will then need to update your path to include these:
-
-```
-export PATH="/usr/local/opt/coreutils/libexec/gnubin:$PATH"
-```
+`make modules-get` before previewing the site with `make server`.
 
 ## Community, discussion, contribution, and support
 

content/en/includes/_index.md

@@ -1,6 +1,3 @@
 ---
 headless: true
-_build:
-  list: never
-  render: false
 ---