Merge pull request #18 from srdp-hub/dev

Merging `dev` into `main`, using Github Flow from now on.

Author: dkapitan

.gitignore

@@ -1,5 +1,9 @@
+# OSX
+.DS_Store
+
 # Environment variables
 .env
+secrets.sh
 
 # Local certificates and PAT for Zitadel
 certs/
@@ -12,4 +16,9 @@ site/
 .terraform
 terraform.tfstate
 terraform.tfstate.backup
-.DS_Store
+kubeconfig.yaml
+
+# Kubernetes
+values-prod.yaml
+charts/
+

Justfile

@@ -0,0 +1,70 @@
+set shell := ["bash", "-c"]
+set dotenv-load := false
+
+namespace := "srdp"
+kubeconfig := justfile_directory() + "/kubernetes/opentofu/kubeconfig.yaml"
+
+default: help
+
+help:
+	@just --list
+
+# Local development
+local-tls:
+	mkdir -p kubernetes/certs
+	mkcert -cert-file kubernetes/certs/selfsigned.crt -key-file kubernetes/certs/selfsigned.key "auth.local.dev" "marimo.local.dev" "quarto.local.dev" "dagster.local.dev"
+	kubectl create namespace {{namespace}} --dry-run=client -o yaml | kubectl apply -f -
+	kubectl create secret tls custom-ingress-cert --namespace {{namespace}} --key kubernetes/certs/selfsigned.key --cert kubernetes/certs/selfsigned.crt --dry-run=client -o yaml | kubectl apply -f -
+
+local-deploy:
+	cd kubernetes/srdp-chart && helm dependency update
+	cd kubernetes/srdp-chart && helm upgrade --install srdp . --namespace {{namespace}} --create-namespace -f values.yaml -f values-local.yaml
+
+local-delete:
+	helm uninstall srdp -n {{namespace}} || true
+	kubectl delete pvc --all -n {{namespace}} || true
+
+# Prod / infra
+prod-apply:
+	cd kubernetes/opentofu && source ./secrets.sh && tofu apply -auto-approve
+
+prod-destroy:
+	just prod-uninstall || echo "Helm uninstall skipped (cluster may already be down)"
+	cd kubernetes/opentofu && source ./secrets.sh && tofu destroy -auto-approve
+
+prod-use-kubeconfig:
+	cd kubernetes/opentofu && tofu output -raw kubeconfig > "{{kubeconfig}}" && echo "kubeconfig written to {{kubeconfig}}"
+
+prod-get-values:
+	@echo "Fetching dynamic values..."
+	@if [ ! -f "{{kubeconfig}}" ]; then echo "kubeconfig not found, run 'just prod-use-kubeconfig' first"; exit 1; fi
+	@KUBECONFIG="{{kubeconfig}}" kubectl get svc srdp-traefik -n {{namespace}} -o jsonpath='{.status.loadBalancer.ingress[0].ip}' | xargs -I{} printf "LOAD_BALANCER_IP:\t%s\n" "{}"
+
+prod-traefik-only:
+	cd kubernetes && \
+		if [ ! -f "{{kubeconfig}}" ]; then echo "kubeconfig not found, run 'just prod-use-kubeconfig' first"; exit 1; fi; \
+		export KUBECONFIG="{{kubeconfig}}"; \
+		helm upgrade --install srdp srdp-chart --namespace {{namespace}} --create-namespace -f srdp-chart/values-prod.yaml --set zitadel.enabled=false --set oauth2-proxy.enabled=false --set dagster.enabled=false --set marimo.enabled=false --set quarto.enabled=false
+
+prod-auth-only:
+	cd kubernetes && \
+		if [ ! -f "{{kubeconfig}}" ]; then echo "kubeconfig not found, run 'just prod-use-kubeconfig' first"; exit 1; fi; \
+		export KUBECONFIG="{{kubeconfig}}"; \
+		helm upgrade srdp srdp-chart --namespace {{namespace}} --reset-values -f srdp-chart/values-prod.yaml --set zitadel.enabled=true --set oauth2-proxy.enabled=true --set dagster.enabled=false --set marimo.enabled=false --set quarto.enabled=false
+
+prod-full:
+	cd kubernetes && \
+		if [ ! -f "{{kubeconfig}}" ]; then echo "kubeconfig not found, run 'just prod-use-kubeconfig' first"; exit 1; fi; \
+		export KUBECONFIG="{{kubeconfig}}"; \
+		helm upgrade srdp srdp-chart --namespace {{namespace}} --reset-values -f srdp-chart/values-prod.yaml
+
+prod-uninstall:
+	cd kubernetes && \
+		if [ ! -f "{{kubeconfig}}" ]; then echo "kubeconfig not found, run 'just prod-use-kubeconfig' first"; exit 1; fi; \
+		export KUBECONFIG="{{kubeconfig}}"; \
+		echo "Deleting LoadBalancer service (releases Scaleway LB)..." && \
+		kubectl delete svc srdp-traefik -n {{namespace}} --ignore-not-found && \
+		echo "Waiting 30s for LB cleanup..." && sleep 30 && \
+		helm uninstall srdp -n {{namespace}} || true && \
+		kubectl delete jobs --all -n {{namespace}} --ignore-not-found && \
+		kubectl delete pvc --all -n {{namespace}} --ignore-not-found

docs/01-prerequisites.md

@@ -7,13 +7,15 @@ icon: lucide/list-todo
 
 Before you begin, ensure you have the following software installed on your local machine. This guide assumes you have a basic understanding of using the command line.
 
-### Core Tools
+Required tools
+- Git
+- Container runtime (Docker/Podman) for building the Marimo, Quarto, and srdp-etl (Dagster user code) images
+- Kubernetes cluster + `kubectl` (tested with 1.32+). For local work, `kind`, `minikube`, or `k3d` with LoadBalancer/NodePort access all work.
+- Helm 3.x
+- [`just`](https://github.com/casey/just) as a task runner for the common Helm/OpenTofu commands in this repo
+- [`mkcert`](https://github.com/FiloSottile/mkcert) to generate local TLS certificates for `*.local.dev`
+- [OpenTofu](https://opentofu.org/docs/intro/install/) for production infrastructure on Scaleway Kapsule
 
-*   **Git:** The project is managed with Git. You will need it to clone the repository.
-    *   [Install Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
-
-*   **Docker and Docker Compose:** All services in this project are containerized using Docker. Docker Compose is used to orchestrate and run the multi-container setup.
-    *   The easiest way to get both is by installing **Docker Desktop** for your operating system. It includes Docker Engine, Docker CLI, and Docker Compose.
-    *   [Install Docker Desktop](https://www.docker.com/products/docker-desktop/)
-
-*   **A Code Editor:** You will need an editor to view and modify files.
+Notes
+- The `just` recipes use a Bash-compatible shell; on Windows, run them from Git Bash or WSL.
+- You need permission to create namespaces/secrets and, for cloud runs, to provision load balancers.

docs/02-configuration.md

@@ -7,109 +7,79 @@ icon: lucide/locate-fixed
 
 This guide will walk you through the steps to get the Single Repo Data Platform (SRDP) running on your local machine.
 
-### Step 1: Clone the Repository
-
-First, clone this repository to a location of your choice on your computer.
-
-Open your terminal, navigate to your development directory, and run:
+## 1) Clone the repo
 
 ```bash
 git clone git@github.com:dkapitan/srdp.git # or git clone https://github.com/dkapitan/srdp.git
-cd srdp/local
+cd srdp
 ```
 
-### Step 2: Configure Local Hostnames
-
-To access the services using friendly names like `marimo.local.dev`, you need to edit your computer's `hosts` file. This file maps domain names to IP addresses. We will map our service domains to your local machine's IP, `127.0.0.1`.
-
-**Why is this necessary?**
-When you type `marimo.local.dev` into your browser, this file tells the browser to send the request to your own computer instead of trying to find a public website on the internet. Traefik, our proxy, will then receive this request and route it to the correct Docker container.
-
----
-
-#### On macOS or Linux:
+## 2) Point DNS at your cluster
 
-1.  Open a terminal.
-2.  Run the following command to open the hosts file with `nano`, a simple text editor. You will be prompted for your password.
-    ```bash
-    sudo nano /etc/hosts
-    ```
-
-#### On Windows:
-
-1.  Press the Windows key, type `Notepad`, right-click on it, and select **"Run as administrator"**.
-2.  In Notepad, go to `File` > `Open`.
-3.  Navigate to `c:\Windows\System32\Drivers\etc\hosts`.
-4.  You may need to change the file type filter in the bottom-right corner from "Text Documents (*.txt)" to **"All files (*.*)"** to see the `hosts` file.
-
----
-
-#### Add the Following Line:
-
-Add this single line to the **bottom** of the `hosts` file.
+The chart uses `*.local.dev` by default. Point those hostnames at the IP you will use to reach Traefik:
+- For NodePort/local clusters: `127.0.0.1` is usually fine.
+- For a LoadBalancer: use the external IP once Traefik comes up.
 
+Add one line to your hosts file:
 ```
-127.0.0.1   marimo.local.dev quarto.local.dev auth.local.dev
+127.0.0.1 auth.local.dev marimo.local.dev quarto.local.dev dagster.local.dev
 ```
 
-Save the file and exit the editor. (In `nano`, press `Ctrl+X`, then `Y`, then `Enter`). **You may need to restart your machine in order for changes to take effect.**
-
-### Step 2b: Generate Certificates
-
-You should generate local development certificates using [mkcert](https://github.com/FiloSottile/mkcert):
+## 3) Create a local TLS secret
 
+Generate a certificate for the local domains and create the secret that the Helm chart expects.
 ```bash
-mkcert -cert-file ./certs/selfsigned.crt -key-file ./certs/selfsigned.key "*.local.dev" "local.dev"
+mkdir -p kubernetes/certs
+mkcert -cert-file kubernetes/certs/selfsigned.crt -key-file kubernetes/certs/selfsigned.key \
+  "auth.local.dev" "marimo.local.dev" "quarto.local.dev" "dagster.local.dev"
+
+# Apply the secret into your target namespace (default here is srdp)
+kubectl create namespace srdp --dry-run=client -o yaml | kubectl apply -f -
+kubectl create secret tls custom-ingress-cert \
+  --namespace srdp \
+  --key kubernetes/certs/selfsigned.key \
+  --cert kubernetes/certs/selfsigned.crt \
+  --dry-run=client -o yaml | kubectl apply -f -
 ```
 
-**Make sure to put these certificates under /local/certs. The easies way to accomplish this is to run the previous comppand inside the /local/ directory.**
-
-### Step 3: Launch the Services
-
-Now you are ready to start all the services. Navigate to the root of the project directory in your terminal and run:
-
+Or simply run:
 ```bash
-docker-compose up --build -d
+just local-tls
 ```
 
-*   `--build`: This flag tells Docker Compose to build the application images (for Marimo and Quarto) from their `Dockerfile`s before starting the services. You should use this the first time you run the command or after you've made changes to the application code.
-*   `-d`: This runs the containers in "detached" mode, meaning they will run in the background and your terminal will be free to use.
-
-**Note:** The first time you run this, Zitadel will be initialized but the database will be empty. You can log in as an admin using:
-
-- **Username:** `zitadel-admin@zitadel.auth.local.dev`
-- **Password:** `Password1!`
-
-After logging in, create a new project (e.g., "Protected Apps"). Then, create a new app with type **Web**, authentication method **Code**, and add the following redirect URIs:
+## 4) Build local container images
 
-- `https://marimo.local.dev/oauth2/callback`
-- `https://quarto.local.dev/oauth2/callback`
+The Helm chart references three application images. For local development the pull policy is set to `Never`, so the images must exist in your local Docker/containerd cache:
 
-And post logout URIs:
-
-- `https://marimo.local.dev`
-- `https://quarto.local.dev`
+```bash
+docker build -t rg.nl-ams.scw.cloud/srdp-registry/marimo:v1.0 local/apps/marimo
+docker build -t rg.nl-ams.scw.cloud/srdp-registry/quarto:v1.0 local/apps/quarto
+docker build -t rg.nl-ams.scw.cloud/srdp-registry/srdp-etl:v1.0 kubernetes/apps/srdp-etl
+```
 
-When the project is created, Zitadel will display a client ID and secret. Copy these and set them in your `.env` file under `/local/`:
+## 5) Fill in secrets and local values
 
-```
-OIDC_CLIENT_ID=your-client-id
-OIDC_CLIENT_SECRET=your-client-secret
-```
+Update `kubernetes/srdp-chart/values-local.yaml` before installing:
+- set your own Zitadel master key, DB passwords, OAuth2 client values, and cookie secret
+- keep `custom-ingress-cert` (created above) or point to another TLS secret if you prefer.
 
-After updating the `.env` file, run:
+## 6) Install the chart locally
 
 ```bash
-docker compose up --force-recreate
+cd kubernetes/srdp-chart
+helm dependency update
+helm upgrade --install srdp . \
+  --namespace srdp --create-namespace \
+  -f values.yaml \
+  -f values-local.yaml
 ```
 
-OR
-
+Or simply run:
 ```bash
-docker compose down
-docker compose up --build
+just local-deploy
 ```
 
-**Important:** Do **not** run `docker compose down -v`, as the `-v` option will destroy existing data, including the project and app you created in Zitadel.
+To re-run with updated values, run the same `helm upgrade` command (or `just local-deploy`).
 
+The chart deploys the full stack: Traefik, PostgreSQL (in-cluster via Bitnami Helm chart), Zitadel, OAuth2-Proxy, Dagster (webserver + daemon + user code), Marimo, and Quarto. PostgreSQL hosts both the `zitadel` and `dagster` databases, created automatically via `zitadel-db.primary.initdb.scripts`.
 **Congratulations! The local environment should now be up and running.** Proceed to the next section, **Usage & Verification**, to confirm that everything is working correctly.

docs/03-usage.md

@@ -5,6 +5,30 @@ icon: lucide/circle-play
 
 # Usage & Verification
 
+### Check the release
+- `helm list -n srdp`
+- `kubectl get pods,svc,ing -n srdp`
+
+### Access services
+- Marimo: `https://marimo.local.dev`
+- Quarto: `https://quarto.local.dev`
+- Dagster: `https://dagster.local.dev`
+- Zitadel: `https://auth.local.dev`
+- Traefik dashboard (if enabled in values): `http://localhost:8080`
+
+All apps (Marimo, Quarto, Dagster) are protected behind OAuth2-Proxy. Accessing any of them will redirect to Zitadel for OIDC login before granting access.
+
+### Update or remove the release
+- Re-apply updated values: rerun the `helm upgrade --install ...` command from [02-configuration.md](./02-configuration.md) (or `just local-deploy`).
+- Remove everything: `helm uninstall srdp -n srdp`
+  - If you also want to clear persistent data: `kubectl delete pvc --all -n srdp`
+- Or use the task runner: `just local-delete`
+
+### Logs
+- Watch all pods: `kubectl logs -n srdp -l app.kubernetes.io/instance=srdp -f`
+- Specific service, e.g. Marimo: `kubectl logs -n srdp deploy/marimo -f`
+- Dagster webserver: `kubectl logs -n srdp deploy/srdp-dagster-webserver -f`
+- Dagster daemon: `kubectl logs -n srdp deploy/srdp-dagster-daemon -f`
 Once you have completed the setup, you can verify that all services are running correctly by accessing them in your web browser.
 
 ### Accessing Services