Implement an API for performance monitoring

Signed-off-by: Alexei Dodon <adodon@cisco.com>

Author: adodon2go

.github/workflows/oci-conformance-action.yml

@@ -33,14 +33,14 @@ jobs:
       run: |
           cd ./zot
           IP=`hostname -I | awk '{print $1}'`
-          echo "ZOT_URL=http://${IP}:5000" >> $GITHUB_ENV
-          ZOT_REF="local-zot:v$(date +%Y%m%d%H%M%S)"
-          docker build -f ./Dockerfile-conformance -t "${ZOT_REF}" .
-          docker run --rm -p 5000:5000 -v "$(pwd)":/go/src/github.com/anuvu/zot -idt "${ZOT_REF}"
+          echo "SERVER_URL=http://${IP}:5000" >> $GITHUB_ENV
+          IMAGE_REF="local-zot:v$(date +%Y%m%d%H%M%S)"
+          docker build -f ./Dockerfile-conformance -t "${IMAGE_REF}" .
+          docker run --rm -p 5000:5000 -v "$(pwd)":/go/src/github.com/anuvu/zot -idt "${IMAGE_REF}"
     - name: Run OCI Distribution Spec conformance tests
       uses: opencontainers/distribution-spec@main
       env:
-        OCI_ROOT_URL: ${{ env.ZOT_URL }}
+        OCI_ROOT_URL: ${{ env.SERVER_URL }}
         OCI_NAMESPACE: oci-conformance/distribution-test
         OCI_TEST_PULL: 1
         OCI_TEST_PUSH: 1

.gitignore

@@ -14,6 +14,8 @@
 bin/
 bazel-*
 coverage.txt
+coverage-extended.txt
+coverage-minimal.txt
 test/data/
 *.orig
 .idea/

CONTRIBUTING.md

@@ -44,6 +44,12 @@ For a minimal dist-spec only zot,
 make binary-minimal
 ```
 
+For a node exporter used by minimal dist-spec only zot,
+
+```
+make exporter-minimal
+```
+
 ## Using container builds (stacker)
 
 ```
@@ -62,12 +68,14 @@ make binary-container
 .
 ...
 ├── cmd/zot             # Source code contains the main logic
+├── cmd/exporter        # Source code contains the main logic for node exporter
 ├── docs                # Source code for Swagger docs
 ├── errors              # Source code for errors
 ├── examples            # Configuration examples to enable various features
 ├── pkg/api             # Source code contains the HTTP handlers
 ├── pkg/cli             # Source code that handles the commandline logic
 ├── pkg/compliance      # Source code that handles the dist-spec compliance logic
+├── pkg/exporter        # Source code used by the node exporter
 ├── pkg/extensions      # Source code that handles the feature extensions
 ├── pkg/log             # Source code that handles logging
 ├── pkg/storage         # Source code that handles image storage

Makefile

@@ -1,46 +1,55 @@
 export GO111MODULE=on
 TOP_LEVEL=$(shell git rev-parse --show-toplevel)
 COMMIT_HASH=$(shell git describe --always --tags --long)
+GO_VERSION=$(shell go version | awk '{print $$3}')
 COMMIT=$(if $(shell git status --porcelain --untracked-files=no),$(COMMIT_HASH)-dirty,$(COMMIT_HASH))
 CONTAINER_RUNTIME := $(shell command -v podman 2> /dev/null || echo docker)
 PATH := bin:$(PATH)
 TMPDIR := $(shell mktemp -d)
 STACKER := $(shell which stacker)
 
 .PHONY: all
-all: swagger binary binary-minimal debug test test-clean check
+all: swagger binary binary-minimal exporter-minimal debug test test-clean check
 
 .PHONY: binary-minimal
 binary-minimal: swagger
-	go build -tags minimal,containers_image_openpgp -v  -ldflags "-X  github.com/anuvu/zot/pkg/api/config.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=minimal" -o bin/zot-minimal ./cmd/zot
+	go build -o bin/zot-minimal -tags minimal,containers_image_openpgp -v -trimpath -ldflags "-X  github.com/anuvu/zot/pkg/api/config.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api/config.BinaryType=minimal -X github.com/anuvu/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot
 
 .PHONY: binary
-binary: swagger 
-	go build -tags extended,containers_image_openpgp -v -ldflags "-X  github.com/anuvu/zot/pkg/api/config.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=extended" -o bin/zot ./cmd/zot
+binary: swagger
+	go build -o bin/zot -tags extended,containers_image_openpgp -v -trimpath -ldflags "-X  github.com/anuvu/zot/pkg/api/config.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api/config.BinaryType=extended -X github.com/anuvu/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot
 
 .PHONY: debug
 debug: swagger
-	go build -tags extended,containers_image_openpgp -v -gcflags all='-N -l' -ldflags "-X  github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=extended" -o bin/zot-debug ./cmd/zot
+	go build -o bin/zot-debug -tags extended,containers_image_openpgp -v -gcflags all='-N -l' -ldflags "-X  github.com/anuvu/zot/pkg/api/config.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api/config.BinaryType=extended -X github.com/anuvu/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot
+
+.PHONY: exporter-minimal
+exporter-minimal: swagger
+	go build -o bin/zot-exporter -tags minimal,containers_image_openpgp -v -trimpath ./cmd/exporter
 
 .PHONY: test
 test:
 	$(shell mkdir -p test/data;  cd test/data; ../scripts/gen_certs.sh; cd ${TOP_LEVEL}; sudo skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TOP_LEVEL}/test/data/zot-test:0.0.1;sudo skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TOP_LEVEL}/test/data/zot-cve-test:0.0.1)
 	$(shell sudo mkdir -p /etc/containers/certs.d/127.0.0.1:8089/; sudo cp test/data/client.* /etc/containers/certs.d/127.0.0.1:8089/; sudo cp test/data/ca.* /etc/containers/certs.d/127.0.0.1:8089/;)
 	$(shell sudo chmod a=rwx /etc/containers/certs.d/127.0.0.1:8089/*.key)
-	go test -tags extended,containers_image_openpgp -v -race -cover -coverpkg ./... -coverprofile=coverage.txt -covermode=atomic ./...
+	go test -tags extended,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./...
+	go test -tags minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
 
 .PHONY: test-clean
 test-clean:
 	$(shell sudo rm -rf /etc/containers/certs.d/127.0.0.1:8089/)
 
 .PHONY: covhtml
 covhtml:
+	tail -n +2 coverage-minimal.txt > tmp.txt && mv tmp.txt coverage-minimal.txt
+	cat coverage-extended.txt coverage-minimal.txt > coverage.txt
 	go tool cover -html=coverage.txt -o coverage.html
 
 .PHONY: check
 check: ./golangcilint.yaml
 	golangci-lint --version || curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.26.0
-	golangci-lint --config ./golangcilint.yaml run --enable-all --build-tags extended,containers_image_openpgp ./cmd/... ./pkg/...
+	golangci-lint --config ./golangcilint.yaml run --enable-all --build-tags extended,containers_image_openpgp ./...
+	golangci-lint --config ./golangcilint.yaml run --enable-all --build-tags minimal,containers_image_openpgp ./...
 
 swagger/docs.go: 
 	swag -v || go install github.com/swaggo/swag/cmd/swag

README.md

@@ -41,6 +41,8 @@ https://anuvu.github.io/zot/
 * Swagger based documentation
 * Single binary for _all_ the above features
 * Released under Apache 2.0 License
+* [Metrics](#metrics) with Prometheus
+  * Using a node exporter in case of dist-spec-only zot
 * ```go get -u github.com/anuvu/zot/cmd/zot```
 
 # Presentations
@@ -267,9 +269,24 @@ runtime interface.
 
 Works with "docker://" transport which is the default.
 
+# Metrics
+
+Can be used for both dist-spec-only zot & the zot with all extensions enabled
+
+## Node Exporter
+The dist-spec-only zot exposes internal metrics into a Prometheus format through a node exporter.
+The configuration of node exporter contains connection details for the zot server it is intend to scrape metrics from. See a [configuration example](./examples/metrics/exporter/config-minimal.json). The metrics are automatically enabled in the zot server on first scrape from the Node Exporter (no extra configuration option is needed). Similarly, the metrics are automatically disabled when Node Exporter did not perform any scrapings in a while.
+
+```
+bin/zot-exporter config _config-file_
+```
+
+## Enable Metrics
+In the zot with all extensions case see [configuration example](./examples/config-metrics.json) for enabling metrics
+
 # Caveats
 
-* go 1.12+
+* go 1.15+
 * The OCI distribution spec is still WIP, and we try to keep up
 
 # Contributing

cmd/exporter/main.go

@@ -0,0 +1,15 @@
+// +build minimal
+
+package main
+
+import (
+	"os"
+
+	"github.com/anuvu/zot/pkg/exporter/cli"
+)
+
+func main() {
+	if err := cli.NewExporterCmd().Execute(); err != nil {
+		os.Exit(1)
+	}
+}

examples/README.md

@@ -23,6 +23,7 @@ Examples of working configurations for various use cases are available [here](..
 * [Authentication](#authentication)
 * [Identity-based Authorization](#identity-based-authorization)
 * [Logging](#logging)
+* [Metrics](#metrics)
 
 
 ## Network
@@ -245,3 +246,24 @@ Enable audit logs and set output file with:
     "audit": "/tmp/zot-audit.log"
   }
 ```
+
+## Metrics
+
+Enable and configure metrics with:
+
+```
+"metrics":{
+    "enable":"true",
+
+```
+
+Set server path on which metrics will be exposed:
+
+```
+    "prometheus": {
+      "path": "/metrics"
+    }
+}
+```
+
+In order to test the Metrics feature locally in a [Kind](https://kind.sigs.k8s.io/) cluster, folow [this guide](metrics/README.md).

examples/config-metrics.json

@@ -0,0 +1,21 @@
+{
+    "version": "0.1.0-dev",
+    "storage": {
+        "rootDirectory": "/tmp/zot"
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080"
+    },
+    "log": {
+        "level": "debug"
+    },
+    "extensions": {
+        "metrics": {
+            "enable": true,
+            "prometheus": {
+                "path": "/metrics"
+            }
+        }
+    }
+}

examples/config-policy.json

@@ -0,0 +1,40 @@
+# ---
+# Stage 1: Install certs, build binary, create default config file
+# ---
+FROM docker.io/golang:1.16 AS builder
+RUN mkdir -p /go/src/github.com/anuvu/zot
+WORKDIR /go/src/github.com/anuvu/zot
+COPY . .
+RUN CGO_ENABLED=0 make clean binary
+RUN echo '{\n\
+    "storage": {\n\
+        "rootDirectory": "/var/lib/registry"\n\
+    },\n\
+    "http": {\n\
+        "address": "0.0.0.0",\n\
+        "port": "5000"\n\
+    },\n\
+    "log": {\n\
+        "level": "debug"\n\
+    },\n\
+    "extensions": {\n\
+        "metrics": {\n\
+            "enable": true,\n\
+            "prometheus": {\n\
+                "path": "/metrics"\n\
+            }\n\
+        }\n\
+    }\n\
+}\n' > config.json && cat config.json
+
+# ---
+# Stage 2: Final image with nothing but certs, binary, and default config file
+# ---
+FROM scratch AS final
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /go/src/github.com/anuvu/zot/bin/zot /zot
+COPY --from=builder /go/src/github.com/anuvu/zot/config.json /etc/zot/config.json
+ENTRYPOINT ["/zot"]
+EXPOSE 5000
+VOLUME ["/var/lib/registry"]
+CMD ["serve", "/etc/zot/config.json"]