Merge branch 'dev' into static-ct-backoff

Author: d-Rickyy-b

.github/goreleaser.yml

@@ -15,23 +15,23 @@ builds:
       - darwin
       - windows
     goarch:
-      - 386
+      - "386"
       - amd64
       - arm
       - arm64
     ignore:
       - goos: darwin
-        goarch: 386
+        goarch: "386"
       - goos: darwin
         goarch: arm
       - goos: windows
         goarch: arm
       - goos: windows
         goarch: arm64
       - goos: windows
-        goarch: 386
+        goarch: "386"
 checksum:
-  name_template: '{{.ProjectName}}_{{.Version}}_checksums.txt'
+  name_template: "{{.ProjectName}}_{{.Version}}_checksums.txt"
 changelog:
   disable: true
 
@@ -41,8 +41,8 @@ dockers_v2:
       - "ghcr.io/d-rickyy-b/{{.ProjectName}}"
     dockerfile: docker/Dockerfile.goreleaser
     tags:
-        - "{{.Tag}}"
-        - "{{ if not .Prerelease }}latest{{ end }}"
+      - "{{.Tag}}"
+      - "{{ if not .Prerelease }}latest{{ end }}"
     platforms:
       - linux/amd64
       - linux/arm64
@@ -58,7 +58,7 @@ dockers_v2:
       "org.opencontainers.image.source": "https://github.com/d-Rickyy-b/certstream-server-go"
 
 archives:
-  - formats: [ "binary" ]
+  - formats: ["binary"]
     name_template: >-
       {{- .ProjectName }}_
       {{- .Version}}_

.github/workflows/release_build.yml

@@ -4,6 +4,8 @@ on:
   push:
     tags:
       - "*"
+    branches:
+      - dev
 
 jobs:
   build:

CHANGELOG.md

@@ -13,9 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated http client settings to prevent timeouts and other connectivity issues
 - Updated http server settings to allow for higher delays
 - Minor code improvements and refactoring, mostly style related
+- Updated batch size from 100 to 256
 ### Removed
 ### Fixed
+- Calculate correct starting tile index for static ct logs
 - Use proper websocket close code (1008) instead of 1005, which wasn't sent to the client
+- Respect ct_index path in config file for the `create-index` command
 ### Docs
 
 ## [1.9.0] - 2026-04-03

config.sample.yaml

@@ -3,8 +3,19 @@ webserver:
   listen_addr: "0.0.0.0"
   listen_port: 8080
   # If you want to use a reverse proxy in front of the server, set this to true
-  # It will use the X-Forwarded-For header to get the real IP of the client
+  # It will use the X-Forwarded-For / X-Real-IP headers to get the real IP of the client
   real_ip: false
+  # List of trusted proxy IPs or CIDR ranges whose X-Forwarded-For / X-Real-IP headers
+  # are honored when real_ip is true. When left empty every connecting IP is trusted.
+  # Restricting this to your actual proxy IPs prevents clients from spoofing their address
+  # by injecting a forged forwarded header. Note: X-Forwarded-For can contain a chain of
+  # IPs (e.g. "client, proxy1, proxy2"); only the first (leftmost) entry is used as the real client IP.
+  trusted_proxies:
+    - "172.16.0.0/12"
+  # If you want to restrict access to the server to specific IPs or CIDR ranges, you can add them to the whitelist.
+  # If the whitelist is empty, all IPs are allowed to access the server.
+  whitelist:
+    - "127.0.0.1/8"
   full_url: "/full-stream"
   lite_url: "/"
   domains_only_url: "/domains-only"
@@ -20,6 +31,9 @@ prometheus:
   metrics_url: "/metrics"
   expose_system_metrics: false
   real_ip: false
+  # List of trusted proxy IPs or CIDR ranges (see webserver.trusted_proxies for details).
+  #trusted_proxies:
+  #  - "127.0.0.1"
   whitelist:
     - "127.0.0.1/8"
 

internal/certificatetransparency/ct-tiled.go

@@ -118,43 +118,6 @@ func FetchCheckpoint(ctx context.Context, client *http.Client, baseURL string) (
 	}, nil
 }
 
-// FetchTile fetches a tile from the tiled CT log using the provided client.
-// If partialWidth > 0, fetches a partial tile with that width (1-255).
-func FetchTile(ctx context.Context, client *http.Client, baseURL string, tileIndex, partialWidth uint64) ([]TileLeaf, error) {
-	baseURL = strings.TrimRight(baseURL, "/")
-	tilePath := encodeTilePath(tileIndex)
-
-	if partialWidth > 0 {
-		tilePath = fmt.Sprintf("%s.p/%d", tilePath, partialWidth)
-	}
-
-	url := fmt.Sprintf("%s/tile/data/%s", baseURL, tilePath)
-
-	req, newReqErr := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if newReqErr != nil {
-		return nil, fmt.Errorf("failed to create tile request: %w", newReqErr)
-	}
-
-	req.Header.Set("User-Agent", UserAgent)
-
-	resp, reqErr := client.Do(req)
-	if reqErr != nil {
-		return nil, fmt.Errorf("fetching tile %d: %w", tileIndex, reqErr)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("%w: unexpected status code %d", ErrRequestFailed, resp.StatusCode)
-	}
-
-	data, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("reading tile data: %w", err)
-	}
-
-	return ParseTileData(data)
-}
-
 // ParseTileData parses the binary tile data into TileLeaf entries using cryptobyte.
 func ParseTileData(data []byte) ([]TileLeaf, error) {
 	var leaves []TileLeaf
@@ -330,7 +293,7 @@ func (s *StaticCTClient) Monitor(ctx context.Context, foundCert func(*ct.RawLogE
 // It returns true if at least one full tile was fetched.
 func (s *StaticCTClient) fetchAndProcessTiles(ctx context.Context, foundCert func(*ct.RawLogEntry), foundPrecert func(*ct.RawLogEntry)) (bool, error) {
 	// Fetch current checkpoint
-	checkpoint, fetchErr := s.fetchCheckpoint(ctx)
+	checkpoint, fetchErr := s.FetchCheckpoint(ctx)
 	if fetchErr != nil {
 		return false, fmt.Errorf("fetching checkpoint: %w", fetchErr)
 	}
@@ -471,8 +434,8 @@ func (s *StaticCTClient) fetchTile(ctx context.Context, tileIndex, partialWidth
 	return ParseTileData(data)
 }
 
-// fetchCheckpoint fetches the checkpoint from a tiled CT log using the provided client.
-func (s *StaticCTClient) fetchCheckpoint(ctx context.Context) (*TiledCheckpoint, error) {
+// FetchCheckpoint fetches the checkpoint from a tiled CT log using the provided client.
+func (s *StaticCTClient) FetchCheckpoint(ctx context.Context) (*TiledCheckpoint, error) {
 	url := s.url + "/checkpoint"
 
 	req, newReqErr := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)

internal/certificatetransparency/ct-watcher.go

@@ -75,7 +75,7 @@ func (w *Watcher) Start() {
 
 		// Start background job to save CTIndexes at regular intervals
 		storageInterval := time.Second * 30
-		go metrics.Metrics.SaveCertIndexesAtInterval(storageInterval, ctIndexFilePath)
+		go metrics.Metrics.SaveCertIndexesAtInterval(w.context, storageInterval, ctIndexFilePath)
 	}
 
 	// initialize the watcher with currently available logs
@@ -309,7 +309,8 @@ func (w *Watcher) CreateIndexFile(filePath string) error {
 			metrics.Metrics.Init(operator.Name, normalizedURL)
 			log.Println("Fetching checkpoint for", normalizedURL)
 
-			checkpoint, fetchErr := FetchCheckpoint(w.context, httpClient, transparencyLog.MonitoringURL)
+			staticCTClient := NewStaticCTClient(transparencyLog.MonitoringURL, httpClient, UserAgent, 0)
+			checkpoint, fetchErr := staticCTClient.FetchCheckpoint(w.context)
 			if fetchErr != nil {
 				log.Printf("Could not get checkpoint for '%s': %s\n", transparencyLog.MonitoringURL, fetchErr)
 				return ErrFetchingSTHFailed
@@ -444,7 +445,7 @@ func (w *worker) runStandardWorker(ctx context.Context) error {
 
 	certScanner := scanner.NewScanner(jsonClient, scanner.ScannerOptions{
 		FetcherOptions: scanner.FetcherOptions{
-			BatchSize:     100,
+			BatchSize:     256,
 			ParallelFetch: 1,
 			StartIndex:    int64(w.ctIndex),
 			Continuous:    true,
@@ -469,19 +470,20 @@ func (w *worker) runStandardWorker(ctx context.Context) error {
 func (w *worker) runTiledWorker(ctx context.Context) error {
 	httpClient := newHTTPClient()
 
+	staticCTClient := NewStaticCTClient(w.ctURL, httpClient, UserAgent, w.ctIndex)
+
 	// If recovery is enabled and the CT index is set, we start at the saved index. Otherwise, we start at the latest checkpoint.
 	validSavedCTIndexExists := config.AppConfig.General.Recovery.Enabled
 	if !validSavedCTIndexExists {
-		checkpoint, err := FetchCheckpoint(ctx, httpClient, w.ctURL)
+		checkpoint, err := staticCTClient.FetchCheckpoint(ctx)
 		if err != nil {
 			log.Printf("Could not get checkpoint for '%s': %s\n", w.ctURL, err)
 			return ErrFetchingSTHFailed
 		}
 		// Start at the latest checkpoint to skip all the past certificates
-		w.ctIndex = checkpoint.Size
+		staticCTClient.ctIndex = checkpoint.Size
 	}
 
-	staticCTClient := NewStaticCTClient(w.ctURL, httpClient, UserAgent, w.ctIndex)
 	err := staticCTClient.Monitor(ctx, w.foundCertCallback, w.foundPrecertCallback)
 	if err != nil {
 		return fmt.Errorf("error scanning for certificates: %w", err)

internal/metrics/logmetrics.go

@@ -1,6 +1,7 @@
 package metrics
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -197,6 +198,13 @@ func (m *LogMetrics) LoadCTIndex(ctIndexFilePath string) {
 		}
 	}
 
+	m.index = make(CTCertIndex)
+
+	if len(bytes) == 0 || string(bytes) == "" {
+		log.Panicln("CT index file is empty!")
+		return
+	}
+
 	jerr := json.Unmarshal(bytes, &m.index)
 	if jerr != nil {
 		log.Printf("Error unmarshalling CT index file: '%s'\n", ctIndexFilePath)
@@ -207,8 +215,7 @@ func (m *LogMetrics) LoadCTIndex(ctIndexFilePath string) {
 }
 
 func (m *LogMetrics) createCTIndexFile(ctIndexFilePath string) error {
-	log.Printf("Specified CT index file does not exist: '%s'\n", ctIndexFilePath)
-	log.Println("Creating CT index file now!")
+	log.Printf("Specified CT index file does not exist, creating now: '%s'\n", ctIndexFilePath)
 
 	file, createErr := os.Create(ctIndexFilePath)
 	if createErr != nil {
@@ -239,13 +246,18 @@ func (m *LogMetrics) createCTIndexFile(ctIndexFilePath string) error {
 // We first create a temp file and write the index data to it. Only then do we move the temp file to the actual
 // permanent index file. This prevents the last good index file from being clobbered if the program was shutdown/killed
 // in-between the write operation.
-func (m *LogMetrics) SaveCertIndexesAtInterval(interval time.Duration, ctIndexFilePath string) {
+func (m *LogMetrics) SaveCertIndexesAtInterval(ctx context.Context, interval time.Duration, ctIndexFilePath string) {
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
 
-	for range ticker.C {
-		if err := m.SaveCertIndexes(ctIndexFilePath); err != nil {
-			log.Printf("Error saving CT indexes at '%s': %s\n", ctIndexFilePath, err)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			if err := m.SaveCertIndexes(ctIndexFilePath); err != nil {
+				log.Printf("Error saving CT indexes at '%s': %s\n", ctIndexFilePath, err)
+			}
 		}
 	}
 }