Merge release/0.7.0 to main

.coderabbit.yaml

@@ -0,0 +1,55 @@
+language: "en-US"
+early_access: false
+knowledge_base: 
+  web_search:
+    enabled: true
+  code_guidelines:
+    enabled: true
+    filePatterns:
+      - AGENTS.md
+reviews:
+  profile: "chill"
+  request_changes_workflow: false
+  auto_review:
+    auto_incremental_review: false
+    base_branches:
+      - "main"
+      - "dev"
+      - "release/.*"
+  path_instructions:
+    - path: "**/*.cs"
+      instructions: |
+        The `HtmlInputSanitizer` class's `Sanitize()` and `SanitizeAndRemove()` methods should be used when dealing with `string` query parameters from REST requests.
+    - path: "**/*.cs"
+      instructions: |
+        Double-check `CancellationToken` plumbing. Flag any of the following:
+        - An `async` method that does not accept a `CancellationToken` parameter when one is reasonably available from the caller (HTTP endpoints, Kafka consumers/listeners,
+          Quartz jobs via `IJobExecutionContext.CancellationToken`, background services' `stoppingToken`, `ExecuteAsync` overrides, etc.).
+        - A method that accepts a `CancellationToken` but does not forward it to every call it makes that has a `CancellationToken` overload
+          (e.g., `HttpClient.GetAsync`/`PostAsync`/`SendAsync`, `HttpContent.ReadAsStringAsync`/`ReadAsStreamAsync`, `DbContext.SaveChangesAsync`,
+          EF Core async LINQ operators `ToListAsync`/`FirstOrDefaultAsync`/`AnyAsync`/`CountAsync`, `Stream.ReadAsync`/`WriteAsync`, `Task.Delay`,
+          `SemaphoreSlim.WaitAsync`, `Channel`/`BlockingCollection` async APIs, `IProducer.Flush`, Polly policies, MediatR `Send`/`Publish`, gRPC client calls, etc.).
+        - Use of `CancellationToken.None` or `default` inside a method that already has a `CancellationToken` in scope — call it out and ask why the ambient token is not used.
+        - A timeout-only `CancellationTokenSource` (e.g., `new CancellationTokenSource(TimeSpan.FromSeconds(...))`) whose token is passed to downstream calls without being
+          linked to the caller's `CancellationToken` via `CancellationTokenSource.CreateLinkedTokenSource(callerToken, timeoutCts.Token)`. Either signal should be honored.
+        - A long-running loop (per-patient, per-row, per-batch) that does not at least call `cancellationToken.ThrowIfCancellationRequested()` per iteration when the inner
+          operation has no `CancellationToken` overload (e.g., `IProducer<K,V>.Produce`).
+        - Public APIs on services/managers/repositories where adding an optional `CancellationToken cancellationToken = default` parameter would improve cooperative
+          cancellation without breaking existing callers — suggest adding it.
+        When flagging, prefer concrete suggestions showing how to thread the token through rather than generic comments.
+    - path: "**"
+      instructions: |
+        Pull requests that have "TECH_DEBT" in the title should only contain changes related to typos, unused code, linter/IDE suggestions, swagger specification updates,
+        and logging improvements. These TECH_DEBT PRs must not affect core functionality. All PRs that are not considered technical debt must include information on what 
+        testing was performed in the description of the PR. If it does not, ask the author to provide details on what testing was performed.
+        When reviewing code, suggest unit tests using XUnit in the following scenarios:
+        - If/Else or Switch/Case blocks are introduced or modified — ensure each branch has a corresponding unit test.
+        - Logic that depends on service or interface configuration — suggest tests to validate different implementations are correctly resolved.
+        - No network activity (HTTP calls, sockets, etc.) should appear in unit tests. Recommend using mocks (via Moq) for any external communication.
+        Large unit tests should be avoided; keeping unit tests small and focused on targeted business logic (i.e. string sanitization)
+        All logging message arguments must be sanitized before they are passed to logger methods to avoid static code scanning security/vulnerability findings.
+
+        **App Config Check**: 
+        If new required configuration keys (that don't have default values) are introduced, verify if new configuration keys have been added. If new required configuration keys are introduced, ensure that the 
+        root-level `/app-config.yaml` file has been updated to include these new keys with appropriate descriptions. If `/app-config.yaml` is not updated 
+        when new configuration is added, flag this as a required change and recommend blocking the PR until the configuration is documented in `/app-config.yaml`.

.docker/collector.yml

@@ -0,0 +1,37 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+exporters:
+  otlp/tempo:
+    endpoint: tempo:4417
+    tls:
+      insecure: true
+
+  prometheus:
+    endpoint: "0.0.0.0:8889"  
+    const_labels:
+      label1: value1
+
+processors:
+  batch:
+
+extensions:
+  health_check:
+  pprof:
+    endpoint: :1888
+  zpages:
+    endpoint: :55679
+
+service:
+  extensions: [pprof, zpages, health_check]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/tempo]
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus]

.docker/create-dbs.sql

@@ -0,0 +1,10 @@
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-audit') CREATE DATABASE [link-audit];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-census') CREATE DATABASE [link-census];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-dataacquisition') CREATE DATABASE [link-dataacquisition];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-normalization') CREATE DATABASE [link-normalization];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-notification') CREATE DATABASE [link-notification];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-tenant') CREATE DATABASE [link-tenant];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-validation') CREATE DATABASE [link-validation];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-querydispatch') CREATE DATABASE [link-querydispatch];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-account') CREATE DATABASE [link-account];
+IF NOT EXISTS (SELECT * FROM sys.databases WHERE name = 'link-submission') CREATE DATABASE [link-submission];

.docker/grafana.yml

@@ -0,0 +1,70 @@
+apiVersion: 1
+datasources:
+- name: Loki
+  type: loki
+  access: proxy
+  orgId: 1
+  url: http://loki:3100
+  basicAuth: false
+  isDefault: true
+  version: 1
+  editable: true
+  jsonData:
+    maxLines: 1000
+    derivedFields:
+      - datasourceUid: Tempo
+        matcherRegex: .*TraceId\":"(\w{32}).*
+        name: TraceID
+        url: '$${__value.raw}'
+- name: Tempo
+  type: tempo
+  access: proxy
+  orgId: 1
+  url: http://tempo:3200
+  basicAuth: false
+  isDefault: false
+  version: 1
+  editable: true
+  apiVersion: 1
+  uid: tempo
+  jsonData:
+      tracesToLogsV2:
+        # Field with an internal link pointing to a logs data source in Grafana.
+        # datasourceUid value must match the uid value of the logs data source.
+        datasourceUid: Loki
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+        tags: ['job', 'instance', 'pod', 'namespace']
+        filterByTraceID: false
+        filterBySpanID: false
+        customQuery: true
+        #query: 'TraceID="${__span.tags.method}"'
+        query: '{app="{Link-BoTW}"} |= `` | json | TraceId = `${__span.tags.traceId}`'
+      tracesToMetrics:
+        datasourceUid: Prometheus
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+        tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }]
+        queries:
+          - name: 'Sample query'
+            query: 'sum(rate(process_runtime_dotnet_gc_committed_memory_size_bytes{$__tags}[5m]))'
+      serviceMap:
+        datasourceUid: Prometheus
+      nodeGraph:
+        enabled: true
+      search:
+        hide: false
+      lokiSearch:
+        datasourceUid: Loki
+      traceQuery:
+        timeShiftEnabled: true
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '-1h'
+      spanBar:
+        type: 'Tag'
+        tag: 'http.path'
+- name: Prometheus
+  type: prometheus
+  url: http://prometheus:9090
+  access: proxy
+  editable: true

.docker/kafka-client.properties

@@ -0,0 +1,5 @@
+security.protocol=SASL_PLAINTEXT
+sasl.mechanism=PLAIN
+sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
+        username="user" \
+        password="password";

.docker/prometheus.yml

@@ -0,0 +1,6 @@
+scrape_configs:
+  - job_name: 'otel-collector'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['otel-collector:8889']
+      - targets: ['otel-collector:8888']

.docker/tempo.yml

@@ -0,0 +1,48 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    jaeger:
+      protocols:
+        thrift_http:
+        grpc:
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        http:
+        grpc: 
+          endpoint: tempo:4417
+    opencensus:
+
+ingester:
+  trace_idle_period: 10s
+  max_block_bytes: 1_000_000
+  max_block_duration: 5m
+
+compactor:
+  compaction:
+    compaction_window: 1h
+    max_block_bytes: 100_000_000
+    block_retention: 1h
+    compacted_block_retention: 10m
+
+storage:
+  trace:
+    backend: local
+    block:
+      bloom_filter_false_positive: .05
+      v2_index_downsample_bytes: 1000
+      v2_encoding: zstd
+    wal:
+      path: /tmp/tempo/wal
+      v2_encoding: snappy
+    local:
+      path: /tmp/tempo/blocks
+    pool:
+      max_workers: 100
+      queue_depth: 10000
+
+

.dockerignore

@@ -1,7 +1,19 @@
+.docker/
+.github/
+.junie/
+Azure_Pipelines/
+docs/
+Java/
+**/*Tests/
+Scripts/
+Web/
 **/.classpath
+**/.coderabbit.yaml
 **/.dockerignore
+**/.editorconfig
 **/.env
 **/.git
+**/.gitattributes
 **/.gitignore
 **/.project
 **/.settings
@@ -12,16 +24,19 @@
 **/*.dbmdl
 **/*.jfm
 **/azds.yaml
-**/bin
+**/bin/
 **/charts
 **/docker-compose*
 **/Dockerfile*
 **/node_modules
 **/npm-debug.log
-**/obj
+**/obj/
 **/secrets.dev.yaml
 **/values.dev.yaml
 **/.idea
 *.iml
+AGENTS.md
 LICENSE
-README.md
+README.md
+SECURITY.md
+topics.txt

.editorconfig

@@ -1,4 +1,17 @@
-[*.cs]
+root = true
+
+[*]
+charset = utf-8
+
+[*.md]
+charset = utf-8-bom
+
+[*.cs]
+charset = utf-8-bom
 
 # CS8603: Possible null reference return.
 dotnet_diagnostic.CS8603.severity = silent
+
+# CA2254: Template should be a static expression
+dotnet_diagnostic.CA2254.severity = error
+dotnet_diagnostic.CA2253.severity = error

.env

@@ -0,0 +1,22 @@
+LINK_DB_USER=link-db-user
+LINK_DB_PASS=7h3I^xMY%cgO
+REDIS_PASS=suT1ChwYtkwWfbV1bUfRvQZO1YNhNRMj
+KAFKA_SASL_CLIENT_USER=user
+KAFKA_SASL_CLIENT_PASSWORD=password
+KAFKA_INTER_BROKER_USER=controller_user
+KAFKA_INTER_BROKER_PASSWORD=controller_password
+SFTP_USER=admin
+SFTP_PASSWORD=password
+AZURITE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==
+AZURITE_CONNECTION_STRING=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=${AZURITE_ACCOUNT_KEY};BlobEndpoint=http://azurite:10000/devstoreaccount1
+INTERNAL_BLOB_CONTAINER_NAME=internal
+EXTERNAL_BLOB_CONTAINER_NAME=external
+EXTERNAL_BLOB_FLATTEN_HIERARCHY=false
+ISSUER_URI=https://oauth.nhsnlink.org/realms/NHSNLink
+
+KafkaConnection__SaslUsername=${KAFKA_SASL_CLIENT_USER}
+KafkaConnection__SaslPassword=${KAFKA_SASL_CLIENT_PASSWORD}
+Redis__Password=${REDIS_PASS}
+InternalBlobStorage__BlobContainerName=${INTERNAL_BLOB_CONTAINER_NAME}
+ExternalBlobStorage__BlobContainerName=${EXTERNAL_BLOB_CONTAINER_NAME}
+ExternalBlobStorage__FlattenHierarchy=${EXTERNAL_BLOB_FLATTEN_HIERARCHY}

.gitattributes

@@ -2,6 +2,8 @@
 # Set default behavior to automatically normalize line endings.
 ###############################################################################
 * text=auto
+*.sh text eol=lf
+/topics.txt text eol=lf
 
 ###############################################################################
 # Set default behavior for command prompt diff.