Multiple parallel KRM function evaluator pods for the same KRM functions (kptdev#336)

* NCDEV-2832 : support sequential KRM function evaluation in podEvaluator

* refactor podevaluator modules into separate files

* NCDEV-2833: Multiple parallel KRM function evaluator pods

* NCOMGITFM-1569 Schedule waitlisted items for failed pod to another

* NCDEV-3650: Unify KRM fn parameter names

* Add pod retrieve logic, fixes, resource request/limits for krm pods

* fix lint error

* lint fixes

* additional changes

* fix goscan

* add some buffer to e2e test

* set max-waitlist-length

* minor fixes

* additional unit test by thc1006

* add missing test file

* more unit tests by thc1006

* Increase the expected time buffer a little for e2e test

* comment fix

---------

Co-authored-by: Bhargav Anil Kumar Singh <singh.bhargav_anil_kumar@nokia.com>
Co-authored-by: Kosimov Javlonbek <javlonbek.kosimov@nokia.com>
Co-authored-by: Mózes László Máté <laszlo.mozes@nokia.com>

Author: 4 people

deployments/porch/2-function-runner.yaml

@@ -48,6 +48,8 @@ spec:
             - --functions=/home/nonroot/functions
             - --pod-namespace=porch-fn-system
             - --max-request-body-size=6291456 # Keep this in sync with porch-server's corresponding argument
+            - --max-parallel-pods-per-function=2
+            - --max-waitlist-length=1
             - --warm-up-pod-cache=true
           env:
             - name: WRAPPER_SERVER_IMAGE
@@ -107,22 +109,79 @@ metadata:
   namespace: porch-system
 data:
   pod-cache-config.yaml: |
-    ghcr.io/kptdev/krm-functions-catalog/apply-replacements:v0.1.1: 30m
-    ghcr.io/kptdev/krm-functions-catalog/apply-setters:v0.2.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/create-setters:v0.1.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/ensure-name-substring:v0.2.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/gatekeeper:v0.2.1: 30m
-    ghcr.io/kptdev/krm-functions-catalog/kubeconform:v0.1.1: 30m
-    ghcr.io/kptdev/krm-functions-catalog/search-replace:v0.2.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.4: 30m
-    ghcr.io/kptdev/krm-functions-catalog/set-enforcement-action:v0.1.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/set-image:v0.1.1: 30m
-    ghcr.io/kptdev/krm-functions-catalog/set-labels:v0.1.5: 30m
-    ghcr.io/kptdev/krm-functions-catalog/set-namespace:v0.4.1: 30m
-    ghcr.io/kptdev/krm-functions-catalog/starlark:v0.4.3: 30m
-    ghcr.io/kptdev/krm-functions-catalog/upsert-resource:v0.2.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/enable-gcp-services:v0.1.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/export-terraform:v0.1.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/generate-folders:v0.1.1: 30m
-    ghcr.io/kptdev/krm-functions-catalog/remove-local-config-resources:v0.1.0: 30m
-    ghcr.io/kptdev/krm-functions-catalog/set-project-id:v0.2.0: 30m
+    - name: ghcr.io/kptdev/krm-functions-catalog/apply-replacements:v0.1.1
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/apply-setters:v0.2.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/create-setters:v0.1.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/ensure-name-substring:v0.2.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/gatekeeper:v0.2.1
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/kubeconform:v0.1.1
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/search-replace:v0.2.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/set-annotations:v0.1.4
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/set-enforcement-action:v0.1.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/set-image:v0.1.1
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/set-labels:v0.1.5
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/set-namespace:v0.4.1
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/starlark:v0.4.3
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/upsert-resource:v0.2.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/enable-gcp-services:v0.1.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/export-terraform:v0.1.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/generate-folders:v0.1.1
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/remove-local-config-resources:v0.1.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2
+    - name: ghcr.io/kptdev/krm-functions-catalog/set-project-id:v0.2.0
+      timeToLive: 30m
+      maxWaitlistLength: 2
+      maxParallelPodsPerFunction: 2

docs/content/en/docs/6_configuration_and_deployments/configurations/components/function-runner-config/_index.md

@@ -34,12 +34,14 @@ args:
 ```bash
 args:
 - --pod-cache-config=/pod-cache-config/pod-cache-config.yaml  # Pod cache config file path
-- --warm-up-pod-cache=true        # Warm up pod cache on startup (default: true)
-- --pod-namespace=porch-fn-system # Namespace for KRM function pods (default: porch-fn-system)
-- --pod-ttl=30m                   # Pod TTL before GC (default: 30m)
-- --scan-interval=1m              # GC scan interval (default: 1m)
-- --function-pod-template=        # ConfigMap with pod specification
-- --max-request-body-size=6291456 # Max gRPC message size in bytes (default: 6MB)
+- --warm-up-pod-cache=true         # Warm up pod cache on startup (default: true)
+- --pod-namespace=porch-fn-system  # Namespace for KRM function pods (default: porch-fn-system)
+- --pod-ttl=30m                    # Pod TTL before GC (default: 30m)
+- --scan-interval=1m               # GC scan interval (default: 1m)
+- --function-pod-template=         # ConfigMap with pod specification
+- --max-request-body-size=6291456  # Max gRPC message size in bytes (default: 6MB)
+- --max-waitlist-length            # Maximum waitlist length per pod
+- --max-parallel-pods-per-function # Maximum parallel pods per function
 ```
 
 #### Private Registry Arguments