[nr-k8s-otel-collector] kube job related metric optimization (#1747)

<!--
Thank you for contributing to New Relic's Helm charts. Before you submit
this PR we'd like to
make sure you are aware of our technical requirements:

*
https://github.com/newrelic-experimental/helm-charts/blob/master/CONTRIBUTING.md#technical-requirements

For a quick overview across what we will look at reviewing your PR,
please read
our review guidelines:

*
https://github.com/newrelic-experimental/helm-charts/blob/master/REVIEW_GUIDELINES.md

Following our best practices right from the start will accelerate the
review process and
help get your PR merged quicker.

When updates to your PR are requested, please add new commits and do not
squash the
history. This will make it easier to identify new changes. The PR will
be squashed
anyways when it is merged. Thanks.

For fast feedback, please @-mention maintainers that are listed in the
Chart.yaml file.

Please make sure you test your changes before you push them. Once
pushed, a Github Action
will run across your changes and do some initial checks and linting.
These checks run
very quickly. Please check the results. We would like these checks to
pass before we
even continue reviewing your changes.
-->
#### Is this a new chart

#### What this PR does / why we need it:

A quick scan of cluster which is heavy on jobs had the following no of
metrics in a 1 hour window
```
Metric Name,Metrics
kube_job_complete,7253178
kube_job_status_failed,2449419
kube_job_spec_parallelism,2428569
kube_job_status_start_time,2428569
kube_job_status_active,2428569
kube_job_created,2428569
kube_job_spec_completions,2428569
kube_job_status_succeeded,2428569
kube_job_owner,2428569
kube_job_status_completion_time,2417726
newrelic.goldenmetrics.infra.kubernetes_job.podsActive,2363142
newrelic.goldenmetrics.infra.kubernetes_job.podsSucceeded,2363083
newrelic.goldenmetrics.infra.kubernetes_job.podsFailed,2362910
newrelic.goldenmetrics.infra.kubernetes_job.completedAt,2352631
kube_pod_container_status_phase,1583100
kube_pod_status_ready,1522572
kube_pod_status_scheduled,1522572
kube_pod_container_status_ready,527700
kube_pod_container_info,527700
kube_pod_container_status_restarts_total,514736
kube_pod_status_scheduled_time,507524
kube_pod_status_phase,507524
kube_pod_created,507524
kube_pod_owner,507524
kube_pod_info,505966
newrelic.goldenmetrics.infra.kubernetes_pod.podScheduled,494193
newrelic.goldenmetrics.infra.kubernetes_pod.podReadiness,494190
container_network_receive_errors_total,379522
container_network_transmit_errors_total,377702
container_network_transmit_bytes_total,238746
container_network_receive_bytes_total,231011
apiserver_storage_objects,211692
container_spec_memory_limit_bytes,126501
container_memory_working_set_bytes,126261
container_cpu_usage_seconds_total,123971
kube_deployment_status_condition,118578
kube_replicaset_owner,95498
kube_pod_container_resource_requests,59581
kube_pod_status_ready_time,52883
kube_pod_container_resource_limits,36167
kube_service_created,34200
kube_service_info,34200
kube_service_spec_type,34200
kube_job_failed,31635
container.filesystem.usage,20385
container.memory.usage,20385
container.filesystem.capacity,20385
newrelic.goldenmetrics.infra.container.memoryUsage,20178
newrelic.goldenmetrics.infra.container.storageUsage,20178
kube_deployment_status_replicas_unavailable,19823
kube_deployment_status_replicas_updated,19823
kube_deployment_spec_replicas,19823
kube_deployment_metadata_generation,19823
kube_deployment_status_replicas_available,19823
kube_deployment_status_replicas_ready,19823
kube_deployment_created,19823
kube_deployment_status_observed_generation,19823
kube_deployment_status_replicas,19823
newrelic.goldenmetrics.infra.kubernetes_deployment.podsAvailable,19250
newrelic.goldenmetrics.infra.kubernetes_deployment.podsDesired,19248
k8s.pod.network.io,18494
kube_deployment_spec_strategy_rollingupdate_max_surge,18446
container.cpu.usage,16665
k8s.pod.filesystem.available,14304
k8s.pod.memory.working_set,14304
k8s.pod.filesystem.capacity,14304
k8s.pod.filesystem.usage,14304
system.filesystem.usage,14080
container_cpu_cfs_periods_total,13163
container_cpu_cfs_throttled_periods_total,13140
kube_persistentvolume_status_phase,12890
system.disk.operation_time,10212
system.disk.operations,10212
kube_node_status_allocatable,9942
kube_node_status_condition,8445
kube_daemonset_status_updated_number_scheduled,8444
kube_daemonset_status_number_ready,8444
kube_daemonset_status_current_number_scheduled,8444
kube_daemonset_status_number_misscheduled,8444
kube_daemonset_status_number_available,8444
kube_daemonset_created,8444
kube_daemonset_status_desired_number_scheduled,8444
kube_daemonset_status_number_unavailable,8444
newrelic.goldenmetrics.infra.kubernetes_daemonset.podsDesired,8216
newrelic.goldenmetrics.infra.kubernetes_daemonset.podsReady,8215
system.filesystem.utilization,7040
system.disk.io_time,5106
system.cpu.utilization,4475
system.memory.usage,4475
kube_statefulset_status_replicas_updated,3234
kube_statefulset_created,3234
kube_statefulset_persistentvolumeclaim_retention_policy,3234
kube_statefulset_status_current_revision,3234
kube_statefulset_status_replicas_available,3234
kube_statefulset_status_replicas,3234
kube_statefulset_replicas,3234
kube_statefulset_status_replicas_ready,3234
kube_statefulset_status_replicas_current,3234
newrelic.goldenmetrics.infra.kubernetes_statefulset.podsDesired,3127
newrelic.goldenmetrics.infra.kubernetes_statefulset.podsReady,3127
kube_persistentvolume_capacity_bytes,2578
kube_persistentvolume_created,2578
kube_persistentvolume_info,2573
newrelic.goldenmetrics.infra.kubernetes_persistentvolume.requestedStorageBytes,2513
newrelic.goldenmetrics.infra.kubernetes_persistentvolume.createdAt,2513
kube_persistentvolumeclaim_resource_requests_storage_bytes,2098
kube_persistentvolumeclaim_created,2098
kube_persistentvolumeclaim_status_phase,2098
kube_persistentvolumeclaim_info,2096
process_resident_memory_bytes,2089
go_threads,2089
go_goroutines,2089
newrelic.goldenmetrics.infra.kubernetes_persistentvolumeclaim.requestedStorageBytes,2045
newrelic.goldenmetrics.infra.kubernetes_persistentvolumeclaim.createdAt,2045
system.memory.utilization,1790
system.network.errors,1760
system.network.packets,1760
system.network.io,1760
system.cpu.load_average.5m,895
system.cpu.load_average.1m,895
system.cpu.load_average.15m,895
k8s.cluster.info,835
k8s.node.filesystem.capacity,537
k8s.node.memory.available,537
k8s.node.filesystem.usage,537
k8s.node.memory.working_set,537
k8s.node.cpu.time,528
newrelic.resourceConsumption.limitValue,491
newrelic.resourceConsumption.currentValue,491
newrelic.resourceConsumption.impact,491
kube_cronjob_spec_suspend,479
kube_cronjob_status_active,479
kube_cronjob_status_last_schedule_time,479
kube_cronjob_created,479
newrelic.goldenmetrics.infra.kubernetes_cronjob.lastScheduled,466
newrelic.goldenmetrics.infra.kubernetes_cronjob.isActive,466
newrelic.goldenmetrics.infra.kubernetes_cronjob.isSuspended,466
newrelic.goldenmetrics.infra.host.memoryUsage,413
newrelic.goldenmetrics.infra.host.storageUsage,413
newrelic.goldenmetrics.infra.host.cpuUsage,413
newrelic.goldenmetrics.infra.host.networkTrafficTx,406
newrelic.goldenmetrics.infra.host.networkTrafficRx,406
k8s.node.cpu.usage,357
kube_pod_container_status_waiting_reason,238
kube_service_status_load_balancer_ingress,180
````

#### Which issue this PR fixes
*(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)`
format, will close that issue when PR gets merged)*
  - fixes #

#### Special notes for your reviewer:

#### Checklist
[Place an '[x]' (no spaces) in all applicable fields. Please remove
unrelated fields.]
- [x] Chart Version bumped
- [x] Variables are documented in the README.md
- [x] Title of the PR starts with chart name (e.g. `[mychartname]`)

---------

Co-authored-by: Daniel Budziwojski <dbudziwojski@newrelic.com>

Author: ramkrishankumarN

charts/nr-k8s-otel-collector/Chart.yaml

@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.8.23
+version: 0.8.24
 
 dependencies:
   - name: common-library

charts/nr-k8s-otel-collector/examples/k8s/rendered/clusterrole.yaml

@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 rules:
   - apiGroups:
     - ""

charts/nr-k8s-otel-collector/examples/k8s/rendered/clusterrolebinding.yaml

@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 subjects:
   - kind: ServiceAccount
     name: example-nr-k8s-otel-collector

charts/nr-k8s-otel-collector/examples/k8s/rendered/daemonset-configmap.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 data:
   daemonset-config.yaml: |
     receivers:
@@ -461,6 +461,15 @@ data:
           datapoint:
             - 'IsMatch(metric.name, "^system.network.*") == true and attributes["device"] == "lo"'
 
+      filter/nr_exclude_container_zero_values:
+        metrics:
+          datapoint:
+            - metric.name == "container_network_receive_errors_total" and value_double < 0.5
+            - metric.name == "container_network_transmit_errors_total" and value_double < 0.5
+            - metric.name == "container_network_transmit_bytes_total" and value_double < 0.5
+            - metric.name == "container_network_receive_bytes_total" and value_double < 0.5
+
+
       attributes/exclude_system_paging:
         include:
           match_type: strict
@@ -497,7 +506,7 @@ data:
             value: 'true'
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.8.23
+            value: 0.8.24
           - key: service.name
             action: delete
           - key: service_name
@@ -659,6 +668,7 @@ data:
             - metricstransform/cadvisor
             - metricstransform/kubelet
             - filter/exclude_metrics_low_data_mode
+            - filter/nr_exclude_container_zero_values
             - transform/truncate
             - resourcedetection/env
             - resourcedetection/cloudproviders

charts/nr-k8s-otel-collector/examples/k8s/rendered/daemonset.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 spec:
   selector:
     matchLabels:
@@ -24,7 +24,7 @@ spec:
         app.kubernetes.io/name: nr-k8s-otel-collector
         component: daemonset
       annotations:
-        checksum/config: 91b1fb70ef5f0e77932c1486c6b1ef931843988237ab206bc6c27e199e62f155
+        checksum/config: d113561eb77f9c64074829fd99381305062fa05fb56a5f2f40ac7ef5221de28e
     spec:
       serviceAccountName: example-nr-k8s-otel-collector
       containers:

charts/nr-k8s-otel-collector/examples/k8s/rendered/deployment-configmap.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 data:
   deployment-config.yaml: |
     receivers:
@@ -438,10 +438,24 @@ data:
         metrics:
           datapoint:
             - metric.name == "kube_pod_status_phase" and value_double < 0.5
+            - metric.name == "kube_pod_status_ready" and value_double < 0.5
             - metric.name == "kube_pod_status_scheduled" and value_double < 0.5
+            - metric.name == "kube_pod_container_status_ready" and value_double < 0.5
             - metric.name == "kube_pod_container_status_phase" and value_double < 0.5
-            - metric.name == "kube_pod_status_ready" and value_double < 0.5
+            - metric.name == "kube_pod_container_status_restarts_total" and value_double < 0.5
             - metric.name == "kube_deployment_status_condition" and value_double < 0.5
+            - metric.name == "kube_pod_container_status_waiting_reason" and value_double < 0.5
+
+
+      filter/nr_exclude_zero_value_kube_jobs:
+        metrics:
+          datapoint:
+            - metric.name == "kube_job_complete" and value_double < 0.5
+            - metric.name == "kube_job_spec_parallelism" and value_double < 0.5
+            - metric.name == "kube_job_status_failed" and value_double < 0.5
+            - metric.name == "kube_job_status_active" and value_double < 0.5
+            - metric.name == "kube_job_status_succeeded" and value_double < 0.5
+
 
       resourcedetection/env:
         detectors: ["env", "system"]
@@ -472,7 +486,7 @@ data:
             value: 'true'
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.8.23
+            value: 0.8.24
           - key: service.name
             action: delete
           - key: service_name
@@ -497,7 +511,7 @@ data:
             value: 'true'
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.8.23
+            value: 0.8.24
 
       transform/events:
         log_statements:
@@ -695,6 +709,7 @@ data:
             - metricstransform/k8s_cluster_info_ldm
             - metricstransform/ksm
             - filter/exclude_metrics_low_data_mode
+            - filter/nr_exclude_zero_value_kube_jobs
             - transform/low_data_mode_inator
             - resource/low_data_mode_inator
             - resource/metrics

charts/nr-k8s-otel-collector/examples/k8s/rendered/deployment.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 spec:
   replicas: 1
   minReadySeconds: 5
@@ -26,7 +26,7 @@ spec:
         app.kubernetes.io/name: nr-k8s-otel-collector
         component: deployment
       annotations:
-        checksum/config: 6ead1f562807e8c3bb9e9ca5547f3e19ab5acc14c1b7a84c5405fa8e62358b77
+        checksum/config: 08ca9977298906fbf26b40b96afa478d37146269422df769606909a65d258bb8
     spec:
       serviceAccountName: example-nr-k8s-otel-collector
       containers:

charts/nr-k8s-otel-collector/examples/k8s/rendered/secret.yaml

@@ -10,6 +10,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 data:
   licenseKey: PE5SX2xpY2Vuc2VLZXk+

charts/nr-k8s-otel-collector/examples/k8s/rendered/service.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
 spec:
   type: ClusterIP
   ports:

charts/nr-k8s-otel-collector/examples/k8s/rendered/serviceaccount.yaml

@@ -10,5 +10,5 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.0.3
-    helm.sh/chart: nr-k8s-otel-collector-0.8.23
+    helm.sh/chart: nr-k8s-otel-collector-0.8.24
   annotations: