Skip to content

Commit de1cf82

Browse files
mrrr3dyanmxa
mrrr3d
and
yanmxa
authored
✨ Federated learning obs sidecar (open-cluster-management-io#62)
* Sidecar container config. Signed-off-by: mrrr61 <mrrr61@outlook.com> * Use unified name of collectors in managed clusters. Signed-off-by: mrrr61 <mrrr61@outlook.com> * Add fl_sidecar Signed-off-by: mrrr61 <mrrr61@outlook.com> * Update manifestwork and server-job's obs-sidecar config Signed-off-by: mrrr61 <mrrr61@outlook.com> * Flower applications export metrics. Signed-off-by: mrrr61 <mrrr61@outlook.com> * reorder Makefile rules for better readability. Signed-off-by: mrrr61 <mrrr61@outlook.com> * read jobname from flags Signed-off-by: mrrr61 <mrrr61@outlook.com> * use metric name without prefix "fl_training_" Signed-off-by: mrrr61 <mrrr61@outlook.com> * modify label name: k8s_namespace_name -> namespace, k8s_pod_name -> pod_name Signed-off-by: mrrr61 <mrrr61@outlook.com> * put round value to metrics label Signed-off-by: mrrr61 <mrrr61@outlook.com> * extract write_metrics functionality and update metrics handling Signed-off-by: mrrr61 <mrrr61@outlook.com> * get obs-sidecar name through annotation Signed-off-by: mrrr61 <mrrr61@outlook.com> * make Obs sidecar image optional Signed-off-by: mrrr61 <mrrr61@outlook.com> * add application level observability guide Signed-off-by: mrrr61 <mrrr61@outlook.com> * update observability configuration guide Signed-off-by: mrrr61 <mrrr61@outlook.com> * add some comment of sidecar Signed-off-by: mrrr61 <mrrr61@outlook.com> * update sidecar docker build process Signed-off-by: mrrr61 <mrrr61@outlook.com> * update sidecar image annotation to federated-learning.io/sidecar-image Signed-off-by: mrrr61 <mrrr61@outlook.com> * update namespace attribute key to pod.namespace Signed-off-by: mrrr61 <mrrr61@outlook.com> * update image name to federated-learning-sidecar Signed-off-by: mrrr61 <mrrr61@outlook.com> * remove internal/sidecar/README.md Signed-off-by: mrrr61 <mrrr61@outlook.com> * change metricfile to json Signed-off-by: mrrr61 <mrrr61@outlook.com> * use quay.io Signed-off-by: mrrr61 <mrrr61@outlook.com> * introduce constant for sidecar image annotation Signed-off-by: mrrr61 <mrrr61@outlook.com> * prevent panic when sidecar image annotation is nil Signed-off-by: mrrr61 <mrrr61@outlook.com> * add OTEL_ENDPOINT environment variable to client and server jobs Signed-off-by: mrrr61 <mrrr61@outlook.com> * implement support for metric labels Signed-off-by: mrrr61 <mrrr61@outlook.com> * improve metric handling and callback registration Signed-off-by: mrrr61 <mrrr61@outlook.com> * enhance metrics Reporter comments Signed-off-by: mrrr61 <mrrr61@outlook.com> * patch Signed-off-by: myan <myan@redhat.com> --------- Signed-off-by: mrrr61 <mrrr61@outlook.com> Signed-off-by: myan <myan@redhat.com> Co-authored-by: myan <myan@redhat.com>
1 parent 8a6001f commit de1cf82

21 files changed

Lines changed: 1225 additions & 8 deletions

File tree

‎federated-learning-controller/Makefile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
REGISTRY ?= quay.io/open-cluster-management
22
# Image URL to use all building/pushing image targets
33
IMG ?= $(REGISTRY)/federated-learning-controller:latest
4+
SIDECAR_IMG ?= $(REGISTRY)/federated-learning-sidecar:latest
45
# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
56
ENVTEST_K8S_VERSION = 1.31.0
67

@@ -121,6 +122,21 @@ docker-platform-build: ## Build docker image for multiple platforms (linux/amd64
121122
docker-platform-push: ## Build and push docker image for multiple platforms (linux/amd64,linux/arm64).
122123
$(CONTAINER_TOOL) buildx build --platform linux/amd64,linux/arm64 --push -t ${IMG} .
123124

125+
.PHONY: docker-sidecar-build
126+
docker-sidecar-build: ## Build sidecar docker image.
127+
$(CONTAINER_TOOL) build -t ${SIDECAR_IMG} -f internal/sidecar/Dockerfile internal/sidecar
128+
129+
.PHONY: docker-sidecar-push
130+
docker-sidecar-push: ## Push sidecar docker image.
131+
$(CONTAINER_TOOL) push ${SIDECAR_IMG}
132+
133+
.PHONY: docker-sidecar-platform-build
134+
docker-sidecar-platform-build: ## Build sidecar docker image for multiple platforms (linux/amd64,linux/arm64).
135+
$(CONTAINER_TOOL) buildx build --platform linux/amd64,linux/arm64 -t ${SIDECAR_IMG} -f internal/sidecar/Dockerfile internal/sidecar
136+
137+
.PHONY: docker-sidecar-platform-push
138+
docker-sidecar-platform-push: ## Build and push sidecar docker image for multiple platforms (linux/amd64,linux/arm64).
139+
$(CONTAINER_TOOL) buildx build --platform linux/amd64,linux/arm64 --push -t ${SIDECAR_IMG} -f internal/sidecar/Dockerfile internal/sidecar
124140
# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
125141
# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
126142
# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/

‎federated-learning-controller/api/v1alpha1/federatedlearning_types.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,10 @@ const (
5656
Other Framework = "other"
5757
)
5858

59+
const (
60+
AnnotationSidecarImage = "federated-learning.io/sidecar-image"
61+
)
62+
5963
// FederatedLearningSpec defines the desired state of FederatedLearning.
6064
type FederatedLearningSpec struct {
6165
// +kubebuilder:default=other
@@ -102,7 +106,7 @@ type ListenerSpec struct {
102106
// +kubebuilder:default:=8080
103107
Port int `json:"port,omitempty"`
104108
Type ListenerType `json:"type,omitempty"`
105-
109+
106110
// IP is the optional bind IP for NodePort services.
107111
// It is only applicable when Type is "NodePort".
108112
// +optional

‎federated-learning-controller/deploy/obs/hack/certs/generate-certs.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ kubectl --context kind-hub create secret tls otel-signer -n open-cluster-managem
3737

3838
# replace root-ca.crt in deploy/resources/addon-template.yaml
3939
awk '{print " " $0}' root-ca.crt > root-ca.crt.tmp
40-
# Cross-platform sed in-place editing
40+
# Cross-platform sed -i compatibility
4141
if [[ "$OSTYPE" == "darwin"* ]]; then
4242
sed -i '' "/PROM_WEB_ROOT_CA/{
4343
r root-ca.crt.tmp

‎federated-learning-controller/docs/configure-environment-observability.md‎

Lines changed: 217 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,4 +187,220 @@ $ curl -ksS https://hub-control-plane:30090/api/v1/query?query=kepler_container_
187187
]
188188
}
189189
}
190-
```
190+
```
191+
192+
## Application Level Observability
193+
194+
In the federated learning process, it is crucial to monitor application-level metrics to gain insights into the model's training performance.
195+
196+
### Background
197+
198+
The sidecar is designed to export application-level metrics, such as the accuracy and loss value during the training process. This allows for real-time monitoring and analysis of the federated learning tasks.
199+
200+
### Architecture
201+
202+
A sidecar container runs alongside the federated learning application. The application writes its metrics to a JSON file at `/metrics/metric.json`. The sidecar watches this file, parses the metrics, and exposes them to an OpenTelemetry (OTel) exporter, which then sends them to an observability backend.
203+
204+
### How to Export Metrics
205+
206+
To export metrics, the training application needs to write a JSON file with a specific schema to the path `/metrics/metric.json`.
207+
208+
**NOTE:**
209+
210+
- When updating the metrics each time, it will automatically add the `timestamp` field to the metrics. But if the `metric.json` file already contains a `timestamp` field, it will overwrite the value with the current timestamp.
211+
212+
To enable the sidecar, specify the `federated-learning.io/sidecar-image` annotation in the FederatedLearning resource with a valid image reference; if the annotation is omitted or set to an empty string, the sidecar will not be injected. For example enabling the sidecar for a FederatedLearning resource:
213+
214+
```yaml
215+
apiVersion: federation-ai.open-cluster-management.io/v1alpha1
216+
kind: FederatedLearning
217+
metadata:
218+
name: federated-learning-sample
219+
annotations:
220+
federated-learning.io/sidecar-image: quay.io/open-cluster-management/federated-learning-sidecar:latest
221+
spec:
222+
framework: flower
223+
server:
224+
image: quay.io/open-cluster-management/flower-app-torch:latest
225+
rounds: 3
226+
minAvailableClients: 2
227+
listeners:
228+
- name: server-listener
229+
port: 8080
230+
type: LoadBalancer
231+
storage:
232+
type: PersistentVolumeClaim
233+
name: model-pvc
234+
path: /data/models
235+
size: 2Gi
236+
client:
237+
image: quay.io/open-cluster-management/flower-app-torch:latest
238+
placement:
239+
clusterSets:
240+
- global
241+
predicates:
242+
- requiredClusterSelector:
243+
claimSelector:
244+
matchExpressions:
245+
- key: federated-learning-sample.client-data
246+
operator: Exists
247+
```
248+
249+
#### JSON Sample
250+
251+
Here is an example of the `metric.json` file format:
252+
253+
```json
254+
{
255+
"metrics": {
256+
"loss": 0.5,
257+
"accuracy": 0.8,
258+
},
259+
"labels": {
260+
"round": 1,
261+
},
262+
}
263+
```
264+
265+
#### Python Helper Function
266+
267+
For applications written in Python, you can leverage the following helper function to write metrics in the correct format.
268+
269+
```python
270+
import json
271+
import os
272+
from typing import Dict, Any
273+
274+
def write_metrics(metrics: Dict[str, Any] = None,
275+
labels: Dict[str, Any] = None,
276+
filepath: str = "/metrics/metric.json"):
277+
"""
278+
Writes dictionaries containing metrics and labels to a JSON file.
279+
280+
Args:
281+
metrics (Dict[str, Any]): A dictionary containing the metric values.
282+
label (Dict[str, Any]): A dictionary containing the label data.
283+
path (str): The full path to the output JSON file.
284+
"""
285+
286+
# default {} if None
287+
metrics = metrics or {}
288+
labels = labels or {}
289+
290+
# Combine label and metrics into a single dictionary
291+
data_to_write = {
292+
"metrics": metrics,
293+
"labels": labels,
294+
}
295+
296+
try:
297+
os.makedirs("/metrics", exist_ok=True)
298+
with open(filepath, "w", encoding="utf-8") as f:
299+
json.dump(data_to_write, f, ensure_ascii=False)
300+
print(f"Metrics written to {filepath}")
301+
except Exception as e:
302+
print("write json file error: ", e)
303+
304+
```
305+
This function takes the `metrics` and `filepath` as input parameters, which can write the metrics to the specified(default: `/metrics/metric.json`) file.
306+
307+
#### Metrics Result Sample
308+
309+
Here is an example of the metrics result:
310+
311+
```json
312+
$ curl -ksS 'https://172.18.0.2:30090/api/v1/query?query=accuracy' | jq
313+
{
314+
"status": "success",
315+
"data": {
316+
"resultType": "vector",
317+
"result": [
318+
{
319+
"metric": {
320+
"__name__": "accuracy",
321+
"cluster_name": "cluster1",
322+
"job": "federated-learning-obs-sidecar",
323+
"pod_name": "federated-learning-sample-client-jm7hl",
324+
"pod_namespace": "default",
325+
"round": "1",
326+
"service_name": "federated-learning-obs-sidecar",
327+
"telemetry_sdk_language": "go",
328+
"telemetry_sdk_name": "opentelemetry",
329+
"telemetry_sdk_version": "1.37.0"
330+
},
331+
"value": [
332+
1756022731.665,
333+
"0.9385"
334+
]
335+
},
336+
{
337+
"metric": {
338+
"__name__": "accuracy",
339+
"cluster_name": "local-cluster",
340+
"job": "federated-learning-obs-sidecar",
341+
"pod_name": "federated-learning-sample-server-t78z4",
342+
"pod_namespace": "default",
343+
"round": "1",
344+
"service_name": "federated-learning-obs-sidecar",
345+
"telemetry_sdk_language": "go",
346+
"telemetry_sdk_name": "opentelemetry",
347+
"telemetry_sdk_version": "1.37.0"
348+
},
349+
"value": [
350+
1756022731.665,
351+
"0.9385"
352+
]
353+
},
354+
{
355+
"metric": {
356+
"__name__": "accuracy",
357+
"cluster_name": "cluster2",
358+
"job": "federated-learning-obs-sidecar",
359+
"pod_name": "federated-learning-sample-client-xwwrj",
360+
"pod_namespace": "default",
361+
"round": "1",
362+
"service_name": "federated-learning-obs-sidecar",
363+
"telemetry_sdk_language": "go",
364+
"telemetry_sdk_name": "opentelemetry",
365+
"telemetry_sdk_version": "1.37.0"
366+
},
367+
"value": [
368+
1756022731.665,
369+
"0.9385"
370+
]
371+
}
372+
]
373+
}
374+
}
375+
```
376+
377+
### Building and Running a Custom Sidecar
378+
379+
If you need to customize the sidecar or build your own image, follow these steps.
380+
381+
#### Build the Image
382+
383+
Run the following command to build the Docker image. Before building and pushing, make sure to update the `REGISTRY` variable in the `internal/sidecar/Makefile` to your own container registry (e.g., `docker.io/username`).
384+
385+
```bash
386+
cd federated-learning-controller
387+
make docker-sidecar-build
388+
```
389+
390+
#### Push the Image
391+
392+
```bash
393+
make docker-sidecar-push
394+
```
395+
396+
#### Run the Container
397+
398+
Use the `docker run` command to start the sidecar. You must mount the metric file's directory into the container and provide the necessary arguments.
399+
400+
```bash
401+
docker run --rm \
402+
-v /path/on/host/to/metrics:/app/metrics \
403+
your-registry/federated-learning-sidecar:latest \
404+
-metricfile /metrics/metric.json \
405+
-endpoint host.docker.internal:4317
406+
```

‎federated-learning-controller/examples/flower/app-torch/app_torch/client_app.py‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from flwr.client import ClientApp, NumPyClient
88
from flwr.common import Context
99
from app_torch.task import Net, get_weights, load_data, set_weights, test, train
10+
from app_torch.utils import write_metrics
1011

1112

1213
# Define Flower Client and client_fn
@@ -32,6 +33,14 @@ def fit(self, parameters, config):
3233
# test_loss = test(model, device, test_loader)
3334
# test_losses.append(test_loss)
3435
loss = train(self.net, self.device, self.trainloader, optimizer, epoch)
36+
metrics = {
37+
"train_loss": loss,
38+
}
39+
labels = {
40+
"round": config["server_round"],
41+
"epoch": epoch,
42+
}
43+
write_metrics(metrics, labels)
3544
epoch_loss += loss
3645
scheduler.step()
3746

@@ -45,8 +54,16 @@ def evaluate(self, parameters, config):
4554
set_weights(self.net, parameters)
4655

4756
loss, accuracy = test(self.net, self.device, self.valloader)
48-
print(f"Loss: {loss}, Accuracy: {accuracy}")
49-
return loss, len(self.valloader.dataset), {"accuracy": accuracy}
57+
print(f"Evaluation Loss: {loss}, Accuracy: {accuracy}")
58+
metrics = {
59+
"loss": loss,
60+
"accuracy": accuracy,
61+
}
62+
labels = {
63+
"round": config["server_round"],
64+
}
65+
write_metrics(metrics, labels)
66+
return loss, len(self.valloader.dataset), {"accuracy": accuracy, "loss": loss}
5067

5168
def client_fn(context: Context):
5269
# Load model and data

‎federated-learning-controller/examples/flower/app-torch/app_torch/server_app.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from flwr.server.strategy import FedAvg
1515

1616
from app_torch.task import Net, get_weights
17-
from app_torch.utils import get_latest_model_file, load_model, save_model
17+
from app_torch.utils import get_latest_model_file, load_model, save_model, write_metrics
1818

1919

2020
# ------------------------------
@@ -40,8 +40,15 @@ def parse_arguments():
4040
def client_weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics:
4141
"""Compute weighted average of client accuracy."""
4242
accuracies = [num_examples * metric["accuracy"] for num_examples, metric in metrics]
43+
losses = [num_examples * metric["loss"] for num_examples, metric in metrics]
4344
examples = [num_examples for num_examples, _ in metrics]
44-
return {"accuracy": sum(accuracies) / sum(examples)}
45+
return {"accuracy": sum(accuracies) / sum(examples), "loss": sum(losses) / sum(examples)}
46+
47+
def fit_evaluate_config(server_round: int):
48+
config = {
49+
"server_round": server_round,
50+
}
51+
return config
4552

4653
def start_server(args):
4754
"""Starts the federated learning server with model aggregation and checkpointing."""
@@ -88,6 +95,16 @@ def aggregate_fit(self, rnd, results, failures):
8895

8996
return aggregated_parameters, aggregated_metrics
9097

98+
def aggregate_evaluate(self, server_round, results, failures):
99+
aggregated_loss, aggregated_metrics = super().aggregate_evaluate(server_round, results, failures)
100+
metrics = {
101+
"loss": aggregated_loss,
102+
"accuracy": aggregated_metrics["accuracy"],
103+
}
104+
labels = {"round": server_round}
105+
write_metrics(metrics, labels)
106+
return aggregated_loss, aggregated_metrics
107+
91108
# Start the FL server
92109
fl.server.start_server(
93110
server_address=args.server_address,
@@ -99,6 +116,8 @@ def aggregate_fit(self, rnd, results, failures):
99116
initial_parameters=initial_parameters,
100117
evaluate_metrics_aggregation_fn=client_weighted_average,
101118
inplace=True,
119+
on_fit_config_fn=fit_evaluate_config,
120+
on_evaluate_config_fn=fit_evaluate_config,
102121
),
103122
)
104123

0 commit comments

Comments
 (0)