Skip to content

Commit 0727b91

Browse files
jdollejdolle
authored
Track composition worker memory (#8256)
1 parent 8c6db14 commit 0727b91

7 files changed

Lines changed: 351 additions & 16 deletions

File tree

‎.changeset/modern-badgers-fall.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'hive': patch
3+
---
4+
5+
Track schema composition worker memory usage; add to schema service dashboard

‎deployment/grafana-dashboards/Schema-Service.json‎

Lines changed: 280 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -661,9 +661,288 @@
661661
],
662662
"title": "Timeouts",
663663
"type": "piechart"
664+
},
665+
{
666+
"datasource": {
667+
"type": "prometheus",
668+
"uid": "PROM_DATASOURCE_UID"
669+
},
670+
"description": "Current memory usage across workers",
671+
"fieldConfig": {
672+
"defaults": {
673+
"color": {
674+
"mode": "palette-classic"
675+
},
676+
"custom": {
677+
"axisCenteredZero": false,
678+
"axisColorMode": "text",
679+
"axisLabel": "",
680+
"axisPlacement": "auto",
681+
"axisShow": false,
682+
"barAlignment": 0,
683+
"drawStyle": "line",
684+
"fillOpacity": 10,
685+
"gradientMode": "none",
686+
"hideFrom": {
687+
"legend": false,
688+
"tooltip": false,
689+
"viz": false
690+
},
691+
"insertNulls": false,
692+
"lineInterpolation": "linear",
693+
"lineWidth": 1,
694+
"pointSize": 5,
695+
"scaleDistribution": {
696+
"type": "linear"
697+
},
698+
"showPoints": "auto",
699+
"spanNulls": false,
700+
"stacking": {
701+
"group": "A",
702+
"mode": "none"
703+
},
704+
"thresholdsStyle": {
705+
"mode": "off"
706+
}
707+
},
708+
"mappings": [],
709+
"thresholds": {
710+
"mode": "absolute",
711+
"steps": [
712+
{
713+
"color": "green",
714+
"value": null
715+
}
716+
]
717+
},
718+
"unit": "bytes"
719+
},
720+
"overrides": []
721+
},
722+
"gridPos": {
723+
"h": 8,
724+
"w": 12,
725+
"x": 0,
726+
"y": 32
727+
},
728+
"id": 9,
729+
"options": {
730+
"legend": {
731+
"calcs": [],
732+
"displayMode": "list",
733+
"placement": "bottom",
734+
"showLegend": true
735+
},
736+
"tooltip": {
737+
"mode": "single",
738+
"sort": "none"
739+
}
740+
},
741+
"targets": [
742+
{
743+
"datasource": {
744+
"type": "prometheus",
745+
"uid": "PROM_DATASOURCE_UID"
746+
},
747+
"editorMode": "code",
748+
"expr": "sum(composition_worker_memory_used_bytes) by (instance)",
749+
"instant": false,
750+
"legendFormat": "{{instance}}",
751+
"range": true,
752+
"refId": "A"
753+
}
754+
],
755+
"title": "Worker Memory Used",
756+
"type": "timeseries"
757+
},
758+
{
759+
"datasource": {
760+
"type": "prometheus",
761+
"uid": "PROM_DATASOURCE_UID"
762+
},
763+
"description": "P95 of composition cache value size",
764+
"fieldConfig": {
765+
"defaults": {
766+
"color": {
767+
"mode": "palette-classic"
768+
},
769+
"custom": {
770+
"axisCenteredZero": false,
771+
"axisColorMode": "text",
772+
"axisLabel": "",
773+
"axisPlacement": "auto",
774+
"axisShow": false,
775+
"barAlignment": 0,
776+
"drawStyle": "line",
777+
"fillOpacity": 10,
778+
"gradientMode": "none",
779+
"hideFrom": {
780+
"legend": false,
781+
"tooltip": false,
782+
"viz": false
783+
},
784+
"insertNulls": false,
785+
"lineInterpolation": "linear",
786+
"lineWidth": 1,
787+
"pointSize": 5,
788+
"scaleDistribution": {
789+
"type": "linear"
790+
},
791+
"showPoints": "auto",
792+
"spanNulls": false,
793+
"stacking": {
794+
"group": "A",
795+
"mode": "none"
796+
},
797+
"thresholdsStyle": {
798+
"mode": "off"
799+
}
800+
},
801+
"mappings": [],
802+
"thresholds": {
803+
"mode": "absolute",
804+
"steps": [
805+
{
806+
"color": "green",
807+
"value": null
808+
}
809+
]
810+
},
811+
"unit": "bytes"
812+
},
813+
"overrides": []
814+
},
815+
"gridPos": {
816+
"h": 8,
817+
"w": 12,
818+
"x": 12,
819+
"y": 32
820+
},
821+
"id": 10,
822+
"options": {
823+
"legend": {
824+
"calcs": [],
825+
"displayMode": "list",
826+
"placement": "bottom",
827+
"showLegend": true
828+
},
829+
"tooltip": {
830+
"mode": "single",
831+
"sort": "none"
832+
}
833+
},
834+
"targets": [
835+
{
836+
"datasource": {
837+
"type": "prometheus",
838+
"uid": "PROM_DATASOURCE_UID"
839+
},
840+
"editorMode": "code",
841+
"expr": "histogram_quantile(0.95, sum(rate(composition_cache_value_size_bytes_bucket[$__rate_interval])) by (le))",
842+
"instant": false,
843+
"legendFormat": "P95 Size",
844+
"range": true,
845+
"refId": "A"
846+
}
847+
],
848+
"title": "Cache Value Size (P95)",
849+
"type": "timeseries"
850+
},
851+
{
852+
"datasource": {
853+
"type": "prometheus",
854+
"uid": "PROM_DATASOURCE_UID"
855+
},
856+
"description": "P95 of worker duration",
857+
"fieldConfig": {
858+
"defaults": {
859+
"color": {
860+
"mode": "palette-classic"
861+
},
862+
"custom": {
863+
"axisCenteredZero": false,
864+
"axisColorMode": "text",
865+
"axisLabel": "",
866+
"axisPlacement": "auto",
867+
"axisShow": false,
868+
"barAlignment": 0,
869+
"drawStyle": "line",
870+
"fillOpacity": 10,
871+
"gradientMode": "none",
872+
"hideFrom": {
873+
"legend": false,
874+
"tooltip": false,
875+
"viz": false
876+
},
877+
"insertNulls": false,
878+
"lineInterpolation": "linear",
879+
"lineWidth": 1,
880+
"pointSize": 5,
881+
"scaleDistribution": {
882+
"type": "linear"
883+
},
884+
"showPoints": "auto",
885+
"spanNulls": false,
886+
"stacking": {
887+
"group": "A",
888+
"mode": "none"
889+
},
890+
"thresholdsStyle": {
891+
"mode": "off"
892+
}
893+
},
894+
"mappings": [],
895+
"thresholds": {
896+
"mode": "absolute",
897+
"steps": [
898+
{
899+
"color": "green",
900+
"value": null
901+
}
902+
]
903+
},
904+
"unit": "ms"
905+
},
906+
"overrides": []
907+
},
908+
"gridPos": {
909+
"h": 8,
910+
"w": 24,
911+
"x": 0,
912+
"y": 40
913+
},
914+
"id": 11,
915+
"options": {
916+
"legend": {
917+
"calcs": [],
918+
"displayMode": "list",
919+
"placement": "bottom",
920+
"showLegend": true
921+
},
922+
"tooltip": {
923+
"mode": "single",
924+
"sort": "none"
925+
}
926+
},
927+
"targets": [
928+
{
929+
"datasource": {
930+
"type": "prometheus",
931+
"uid": "PROM_DATASOURCE_UID"
932+
},
933+
"editorMode": "code",
934+
"expr": "histogram_quantile(0.95, sum(rate(composition_worker_duration_ms_bucket[$__rate_interval])) by (le))",
935+
"instant": false,
936+
"legendFormat": "P95 Duration",
937+
"range": true,
938+
"refId": "A"
939+
}
940+
],
941+
"title": "Worker Duration (P95)",
942+
"type": "timeseries"
664943
}
665944
],
666-
"refresh": "5s",
945+
"refresh": "30s",
667946
"schemaVersion": 38,
668947
"tags": [],
669948
"templating": {

‎deployment/services/schema.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ export function deploySchema({
4343
SCHEMA_CACHE_TTL_MS: '65000' /* 65s */,
4444
SCHEMA_CACHE_SUCCESS_TTL_MS: String(hourInMS * 2),
4545
COMPOSITION_WORKER_MAX_OLD_GENERATION_SIZE_MB: '716',
46+
COMPOSITION_WORKER_TRACK_MEMORY_USAGE: '1',
4647
OPENTELEMETRY_COLLECTOR_ENDPOINT:
4748
observability.enabled && observability.tracingEndpoint
4849
? observability.tracingEndpoint

‎packages/services/schema/src/composition-scheduler.ts‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,22 @@ import { Worker } from 'node:worker_threads';
33
import fastq from 'fastq';
44
import * as Sentry from '@sentry/node';
55
import { registerWorkerLogging, type Logger } from '../../api/src/modules/shared/providers/logger';
6-
import type { CompositionEvent, CompositionResultEvent } from './composition-worker';
6+
import type {
7+
CompositionEvent,
8+
CompositionResultEvent,
9+
ErrorResultEvent,
10+
TrackMetricEvent,
11+
} from './composition-worker';
712
import {
813
compositionQueueDurationMS,
914
compositionTotalDurationMS,
1015
compositionWorkerDurationMS,
16+
compositionWorkerMemoryUsedBytes,
1117
} from './metrics';
1218

1319
type WorkerRunArgs = {
1420
data: CompositionEvent['data'];
21+
targetId?: string;
1522
requestId: string;
1623
abortSignal: AbortSignal;
1724
};
@@ -126,18 +133,24 @@ export class CompositionScheduler {
126133

127134
registerWorkerLogging(this.logger, worker, name);
128135

129-
worker.on(
130-
'message',
131-
(data: CompositionResultEvent | { event: 'error'; id: string; err: Error }) => {
132-
if (data.event === 'error') {
133-
workerState?.task.reject(data.err);
134-
}
136+
worker.on('message', (data: CompositionResultEvent | ErrorResultEvent | TrackMetricEvent) => {
137+
if (data.event === 'error') {
138+
workerState?.task.reject(data.err);
139+
}
140+
141+
if (data.event === 'compositionResult') {
142+
workerState?.task.resolve(data);
143+
}
135144

136-
if (data.event === 'compositionResult') {
137-
workerState?.task.resolve(data);
145+
if (data.event === 'metric') {
146+
if (data.type === 'heapUsed') {
147+
compositionWorkerMemoryUsedBytes.set(
148+
{ target: workerState?.args.targetId, type: workerState?.args.data.type },
149+
data.value,
150+
);
138151
}
139-
},
140-
);
152+
}
153+
});
141154

142155
const { logger: baseLogger } = this;
143156

@@ -181,6 +194,7 @@ export class CompositionScheduler {
181194
event: 'composition',
182195
id: taskId,
183196
data: args.data,
197+
targetId: args.targetId,
184198
taskId,
185199
requestId: args.requestId,
186200
} satisfies CompositionEvent);

0 commit comments

Comments
 (0)