Merge pull request #2530 from balena-os/async-os-updates

Async os updates

Author: pipex

docker-compose.test.yml

@@ -20,7 +20,7 @@ services:
       - mock-systemd
     volumes:
       - dbus:/shared/dbus
-      - tmp:/mnt/root/tmp/balena-supervisor/services
+      - services:/mnt/root/tmp/balena-supervisor/services
       - ./test/data/root:/mnt/root
       - ./test/data/root/mnt/boot:/mnt/boot
       - ./test/lib/wait-for-it.sh:/wait-for-it.sh
@@ -78,7 +78,7 @@ services:
     stop_grace_period: 3s
     volumes:
       - dbus:/shared/dbus
-      - tmp:/mnt/root/tmp/balena-supervisor/services
+      - services:/mnt/root/tmp/balena-supervisor/services
     # Set required supervisor configuration variables here
     environment:
       DOCKER_HOST: tcp://docker:2375
@@ -110,7 +110,7 @@ volumes:
     driver_opts:
       type: tmpfs
       device: tmpfs
-  tmp:
+  services:
     driver_opts:
       type: tmpfs
       device: tmpfs

docker-compose.yml

@@ -1,5 +1,8 @@
 version: '2.3'
 
+x-helios-image: &helios-image
+  image: ghcr.io/balena-io/helios:0.25.21
+
 services:
   # This service can only be named `main`, `balena-supervisor` or `core`
   # in a multi-container supervisor, the API `supervisor_release` needs to be able to
@@ -8,10 +11,81 @@ services:
   core:
     build: ./
     privileged: true
-    tty: true
+    tty: false
     restart: always
     network_mode: host
     labels:
       io.balena.features.balena-api: '1'
       io.balena.features.dbus: '1'
       io.balena.features.balena-socket: '1'
+
+  # This is the next generation supervisor service. When started, it becomes a proxy
+  # for calls between the existing supervisor and the balenaAPI and containers
+  core-next:
+    <<: *helios-image
+    restart: on-failure
+    # required to distinguish between stderr/stdout, the supervisor defaults to true
+    tty: false
+    labels:
+      # Needed to get access to supervisor api variables
+      # and credentials
+      io.balena.features.supervisor-api: '1'
+      # Needed for access to DOCKER_HOST
+      io.balena.features.balena-socket: '1'
+      # Needed to stop the supervisor service
+      io.balena.features.dbus: '1'
+      # Needed to get access to the API credentials
+      io.balena.features.balena-api: '1'
+      # Needed to get Host OS metadata
+      io.balena.features.host-os.board-rev: '1'
+    environment:
+      - DBUS_SYSTEM_BUS_ADDRESS=unix:path=/host/run/dbus/system_bus_socket
+      # Needed for locks and balenahup. This needs to match the device
+      # used for the `runtime` volume
+      - BALENA_HOST_RUNTIME_DIR=/tmp/balena-supervisor
+      # Only send error logs to dashboard
+      - BALENA_LOGS_LEVEL=error
+    volumes:
+      # Critical configurations
+      - config:/config/helios
+      # Request caching
+      - cache:/cache/helios
+      # Persistent system state data that can be reconstructed from
+      # the target state
+      - state:/local/helios
+      # Runtime directory. Data here does not persist after reboots
+      - runtime:/tmp/run
+
+  # The service below relays requests between tcp port 48484 and the socket exposed by
+  # the core-next service
+  service-relay:
+    <<: *helios-image
+    command: socat TCP-LISTEN:48484,reuseaddr,fork UNIX-CONNECT:/tmp/run/helios.sock
+    # required to distinguish between stderr/stdout, the supervisor defaults to true
+    tty: false
+    ports:
+      - 48484
+    depends_on:
+      - core
+    volumes:
+      - runtime:/tmp/run
+    environment:
+      # Only send error logs to dashboard
+      - BALENA_LOGS_LEVEL=error
+    healthcheck:
+      test: ['CMD', 'wget', '-O', '-', '-q', 'http://127.0.0.1:48484/ping']
+
+volumes:
+  config:
+  state:
+    labels:
+      # the database version, can be bumped to re-create the state
+      io.balena.private.db-version: 0
+  cache:
+  runtime:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      # XXX: this directory needs to exist on the host before the container is started
+      device: /tmp/balena-supervisor

src/compose/service-manager.ts

@@ -372,7 +372,7 @@ export async function start(service: Service) {
 			);
 		}
 
-		logger.attach(container.id, { serviceId });
+		logger.attach(container.id, { serviceId, logLevel: service.logLevel() });
 
 		if (!alreadyStarted) {
 			logger.logSystemEvent(LogTypes.startServiceSuccess, { service });
@@ -433,8 +433,10 @@ export function listenToEvents() {
 										`serviceId not defined for service: ${service.serviceName} in ServiceManager.listenToEvents`,
 									);
 								}
+
 								logger.attach(data.id, {
 									serviceId,
+									logLevel: service.logLevel(),
 								});
 							} else if (status === 'destroy') {
 								logMonitor.detach(data.id);
@@ -489,8 +491,10 @@ export async function attachToRunning() {
 					`containerId not defined for service: ${service.serviceName} in ServiceManager.attachToRunning`,
 				);
 			}
+
 			logger.attach(service.containerId, {
 				serviceId,
+				logLevel: service.logLevel(),
 			});
 		}
 	}

src/compose/service.ts

@@ -14,6 +14,7 @@ import log from '../lib/supervisor-console';
 import * as conversions from '../lib/conversions';
 import { checkInt } from '../lib/validation';
 import { InternalInconsistencyError } from '../lib/errors';
+import { LogLevel } from '../logging/types';
 
 import type { EnvVarObject } from '../types';
 import type {
@@ -1185,6 +1186,18 @@ class ServiceImpl implements Service {
 			`${updateLock.lockPath(appId, serviceName)}:/tmp/balena`,
 		];
 	}
+
+	public logLevel(): LogLevel {
+		switch (this.config.environment['BALENA_LOGS_LEVEL']) {
+			case 'none':
+				return LogLevel.None;
+			case 'err':
+			case 'error':
+				return LogLevel.Err;
+			default:
+				return LogLevel.Info;
+		}
+	}
 }
 
 export const Service = ServiceImpl;

src/compose/types/service.ts

@@ -3,6 +3,7 @@ import type Dockerode from 'dockerode';
 import { isAbsolute } from 'path';
 
 import type { PortMap } from '../ports';
+import type { LogLevel } from '../../logging/types';
 
 export interface ComposeHealthcheck {
 	test: string | string[];
@@ -392,4 +393,5 @@ export interface Service {
 		containerIds: Dictionary<string>;
 	}): Dockerode.ContainerCreateOptions;
 	handoverCompleteFullPathsOnHost(): string[];
+	logLevel(): LogLevel;
 }

src/lib/supervisor-metadata.ts

@@ -18,10 +18,6 @@ const SUPERVISOR_APPS: { [arch: string]: string } = {
 	rpi: '6822565f766e413e96d9bebe2227cdcc',
 };
 
-export function isSupervisorApp(appUuid: string): boolean {
-	return Object.values(SUPERVISOR_APPS).some((uuid) => uuid === appUuid);
-}
-
 /**
  * Check if the supervisor in the target state belongs to the known
  * supervisors

src/logging/index.ts

@@ -6,7 +6,7 @@ import type { LogType } from '../lib/log-types';
 import { BalenaLogBackend } from './balena-backend';
 import { LocalLogBackend } from './local-backend';
 import type { LogBackend } from './log-backend';
-import type { LogMessage } from './types';
+import { LogLevel, type LogMessage } from './types';
 import logMonitor from './monitor';
 
 import * as globalEventBus from '../event-bus';
@@ -126,18 +126,31 @@ export function logSystemMessage(
 	}
 }
 
-type ServiceInfo = { serviceId: number };
-export function attach(containerId: string, { serviceId }: ServiceInfo): void {
+type ServiceInfo = { serviceId: number; logLevel: LogLevel };
+export function attach(
+	containerId: string,
+	{ serviceId, logLevel }: ServiceInfo,
+): void {
 	// First detect if we already have an attached log stream
 	// for this container
 	if (logMonitor.isAttached(containerId)) {
 		return;
 	}
 
+	// skip service attach LogLevel is None
+	if (logLevel === LogLevel.None) {
+		return;
+	}
+
 	// We do not grab a container lock here as we are only adding it to the list of container ids to monitor
 	// and not directly affecting the container itself, and grabbing the lock can cause a delay which means
 	// that early messages logged from the container are missed and never sent to the backend.
 	logMonitor.attach(containerId, async (message) => {
+		// ignore non stderr messages for an error only service
+		if (logLevel === LogLevel.Err && !message.isStdErr) {
+			return;
+		}
+
 		await log({ ...message, serviceId });
 	});
 }

src/logging/types.ts

@@ -11,3 +11,9 @@ type ContainerLogMessage = BaseLogMessage & {
 	isSystem?: false;
 };
 export type LogMessage = SystemLogMessage | ContainerLogMessage;
+
+export enum LogLevel {
+	None,
+	Info,
+	Err,
+}

src/supervisor.ts

@@ -19,10 +19,6 @@ import * as firewall from './lib/firewall';
 import * as constants from './lib/constants';
 import * as uname from './lib/uname';
 
-// FIXME: remove when we release supervisor v19
-import * as servicesManager from './compose/service-manager';
-import * as supervisorMetadata from './lib/supervisor-metadata';
-
 const startupConfigFields: config.ConfigKey[] = [
 	'uuid',
 	'listenPort',
@@ -36,42 +32,6 @@ const startupConfigFields: config.ConfigKey[] = [
 	'legacyAppsPresent',
 ];
 
-// This function removes any `helios` overrides in order to restore
-// the supervisor functionality as the only service.
-//
-// - It removes the port/endpoint overrides
-// - It stops helios
-//
-// Returns true if the changes were applied
-//
-// FIXME: remove when we release supervisor v19
-async function recoverFromOverrides() {
-	const { listenPortOverride, apiEndpointOverride } = await config.getMany([
-		'listenPortOverride',
-		'apiEndpointOverride',
-	]);
-	if (listenPortOverride != null || apiEndpointOverride != null) {
-		// stop helios-api
-		const [heliosApi] = (
-			await servicesManager.getAll(`service-name=helios-api`)
-		).filter(
-			(svc) =>
-				svc.appUuid != null && supervisorMetadata.isSupervisorApp(svc.appUuid),
-		);
-		if (heliosApi != null) {
-			await servicesManager.kill(heliosApi, { wait: true });
-		}
-
-		log.info('Removing supervisor overrides');
-		await db.models('config').del().where({ key: 'listenPortOverride' });
-		await db.models('config').del().where({ key: 'apiEndpointOverride' });
-
-		// terminate the process
-		log.info('restarting');
-		process.exit(1);
-	}
-}
-
 export class Supervisor {
 	private api: SupervisorAPI;
 
@@ -80,10 +40,6 @@ export class Supervisor {
 
 		await db.initialized();
 		await config.initialized();
-
-		// FIXME: remove when we release supervisor v19
-		await recoverFromOverrides();
-
 		await avahi.initialized();
 		log.debug('Starting logging infrastructure');
 		await logger.initialized();