Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 9 additions & 13 deletions src/commands/mirror-node.ts
Original file line number Diff line number Diff line change
Expand Up @@ -865,20 +865,10 @@ export class MirrorNodeCommand extends BaseCommand {
MirrorNodeCommand.MIRROR_ENVIRONMENT_VARIABLE_PREFIX,
);
},
skip: ({config}: MirrorNodeDeployContext): boolean =>
config.useExternalDatabase || !config.installSharedResources,
skip: ({config}: MirrorNodeDeployContext): boolean => config.useExternalDatabase,
};
}

/**
* Installs the mirror chart with all application components disabled in order to create the
* `mirror-passwords` secret. The init script (run by {@link initializeSharedPostgresDatabaseTask})
* reads that secret to obtain the DB user passwords, so the secret must exist before init runs.
* The importer must not be running during init (it would hold a session that blocks DROP DATABASE),
* so we use this lightweight prime install instead of a full chart install.
*
* Skipped when the secret already exists (upgrade path) or when using an external database.
*/
/**
* Deletes the `<release>-redis` secret so that the subsequent mirror chart install/upgrade
* re-creates it cleanly. This is necessary because Kubernetes strategic-merge-patch does not
Expand All @@ -899,6 +889,13 @@ export class MirrorNodeCommand extends BaseCommand {
};
}

/**
* Installs the mirror chart with all application components disabled in order to create the
* `mirror-passwords` secret. The init script (run by {@link initializeSharedPostgresDatabaseTask})
* reads that secret to obtain the DB user passwords, so the secret must exist before init runs.
* The importer must not be running during init (it would hold a session that blocks DROP DATABASE),
* so we use this lightweight prime install instead of a full chart install.
*/
private primePostgresSecretTask(): SoloListrTask<AnyListrContext> {
return {
title: 'Prime mirror-node postgres secret',
Expand Down Expand Up @@ -948,8 +945,7 @@ export class MirrorNodeCommand extends BaseCommand {
true,
);
},
skip: ({config}: MirrorNodeDeployContext): boolean =>
config.useExternalDatabase || !config.installSharedResources,
skip: ({config}: MirrorNodeDeployContext): boolean => config.useExternalDatabase,
};
}

Expand Down
4 changes: 2 additions & 2 deletions src/core/shared-resources/postgres.ts
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ export class PostgresSharedResource {
'# Check for the sentinel comment that marks a fully completed initialization.',
'# Using a DB comment means the sentinel survives pod restarts and is only written',
'# after init-postgres.sh completes successfully (see end of this script).',
`SENTINEL=$(psql -tc "SELECT obj_description(oid, 'pg_database') FROM pg_database WHERE datname = '${databaseName}'" 2>/dev/null | tr -d '[:space:]')`,
`SENTINEL=$(psql -tc "SELECT shobj_description(oid, 'pg_database') FROM pg_database WHERE datname = '${databaseName}'" 2>/dev/null | tr -d '[:space:]')`,
'if [[ "${SENTINEL}" == "solo-initialized" ]]; then',
` echo "Initialization sentinel found on database '${databaseName}' — already complete, skipping."`,
' exit 0',
Expand All @@ -221,7 +221,7 @@ export class PostgresSharedResource {
` echo "Partial initialization detected: database '${databaseName}' exists but no sentinel. Cleaning up for fresh initialization."`,
` psql -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${databaseName}' AND pid <> pg_backend_pid();" 2>/dev/null || true`,
` psql -c "DROP DATABASE IF EXISTS ${databaseName};"`,
` for role in mirror_graphql mirror_grpc mirror_importer mirror_api mirror_rest_java mirror_rosetta mirror_web3 ${ownerUsername}; do`,
` for role in mirror_graphql mirror_grpc mirror_importer mirror_api mirror_rest mirror_rest_java mirror_rosetta mirror_web3 ${ownerUsername}; do`,
' psql -c "DROP USER IF EXISTS ${role};" 2>/dev/null || true',
' done',
' psql -c "DROP ROLE IF EXISTS temporary_admin, readwrite, readonly;" 2>/dev/null || true',
Expand Down
60 changes: 59 additions & 1 deletion src/core/shared-resources/shared-resource-manager.ts
Comment thread
jan-milenkov marked this conversation as resolved.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cleanup rogue function exports, look for opportunity to make the code more readable, avoid unnecessary complexion

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated

Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,18 @@ type HelmValuesObject = Record<string, unknown>;
type HelmMapValue = Record<string, HelmChartValue>;
type HelmToleration = HelmMapValue;

const ROLE_SCHEDULING_KEY: string = 'solo.hashgraph.io/role';
const REDIS_ROLE_FALLBACK_PATHS: string[] = [
'postgresql.postgresql',
'postgresql.primary',
'importer',
'grpc',
'rest',
'restjava',
'web3',
'monitor',
];

interface SharedResourceSchedulingValues {
postgresNodeSelector: HelmMapValue;
postgresTolerations: HelmToleration[];
Expand Down Expand Up @@ -176,6 +188,8 @@ function mergeSchedulingValues(target: SharedResourceSchedulingValues, values: H
Object.assign(target.redisNodeSelector, getMapValue(values, `${path}.nodeSelector`));
addTolerations(target.redisTolerations, getTolerations(values, `${path}.tolerations`));
}

mergeRedisRoleScheduling(target, values);
}

function getMapValue(values: HelmValuesObject, path: string): HelmMapValue {
Expand Down Expand Up @@ -236,6 +250,50 @@ function addTolerations(target: HelmToleration[], tolerations: HelmToleration[])
}
}

function mergeRedisRoleScheduling(target: SharedResourceSchedulingValues, values: HelmValuesObject): void {
if (target.redisNodeSelector[ROLE_SCHEDULING_KEY] === undefined) {
const role: HelmChartValue | undefined = findRoleNodeSelector(values);
if (role !== undefined) {
target.redisNodeSelector[ROLE_SCHEDULING_KEY] = role;
}
}

if (!hasTolerationForKey(target.redisTolerations, ROLE_SCHEDULING_KEY)) {
const toleration: HelmToleration | undefined = findRoleToleration(values);
if (toleration) {
addTolerations(target.redisTolerations, [toleration]);
}
}
}

function findRoleNodeSelector(values: HelmValuesObject): HelmChartValue | undefined {
for (const path of REDIS_ROLE_FALLBACK_PATHS) {
const role: HelmChartValue | undefined = getMapValue(values, `${path}.nodeSelector`)[ROLE_SCHEDULING_KEY];
if (role !== undefined) {
return role;
}
}

return undefined;
}

function findRoleToleration(values: HelmValuesObject): HelmToleration | undefined {
for (const path of REDIS_ROLE_FALLBACK_PATHS) {
const toleration: HelmToleration | undefined = getTolerations(values, `${path}.tolerations`).find(
(candidate: HelmToleration): boolean => candidate.key === ROLE_SCHEDULING_KEY,
);
if (toleration) {
return toleration;
}
}

return undefined;
}

function hasTolerationForKey(tolerations: HelmToleration[], key: string): boolean {
return tolerations.some((toleration: HelmToleration): boolean => toleration.key === key);
}

function toChartValues(schedulingValues: SharedResourceSchedulingValues): HelmChartValues {
const chartValues: HelmChartValues = new HelmChartValues();

Expand All @@ -252,7 +310,7 @@ function toChartValues(schedulingValues: SharedResourceSchedulingValues): HelmCh

function addNodeSelectorChartValues(chartValues: HelmChartValues, path: string, nodeSelector: HelmMapValue): void {
for (const [key, value] of Object.entries(nodeSelector)) {
chartValues.setLiteral(`${path}.${escapeHelmPathSegment(key)}`, value);
chartValues.setString(`${path}.${escapeHelmPathSegment(key)}`, value);
}
}

Expand Down
5 changes: 5 additions & 0 deletions src/integration/helm/model/values.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@ export class HelmChartValues {
return this;
}

public setString(name: string, value: HelmChartValue): this {
this._arguments.push('--set-string', `${name}=${value}`);
return this;
}

public setFile(name: string, path: string): this {
this._arguments.push('--set-file', `${name}=${path}`);
return this;
Expand Down
60 changes: 60 additions & 0 deletions test/unit/commands/mirror-node.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import * as constants from '../../../src/core/constants.js';
import * as versions from '../../../version.js';
import {resetForTest} from '../../test-container.js';
import {HelmChartValues} from '../../../src/integration/helm/model/values.js';
import {type SoloListrTask} from '../../../src/types/index.js';

interface MirrorNodeMemoryOverrideConfig {
mirrorNodeVersion: string;
Expand All @@ -21,6 +22,22 @@ interface MirrorNodeCommandInternal {
hasMirrorNodeMemoryImprovements: boolean,
config: MirrorNodeMemoryOverrideConfig,
) => void;
initializeSharedPostgresDatabaseTask: () => SoloListrTask<MirrorNodeDatabaseTaskContext>;
primePostgresSecretTask: () => SoloListrTask<MirrorNodeDatabaseTaskContext>;
}

interface MirrorNodeDatabaseTaskContext {
config: {
useExternalDatabase: boolean;
installSharedResources: boolean;
};
}

type MirrorNodeDatabaseSkip = (context: MirrorNodeDatabaseTaskContext) => boolean;

function getSkipFunction(task: SoloListrTask<MirrorNodeDatabaseTaskContext>): MirrorNodeDatabaseSkip {
expect(task.skip).to.be.a('function');
return task.skip as MirrorNodeDatabaseSkip;
}

describe('MirrorNodeCommand unit tests', (): void => {
Expand Down Expand Up @@ -136,4 +153,47 @@ describe('MirrorNodeCommand unit tests', (): void => {
expect(valuesArguments).to.not.include(`web3.image.repository=${constants.MIRROR_NODE_OLD_IMAGE_REPO_ROOT}web3`);
expect(valuesArguments).to.include(`web3.resources.limits.memory=${constants.MIRROR_NODE_OLD_MEMORY_WEB3}`);
});

it('should run shared postgres initialization when shared resources already exist', (): void => {
const mirrorNodeCommandInternal: MirrorNodeCommandInternal =
mirrorNodeCommand as unknown as MirrorNodeCommandInternal;
const task: SoloListrTask<MirrorNodeDatabaseTaskContext> =
mirrorNodeCommandInternal.initializeSharedPostgresDatabaseTask();
const context: MirrorNodeDatabaseTaskContext = {
config: {
useExternalDatabase: false,
installSharedResources: false,
},
};

expect(getSkipFunction(task)(context)).to.equal(false);
});

it('should run postgres secret priming when shared resources already exist', (): void => {
const mirrorNodeCommandInternal: MirrorNodeCommandInternal =
mirrorNodeCommand as unknown as MirrorNodeCommandInternal;
const task: SoloListrTask<MirrorNodeDatabaseTaskContext> = mirrorNodeCommandInternal.primePostgresSecretTask();
const context: MirrorNodeDatabaseTaskContext = {
config: {
useExternalDatabase: false,
installSharedResources: false,
},
};

expect(getSkipFunction(task)(context)).to.equal(false);
});

it('should skip shared postgres tasks for external database deployments', (): void => {
const mirrorNodeCommandInternal: MirrorNodeCommandInternal =
mirrorNodeCommand as unknown as MirrorNodeCommandInternal;
const context: MirrorNodeDatabaseTaskContext = {
config: {
useExternalDatabase: true,
installSharedResources: false,
},
};

expect(getSkipFunction(mirrorNodeCommandInternal.initializeSharedPostgresDatabaseTask())(context)).to.equal(true);
expect(getSkipFunction(mirrorNodeCommandInternal.primePostgresSecretTask())(context)).to.equal(true);
});
});
21 changes: 21 additions & 0 deletions test/unit/core/shared-resources/postgres.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,27 @@ describe('PostgresSharedResource', (): void => {
expect(writtenContent).to.include('export OWNER_USERNAME=mirror_node_owner');
});

it('reads the initialization sentinel from the database shared-object comment', async (): Promise<void> => {
await postgres.initializeMirrorNode(namespace, context);

const wrapperArguments: string[] = writeFileSyncStub
.getCalls()
.find((call: SinonSpyCall<string[], string>): boolean => (call.args[0] as string).includes('run-init'))!.args;
const writtenContent: string = wrapperArguments[1] as string;
expect(writtenContent).to.include("SELECT shobj_description(oid, 'pg_database') FROM pg_database");
expect(writtenContent).to.not.include("SELECT obj_description(oid, 'pg_database') FROM pg_database");
});

it('drops legacy and current REST users during partial initialization cleanup', async (): Promise<void> => {
await postgres.initializeMirrorNode(namespace, context);

const wrapperArguments: string[] = writeFileSyncStub
.getCalls()
.find((call: SinonSpyCall<string[], string>): boolean => (call.args[0] as string).includes('run-init'))!.args;
const writtenContent: string = wrapperArguments[1] as string;
expect(writtenContent).to.include('mirror_api mirror_rest mirror_rest_java');
});

it('wrapper script contains all required service passwords', async (): Promise<void> => {
await postgres.initializeMirrorNode(namespace, context);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,12 +197,60 @@ describe('SharedResourceManager', (): void => {
await manager.installChart(namespace, '', chartVersion, context);

const valueArguments: string[] = chartValueArguments();
const postgresNodeSelectorArgument: string = String.raw`postgresql.primary.nodeSelector.solo\.hashgraph\.io/role=database`;

expect(valueArguments).to.include(String.raw`postgresql.primary.nodeSelector.solo\.hashgraph\.io/role=database`);
expect(valueArguments).to.include(postgresNodeSelectorArgument);
expect(valueArguments[valueArguments.indexOf(postgresNodeSelectorArgument) - 1]).to.equal('--set-string');
expect(valueArguments).to.include('postgresql.primary.tolerations[0].value=adhoc-single-day-test');
expect(valueArguments).to.include('redis.replica.tolerations[0].value=adhoc-performance-test');
});

it('adds role scheduling to redis from mirror components when redis does not define it', async (): Promise<void> => {
const valuesFilePath: string = PathEx.join(temporaryDirectory, 'redis-role-fallback-values.yaml');
fs.writeFileSync(
valuesFilePath,
[
'nodeSelector:',
' solo.hashgraph.io/owner: "adhoc-performance-test"',
' solo.hashgraph.io/network-id: "7"',
'tolerations:',
' - key: "solo.hashgraph.io/owner"',
' operator: "Equal"',
' value: "adhoc-performance-test"',
' effect: "NoSchedule"',
' - key: "solo.hashgraph.io/network-id"',
' operator: "Equal"',
' value: "7"',
' effect: "NoSchedule"',
'redis:',
' enabled: true',
'importer:',
' nodeSelector:',
' solo.hashgraph.io/role: "consensus-node"',
' tolerations:',
' - key: "solo.hashgraph.io/role"',
' operator: "Equal"',
' value: "consensus-node"',
' effect: "NoSchedule"',
'',
].join('\n'),
);

manager.setSchedulingChartValues(new HelmChartValues().file(valuesFilePath));

await manager.installChart(namespace, '', chartVersion, context);

const valueArguments: string[] = chartValueArguments();
const redisRoleSelectorArgument: string = String.raw`redis.replica.nodeSelector.solo\.hashgraph\.io/role=consensus-node`;

expect(valueArguments).to.include(redisRoleSelectorArgument);
expect(valueArguments[valueArguments.indexOf(redisRoleSelectorArgument) - 1]).to.equal('--set-string');
expect(valueArguments).to.include('redis.replica.tolerations[2].key=solo.hashgraph.io/role');
expect(valueArguments).to.include('redis.replica.tolerations[2].value=consensus-node');
expect(valueArguments).to.include('redis.master.tolerations[2].key=solo.hashgraph.io/role');
expect(valueArguments).to.include('redis.master.tolerations[2].value=consensus-node');
});

it('installs chart with the correct release name and chart name', async (): Promise<void> => {
await manager.installChart(namespace, '', chartVersion, context);

Expand Down
Loading