CSD-100: Lambdas reporting InvalidSignatureException errors (#4375)

* CSD-100 off master

* add lerna run prepublish to bootstrap

---------

Co-authored-by: jennyhliu <jenny.h.liu@nasa.gov>

Author: Nnaga1

CHANGELOG.md

@@ -42,6 +42,9 @@ Phase 2 — full apply to clean up old S3 objects and apply remaining changes:
 - **Security Vulnerabilities**
   - Upgraded package `lodash` to version 4.18.1.
   - Updated package overrides to address CVEs GHSA-43fc-jf86-j433 and GHSA-r5fr-rjxr-66jc.
+- **CSD-100**
+  - made changes to the `PrivateApiLambda` and `ApiEndpoints` lambdas to ensure the environment variables
+    are loaded after the handler invocation to circumvent `InvalidSignatureException` errors that were being reported
 
 ## [v21.3.2] 2026-03-20
 

package.json

@@ -49,7 +49,7 @@
     "tsc": "lerna run tsc",
     "tsc:listEmittedFiles": "lerna run tsc:listEmittedFiles --stream --no-prefix",
     "package": "lerna run package",
-    "bootstrap": "npm install && lerna run build && lerna run package && npm run bootstrap-python && git rev-parse --short HEAD > .bootstrapSha",
+    "bootstrap": "npm install && lerna run build && lerna run package && npm run bootstrap-python && lerna run prepublish && git rev-parse --short HEAD > .bootstrapSha",
     "bootstrap-python": "uv sync --all-groups",
     "bootstrap-no-build": "npm install && npm run bootstrap-python",
     "bootstrap-no-build-no-scripts": "npm install --ignore-scripts && npm run bootstrap-python",

packages/api/app/index.js

@@ -21,8 +21,10 @@ const boom = require('../lib/expressBoom');
 
 const log = new Logger({ sender: '@api/index' });
 
+let initEnvVars;
+
 // Load Environment Variables
-// This should be done outside of the handler to minimize Secrets Manager calls.
+// Called once per Lambda container to minimize Secrets Manager calls
 const initEnvVarsFunction = async () => {
   if (inTestMode() && process.env.INIT_ENV_VARS_FUNCTION_TEST !== 'true') {
     return undefined;
@@ -44,9 +46,19 @@ const initEnvVarsFunction = async () => {
     log.error(`Encountered error trying to set environment variables from secret ${apiConfigSecretId}`, error);
     throw error;
   }
+  log.info('Environment variables successfully initialized');
   return undefined;
 };
-const initEnvVars = initEnvVarsFunction();
+
+const ensureEnvVarsInitialized = () => {
+  if (initEnvVars === undefined) {
+    initEnvVars = initEnvVarsFunction().catch((error) => {
+      initEnvVars = undefined; // allow retry
+      throw error;
+    });
+  }
+  return initEnvVars;
+};
 
 // Setup express app
 const app = express();
@@ -95,7 +107,9 @@ app.use((err, _req, res, _next) => {
 const server = awsServerlessExpress.createServer(app);
 
 const handler = async (event, context) => {
-  await initEnvVars; // Wait for environment vars to resolve from initEnvVarsFunction
+  // Ensures environment variables are initialized once per container;
+  // subsequent invocations reuse the result or allow for re-initialization on failure
+  await ensureEnvVarsInitialized();
   const dynamoTableNames = JSON.parse(getRequiredEnvVar('dynamoTableNameString'));
   // Set Dynamo table names as environment variables for Lambda
   Object.keys(dynamoTableNames).forEach((tableEnvVarName) => {

packages/api/tests/app/test-app-secrets.js

@@ -0,0 +1,81 @@
+const test = require('ava');
+const sinon = require('sinon');
+
+const awsServices = require('@cumulus/aws-client/services');
+const { randomString } = require('@cumulus/common/test-utils');
+
+let secretId;
+const secretString = JSON.stringify({ testKey: 'testVal' });
+
+test.before(() => {
+  process.env.INIT_ENV_VARS_FUNCTION_TEST = 'true';
+  secretId = randomString(10);
+  process.env.api_config_secret_id = secretId;
+});
+
+test.after.always(() => {
+  delete process.env.INIT_ENV_VARS_FUNCTION_TEST;
+});
+
+test.serial('secretsManager is not called at module load time and called once for multiple invocations', async (t) => {
+  const stub = sinon.stub(awsServices, 'secretsManager');
+  const fakeClient = {
+    getSecretValue: sinon.stub().resolves({
+      SecretString: secretString,
+    }),
+  };
+  stub.returns(fakeClient);
+
+  // eslint-disable-next-line global-require
+  const { handler } = require('../../app');
+  process.env.dynamoTableNameString = '{}';
+
+  // since secretsManager is now called within the handler and not during module load
+  // which is done on line 38 above, there should be no calls to secretsManager until
+  // the handler itself is called
+  t.is(fakeClient.getSecretValue.callCount, 0);
+  await handler({});
+
+  // calling handler again to make sure the initPromise cache is working and
+  // secretsManager itself is not called again
+  await handler({});
+  await handler({});
+
+  // since the handler ran, there should be one call to secretsManager, even if it
+  // ran multiple times, the initPromise cache should prevent multiple calls to secretsManager
+  t.is(fakeClient.getSecretValue.callCount, 1);
+  t.teardown(() => stub.restore());
+});
+
+test.serial('handler retries successfully after a secretsManager failure', async (t) => {
+  const stub = sinon.stub(awsServices, 'secretsManager');
+
+  const fakeClient = {
+    getSecretValue: sinon.stub().resolves({
+      SecretString: secretString,
+    }),
+  };
+  stub.returns(fakeClient);
+
+  fakeClient.getSecretValue.onFirstCall().rejects();
+  fakeClient.getSecretValue.onSecondCall().resolves({
+    SecretString: secretString,
+  });
+
+  // refresh the module/app so the previous test doesn't affect this one
+  // and then reimport the handler
+  delete require.cache[require.resolve('../../app')];
+  // eslint-disable-next-line global-require
+  const { handler } = require('../../app');
+
+  t.is(fakeClient.getSecretValue.callCount, 0);
+  await t.throwsAsync(handler({}));
+  t.is(fakeClient.getSecretValue.callCount, 1);
+
+  await handler({});
+  t.is(fakeClient.getSecretValue.callCount, 2);
+
+  await handler({});
+  t.is(fakeClient.getSecretValue.callCount, 2);
+  t.teardown(() => stub.restore());
+});