Skip to content
Merged
16 changes: 8 additions & 8 deletions services/server/src/server/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,18 @@ import yamljs from "yamljs";

// local imports
import logger from "../common/logger";
import {
initializeSourcifyChains,
sourcifyChainsMap,
} from "../sourcify-chains";
import { initializeSourcifyChains } from "../sourcify-chains";
import type { SourcifyChainMap } from "@ethereum-sourcify/lib-sourcify";
import type { LibSourcifyConfig } from "./server";
import { Server } from "./server";
import { SolcLocal } from "./services/compiler/local/SolcLocal";
import { VyperLocal } from "./services/compiler/local/VyperLocal";
import { FeLocal } from "./services/compiler/local/FeLocal";

export const getEtherscanApiKeyForEachChain = (): Record<string, string> =>
Object.entries(sourcifyChainsMap).reduce<Record<string, string>>(
export const getEtherscanApiKeyForEachChain = (
chainsMap: SourcifyChainMap,
): Record<string, string> =>
Object.entries(chainsMap).reduce<Record<string, string>>(
(acc, [chainId, { supported, etherscanApi }]) => {
const envName = supported ? etherscanApi?.apiKeyEnvName : undefined;
const value = envName ? process.env[envName] : undefined;
Expand Down Expand Up @@ -96,7 +96,7 @@ Object.defineProperty(RegExp.prototype, "toJSON", {
});

// Load chain config first so getEtherscanApiKeyForEachChain() sees the populated map
await initializeSourcifyChains();
const sourcifyChainsMap = await initializeSourcifyChains();

const server = new Server(
{
Expand Down Expand Up @@ -207,7 +207,7 @@ Object.defineProperty(RegExp.prototype, "toJSON", {
EtherscanVerify: {
defaultApiKey: process.env.ETHERSCAN_API_KEY as string,
// Extract the etherscanApiKey env vars from the supported chains
apiKeys: getEtherscanApiKeyForEachChain(),
apiKeys: getEtherscanApiKeyForEachChain(sourcifyChainsMap),
},
BlockscoutVerify: {
defaultApiKey: process.env.BLOCKSCOUT_API_KEY as string,
Expand Down
25 changes: 12 additions & 13 deletions services/server/src/sourcify-chains.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,8 @@ function buildCustomRpcs(
return rpcs;
}

export const sourcifyChainsMap: SourcifyChainMap = {};

/**
* Loads the chain configuration and populates sourcifyChainsMap.
* Loads the chain configuration and returns a populated SourcifyChainMap.
*
* Priority:
* 1. Local sourcify-chains.json (self-hosted override)
Expand All @@ -214,8 +212,10 @@ export const sourcifyChainsMap: SourcifyChainMap = {};
* Called by Server.init() so that both the CLI and test fixtures initialize chains
* through the same code path.
*/
export async function initializeSourcifyChains(): Promise<void> {
let chainsExtensions: SourcifyChainsExtensionsObjectWithHeaderEnvName;
export async function initializeSourcifyChains(): Promise<SourcifyChainMap> {
let chainsExtensions:
| SourcifyChainsExtensionsObjectWithHeaderEnvName
| undefined;

// Priority 1: local sourcify-chains.json (self-hosted override)
if (fs.existsSync(path.resolve(__dirname, "./sourcify-chains.json"))) {
Expand Down Expand Up @@ -264,17 +264,14 @@ export async function initializeSourcifyChains(): Promise<void> {
}
}
}
if (!chainsExtensions!) {
if (!chainsExtensions) {
throw new Error(
`Failed to fetch chains config after ${maxAttempts} attempts: ${lastError?.message}`,
);
}
}

// Clear the map before populating (allows re-initialization)
for (const key of Object.keys(sourcifyChainsMap)) {
delete sourcifyChainsMap[key];
}
const sourcifyChainsMap: SourcifyChainMap = {};

// Add LOCAL_CHAINS in non-production
if (process.env.NODE_ENV !== "production") {
Expand All @@ -285,18 +282,18 @@ export async function initializeSourcifyChains(): Promise<void> {

// Build SourcifyChain objects directly from the loaded extensions
for (const [chainIdStr, extension] of Object.entries(chainsExtensions)) {
const chainId = parseInt(chainIdStr);
// Skip local test chains (already added above)
if (chainId in sourcifyChainsMap) continue;
if (chainIdStr in sourcifyChainsMap) continue;

const chainId = parseInt(chainIdStr);
const rpcs = buildCustomRpcs(extension.rpc || []);
if (rpcs.length === 0 && extension.supported) {
logger.warn(
`Skipping supported chain ${chainId} (${extension.sourcifyName}): no usable RPCs configured`,
);
continue;
}
sourcifyChainsMap[chainId] = new SourcifyChain({
sourcifyChainsMap[chainIdStr] = new SourcifyChain({
name: extension.sourcifyName,
chainId,
supported: extension.supported,
Expand All @@ -309,4 +306,6 @@ export async function initializeSourcifyChains(): Promise<void> {
logger.info("SourcifyChains loaded", {
totalChains: Object.keys(chainsExtensions).length,
});

return sourcifyChainsMap;
}
12 changes: 6 additions & 6 deletions services/server/test/chains/chain-tests.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@ import chai from "chai";
import chaiHttp from "chai-http";
import addContext from "mochawesome/addContext";
import testEtherscanContracts from "../helpers/etherscanInstanceContracts.json";
import {
initializeSourcifyChains,
sourcifyChainsMap,
} from "../../src/sourcify-chains";
import { initializeSourcifyChains } from "../../src/sourcify-chains";
import _storageAddresses from "./sources/storage-contract-chain-addresses.json";
const storageAddresses: Record<string, string> = _storageAddresses; // add types
import createXInput from "./sources/createX.input.json";
Expand Down Expand Up @@ -62,7 +59,7 @@ chai.use(chaiHttp);

// Chains config is loaded async; use Mocha's --delay + run() to defer test registration
(async () => {
await initializeSourcifyChains();
const sourcifyChainsMap = await initializeSourcifyChains();

const chainsToTest = Object.entries(sourcifyChainsMap)
.filter(([id, chain]) => {
Expand Down Expand Up @@ -274,4 +271,7 @@ chai.use(chaiHttp);
});

run();
})();
})().catch((err) => {
console.error("Failed to initialize chains for chain tests:", err);
process.exit(1);
});
3 changes: 2 additions & 1 deletion services/server/test/chains/deployContracts.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { deployFromPrivateKey } from "../helpers/helpers";
import StorageArtifact from "./sources/storage.artifact.json";
import { sourcifyChainsMap } from "../../src/sourcify-chains";
import { initializeSourcifyChains } from "../../src/sourcify-chains";
import { program } from "commander";
import { JsonRpcProvider } from "ethers";
import { ChainRepository } from "../../src/sourcify-chain-repository";
Expand Down Expand Up @@ -36,6 +36,7 @@ if (require.main === module) {
}

async function main(chainId: number, privateKey: string) {
const sourcifyChainsMap = await initializeSourcifyChains();
const chainRepository = new ChainRepository(sourcifyChainsMap);
const chains = chainRepository.supportedChainsArray;

Expand Down
84 changes: 61 additions & 23 deletions services/server/test/chains/etherscan-instances.spec.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Periodical tests of Import from Etherscan for each instance e.g. Arbiscan, Etherscan, Bscscan, etc.

import testContracts from "../helpers/etherscanInstanceContracts.json";
import { sourcifyChainsMap } from "../../src/sourcify-chains";
import { hookIntoVerificationWorkerRun } from "../helpers/helpers";
import chai, { request } from "chai";
import { ServerFixture } from "../helpers/ServerFixture";
Expand All @@ -25,41 +24,77 @@ describe("Test each Etherscan instance", function () {
sandbox.restore();
});

const sourcifyChainsArray = new ChainRepository(sourcifyChainsMap)
.sourcifyChainsArray;

const testedChains: number[] = [];
let chainId: keyof typeof testContracts;
for (chainId in testContracts) {
if (!sourcifyChainsMap[chainId].supported) {
throw new Error(
`Unsupported chain (${chainId}) found in test configuration`,
);
}
if (process.env.TEST_CHAIN && process.env.TEST_CHAIN !== chainId) continue;
testedChains.push(parseInt(chainId));

describe(`#${chainId} ${sourcifyChainsMap[chainId].name}`, () => {
testContracts[chainId].forEach((contract) => {
// describe() and it() titles are registered synchronously, before the
// server fixture's before() hook has run, so serverFixture.sourcifyChainsMap
// is not yet available here. We start with just the chainId and inject the
// human-readable chain name at run time (see before/beforeEach below).
describe(`#${chainId}`, function () {
// `chainId` is a shared loop variable (`let` declared outside the loop),
// so closures that run asynchronously (before/beforeEach/it callbacks)
// would all see the last iteration's value. Capture the current value in
// a `const` so every describe scope gets its own immutable copy.
const id = chainId;

// chainName is set in before() and read in beforeEach() — both close
// over this variable so it stays in sync.
let chainName: string;

before(function () {
const chain = serverFixture.sourcifyChainsMap[id];

if (!chain?.supported) {
throw new Error(
`Unsupported chain (${id}) found in test configuration`,
);
}

chainName = chain.name;

// Rename the suite so reporters show the human-readable chain name.
// Most Mocha reporters (spec, mochawesome, …) read suite.title after
// all before() hooks have completed, so the renamed title appears in
// the output correctly.
this.test!.parent!.title = `#${id} ${chainName}`;
});

// it() titles are also fixed at registration time, so we patch
// this.currentTest.title here — same reasoning as the suite rename above.
beforeEach(function () {
if (this.currentTest) {
this.currentTest.title = this.currentTest.title.replace(
`for chain ${id}`,
`for chain ${chainName} (${id})`,
);
}
});

testContracts[id].forEach((contract) => {
const address = contract.address;
const expectedMatch = toMatchLevel(
contract.expectedStatus as VerificationStatus,
);
const type = contract.type;
const chain = chainId;

it(`Should import a ${type} contract from Etherscan for chain ${sourcifyChainsMap[chain].name} (${chain}) and verify the contract, finding a ${expectedMatch}`, async () => {
// "for chain ${id}" is a placeholder that beforeEach() replaces
// with the full "for chain ${chainName} (${id})" at run time.
it(`Should import a ${type} contract from Etherscan for chain ${id} and verify the contract, finding a ${expectedMatch}`, async () => {
const { resolveWorkers } = makeWorkersWait();

const verifyRes = await request(serverFixture.server.app)
.post(`/v2/verify/etherscan/${chain}/${address}`)
.post(`/v2/verify/etherscan/${id}/${address}`)
.send({});

await assertJobVerification(
serverFixture,
verifyRes,
resolveWorkers,
chain,
id,
getAddress(address),
expectedMatch,
);
Expand All @@ -68,18 +103,21 @@ describe("Test each Etherscan instance", function () {
});
}

describe("Double check that all supported chains are tested", () => {
const supportedEtherscanChains = sourcifyChainsArray.filter(
(chain) => chain.etherscanApi?.supported && chain.supported,
);

describe("Double check that all supported chains are tested", function () {
it("should have tested all supported chains", function (done) {
const untestedChains = supportedEtherscanChains.filter(
(chain) => !testedChains.includes(chain.chainId),
);
if (process.env.TEST_CHAIN) {
return this.skip();
}

const supportedEtherscanChains = new ChainRepository(
serverFixture.sourcifyChainsMap,
).sourcifyChainsArray.filter(
(chain) => chain.etherscanApi?.supported && chain.supported,
);

const untestedChains = supportedEtherscanChains.filter(
(chain) => !testedChains.includes(chain.chainId),
);
chai.assert(
untestedChains.length == 0,
`There are untested supported chains!: ${untestedChains
Expand Down
13 changes: 6 additions & 7 deletions services/server/test/helpers/ServerFixture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,7 @@ import { resetDatabase } from "../helpers/helpers";
import type { ServerOptions } from "../../src/server/server";
import { Server } from "../../src/server/server";
import config from "config";
import {
initializeSourcifyChains,
sourcifyChainsMap,
} from "../../src/sourcify-chains";
import { initializeSourcifyChains } from "../../src/sourcify-chains";
import type { StorageIdentifiers } from "../../src/server/services/storageServices/identifiers";
import { RWStorageIdentifiers } from "../../src/server/services/storageServices/identifiers";
import type { Pool } from "pg";
Expand All @@ -33,6 +30,8 @@ export class ServerFixture {
readonly repositoryV1Path: string;
readonly testS3Path: string = testS3Path;
readonly testS3Bucket: string = testS3Bucket;
// Assigned in the before() hook below; safe to read inside it/beforeEach.
sourcifyChainsMap!: SourcifyChainMap;
Comment thread
kuzdogan marked this conversation as resolved.
Outdated

private _server?: Server;

Expand Down Expand Up @@ -65,7 +64,7 @@ export class ServerFixture {
this.repositoryV1Path = config.get<string>("repositoryV1.path");

before(async () => {
await initializeSourcifyChains();
this.sourcifyChainsMap = await initializeSourcifyChains();

process.env.SOURCIFY_POSTGRES_PORT =
process.env.DOCKER_HOST_POSTGRES_TEST_PORT || "5431";
Expand All @@ -84,7 +83,7 @@ export class ServerFixture {
port: fixtureOptions_?.port || config.get<number>("server.port"),
maxFileSize: config.get<number>("server.maxFileSize"),
corsAllowedOrigins: config.get<string[]>("corsAllowedOrigins"),
chains: fixtureOptions_?.chains || sourcifyChainsMap,
chains: fixtureOptions_?.chains || this.sourcifyChainsMap,
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not a problem of this PR, but a problem in general:

below we always pass the this.sourcifyChainsMap to the verification service, but it should also prioritize the fixtureOptions.chains.

I think in general it would be better to remove the sourcifyChainMap from the VerificationServiceOptions and instead pass a separate argument from the Server constructor to the VerificationService in order to avoid such mistakes.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think it should be in this PR? The change should be straightforward?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See c16794e about prioritizing fixtureOptions.chains

Above comment for removing the sourcifyChainMap from VerificationServiceOptions

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

doesn't have to be in this PR, but should be straight forward, yes. I would simply add the chain map as a separate argument to the VerificationService constructor.

solc: new SolcLocal(config.get("solcRepo"), config.get("solJsonRepo")),
vyper: new VyperLocal(config.get("vyperRepo")),
fe: new FeLocal(config.get("feRepo")),
Expand All @@ -97,7 +96,7 @@ export class ServerFixture {
this._server = new Server(
serverOptions,
{
sourcifyChainMap: sourcifyChainsMap,
sourcifyChainMap: this.sourcifyChainsMap,
solcRepoPath: config.get("solcRepo"),
solJsonRepoPath: config.get("solJsonRepo"),
vyperRepoPath: config.get("vyperRepo"),
Expand Down
Loading