Skip to content

Commit 4fcedbc

Browse files
committed
Add initializeBucketCapacity to seed empty-bucket metrics
Add a MongoClientInterface method that idempotently inserts a zero-value bucket capacity document into __infostore when none exists (upsert with $setOnInsert, so an existing count-items document with real values is never overwritten). Exposed through MetadataWrapper (no-op for non-mongodb backends). This lets CloudServer seed a metric document at bucket creation (or when a quota is enabled on an empty bucket) so bucket quota checks are served before the periodic count-items job runs. Issue: ARSN-610
1 parent c63bf83 commit 4fcedbc

3 files changed

Lines changed: 187 additions & 47 deletions

File tree

lib/storage/metadata/MetadataWrapper.js

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,6 +462,16 @@ class MetadataWrapper {
462462
return this.client.getUUID(log, cb);
463463
}
464464

465+
initializeBucketCapacity(bucketName, creationDate, log, cb) {
466+
if (!this.client.initializeBucketCapacity) {
467+
log.debug('initializeBucketCapacity not supported by backend, skipping', {
468+
implName: this.implName,
469+
});
470+
return cb(null);
471+
}
472+
return this.client.initializeBucketCapacity(bucketName, creationDate, log, cb);
473+
}
474+
465475
getDiskUsage(log, cb) {
466476
if (!this.client.getDiskUsage) {
467477
log.debug('returning empty disk usage as fallback', {

lib/storage/metadata/mongoclient/MongoClientInterface.ts

Lines changed: 110 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -151,63 +151,75 @@ export type InternalListObjectParams = {
151151
gt?: undefined;
152152
};
153153

154+
export interface UsedCapacityMetrics {
155+
current: Long | number;
156+
nonCurrent: Long | number;
157+
_currentCold: Long | number;
158+
_nonCurrentCold: Long | number;
159+
_currentRestored: Long | number;
160+
_currentRestoring: Long | number;
161+
_nonCurrentRestored: Long | number;
162+
_nonCurrentRestoring: Long | number;
163+
_incompleteMPUParts: Long | number;
164+
// present on per-location and inflight-tagged bucket entries
165+
_inflightsPreScan?: Long | number;
166+
}
167+
168+
export interface ObjectCountMetrics {
169+
current: Long | number;
170+
nonCurrent: Long | number;
171+
deleteMarker: Long | number;
172+
_currentCold: Long | number;
173+
_nonCurrentCold: Long | number;
174+
_currentRestored: Long | number;
175+
_currentRestoring: Long | number;
176+
_nonCurrentRestored: Long | number;
177+
_nonCurrentRestoring: Long | number;
178+
_incompleteMPUUploads: Long | number;
179+
}
180+
154181
export interface InfostoreDocument extends Document {
155182
_id: string | 'uuid';
156183
value?: string | ObjectMDStats;
157184
measuredOn?: string;
158-
objectCount?: {
159-
current: Long | number;
160-
_currentCold: Long | number;
161-
deleteMarker: Long | number;
162-
nonCurrent: Long | number;
163-
_nonCurrentCold: Long | number;
164-
_currentRestored: Long | number;
165-
_currentRestoring: Long | number;
166-
_nonCurrentRestored: Long | number;
167-
_nonCurrentRestoring: Long | number;
168-
_incompleteMPUUploads: Long | number;
169-
};
170-
usedCapacity?: {
171-
current: Long | number;
172-
_currentCold: Long | number;
173-
nonCurrent: Long | number;
174-
_nonCurrentCold: Long | number;
175-
_currentRestored: Long | number;
176-
_currentRestoring: Long | number;
177-
_nonCurrentRestored: Long | number;
178-
_nonCurrentRestoring: Long | number;
179-
_incompleteMPUParts: Long | number;
180-
};
185+
objectCount?: ObjectCountMetrics;
186+
usedCapacity?: UsedCapacityMetrics;
181187
locations: {
182188
[key: string]: {
183-
usedCapacity: {
184-
current: Long | number;
185-
nonCurrent: Long | number;
186-
_currentCold: Long | number;
187-
_nonCurrentCold: Long | number;
188-
_currentRestored: Long | number;
189-
_currentRestoring: Long | number;
190-
_nonCurrentRestored: Long | number;
191-
_nonCurrentRestoring: Long | number;
192-
_inflightsPreScan: Long | number;
193-
_incompleteMPUParts: Long | number;
194-
};
195-
objectCount: {
196-
current: Long | number;
197-
nonCurrent: Long | number;
198-
_currentCold: Long | number;
199-
_nonCurrentCold: Long | number;
200-
_currentRestored: Long | number;
201-
_currentRestoring: Long | number;
202-
_nonCurrentRestored: Long | number;
203-
_nonCurrentRestoring: Long | number;
204-
_incompleteMPUUploads: Long | number;
205-
deleteMarker: Long | number;
206-
};
189+
usedCapacity: UsedCapacityMetrics;
190+
objectCount: ObjectCountMetrics;
207191
};
208192
};
209193
}
210194

195+
// Canonical zero-value capacity/count metrics for an empty bucket, matching the
196+
// structure downstream metric readers expect.
197+
const emptyBucketCapacityMetrics: { usedCapacity: UsedCapacityMetrics; objectCount: ObjectCountMetrics } = {
198+
usedCapacity: {
199+
current: 0,
200+
nonCurrent: 0,
201+
_currentCold: 0,
202+
_nonCurrentCold: 0,
203+
_currentRestored: 0,
204+
_currentRestoring: 0,
205+
_nonCurrentRestored: 0,
206+
_nonCurrentRestoring: 0,
207+
_incompleteMPUParts: 0,
208+
},
209+
objectCount: {
210+
current: 0,
211+
nonCurrent: 0,
212+
deleteMarker: 0,
213+
_currentCold: 0,
214+
_nonCurrentCold: 0,
215+
_currentRestored: 0,
216+
_currentRestoring: 0,
217+
_nonCurrentRestored: 0,
218+
_nonCurrentRestoring: 0,
219+
_incompleteMPUUploads: 0,
220+
},
221+
};
222+
211223
export type ObjectMDStats = {
212224
versions: number;
213225
objects: any;
@@ -2711,6 +2723,57 @@ class MongoClientInterface {
27112723
});
27122724
}
27132725

2726+
/**
2727+
* Initialize a zero-value bucket capacity metric document in the
2728+
* `__infostore` collection, unless one already exists. This lets quota
2729+
* checks be served for a freshly created (or newly quota-enabled empty)
2730+
* bucket before the periodic metrics job has produced real values.
2731+
*
2732+
* Idempotent: an existing document (e.g. one already holding real metrics)
2733+
* is never overwritten, thanks to `$setOnInsert`.
2734+
*
2735+
* @param bucketName - name of the bucket
2736+
* @param creationDate - bucket creation date used to build the metric id;
2737+
* must match the id the metrics job uses so lookups resolve to the same
2738+
* document
2739+
* @param log - logger instance
2740+
* @param cb - callback
2741+
* @returns undefined
2742+
*/
2743+
initializeBucketCapacity(
2744+
bucketName: string,
2745+
creationDate: string,
2746+
log: werelogs.Logger,
2747+
cb: ArsenalCallback<void>,
2748+
) {
2749+
const i = this.getCollection<InfostoreDocument>(INFOSTORE);
2750+
if (!i) {
2751+
log.error('initializeBucketCapacity: error getting infostore collection');
2752+
return cb(errors.InternalError);
2753+
}
2754+
const timestamp = new Date(creationDate).getTime();
2755+
if (Number.isNaN(timestamp)) {
2756+
log.error('initializeBucketCapacity: invalid creationDate', { bucketName, creationDate });
2757+
return cb(errors.InternalError);
2758+
}
2759+
const _id = `bucket_${bucketName}_${timestamp}`;
2760+
const doc = <InfostoreDocument>{
2761+
_id,
2762+
measuredOn: new Date().toJSON(),
2763+
...emptyBucketCapacityMetrics,
2764+
};
2765+
return i
2766+
.updateOne({ _id }, { $setOnInsert: doc }, { upsert: true })
2767+
.then(() => cb(null))
2768+
.catch(err => {
2769+
log.error('initializeBucketCapacity: error initializing bucket capacity', {
2770+
error: err.message,
2771+
bucketName,
2772+
});
2773+
return cb(errors.InternalError);
2774+
});
2775+
}
2776+
27142777
/*
27152778
* we always try to generate a new UUID in order to be atomic in
27162779
* case of concurrency. The write will fail if it already exists.

tests/unit/storage/metadata/mongoclient/MongoClientInterface.spec.js

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1609,6 +1609,73 @@ describe('MongoClientInterface, readUUID', () => {
16091609
});
16101610
});
16111611

1612+
describe('MongoClientInterface, initializeBucketCapacity', () => {
1613+
let client;
1614+
let sandbox;
1615+
1616+
beforeEach(async () => {
1617+
sandbox = sinon.createSandbox();
1618+
client = createClient();
1619+
await promisify(client.setup).bind(client)();
1620+
});
1621+
1622+
afterEach(async () => {
1623+
sandbox.restore();
1624+
if (client) {
1625+
await promisify(client.close).bind(client)();
1626+
}
1627+
});
1628+
1629+
it('should insert a zero-value bucket capacity document', async () => {
1630+
const bucketName = 'init-capacity-bucket';
1631+
const creationDate = new Date('2023-03-08T14:13:28.000Z').toJSON();
1632+
const expectedId = `bucket_${bucketName}_${new Date(creationDate).getTime()}`;
1633+
await promisify(client.initializeBucketCapacity).bind(client)(bucketName, creationDate, logger);
1634+
const doc = await client.getCollection('__infostore').findOne({ _id: expectedId });
1635+
assert(doc, 'Expected the capacity document to be inserted');
1636+
assert.strictEqual(doc.usedCapacity.current, 0);
1637+
assert.strictEqual(doc.usedCapacity.nonCurrent, 0);
1638+
assert.strictEqual(doc.objectCount.current, 0);
1639+
assert.strictEqual(doc.objectCount.deleteMarker, 0);
1640+
assert(doc.measuredOn, 'Expected measuredOn to be set');
1641+
});
1642+
1643+
it('should not overwrite an existing capacity document (idempotent)', async () => {
1644+
const bucketName = 'existing-capacity-bucket';
1645+
const creationDate = new Date('2023-03-08T14:13:28.000Z').toJSON();
1646+
const expectedId = `bucket_${bucketName}_${new Date(creationDate).getTime()}`;
1647+
const coll = client.getCollection('__infostore');
1648+
await coll.insertOne({
1649+
_id: expectedId,
1650+
measuredOn: new Date().toJSON(),
1651+
usedCapacity: { current: 12345, nonCurrent: 0 },
1652+
objectCount: { current: 7, deleteMarker: 0 },
1653+
});
1654+
await promisify(client.initializeBucketCapacity).bind(client)(bucketName, creationDate, logger);
1655+
const doc = await coll.findOne({ _id: expectedId });
1656+
assert.strictEqual(doc.usedCapacity.current, 12345, 'Existing capacity must not be overwritten');
1657+
assert.strictEqual(doc.objectCount.current, 7);
1658+
});
1659+
1660+
it('should return InternalError when the update fails', async () => {
1661+
const mockCollection = {
1662+
updateOne: sandbox.stub().rejects(new Error('Simulated MongoDB error')),
1663+
};
1664+
sandbox.stub(client, 'getCollection').returns(mockCollection);
1665+
await assert.rejects(
1666+
promisify(client.initializeBucketCapacity).bind(client)('b', new Date().toJSON(), logger),
1667+
err => err.code === 500,
1668+
);
1669+
});
1670+
1671+
it('should return InternalError when creationDate is invalid', async () => {
1672+
await assert.rejects(
1673+
promisify(client.initializeBucketCapacity).bind(client)('b', 'not-a-date', logger),
1674+
err => err.code === 500,
1675+
);
1676+
});
1677+
});
1678+
16121679
describe('MongoClientInterface, writeUUIDIfNotExists', () => {
16131680
let client;
16141681
let sandbox;

0 commit comments

Comments
 (0)