feat: Pushing local created files to catalog service by creating new Entry.

toolbox/mdcode/src/libts/gcp/dataplex.ts

@@ -177,6 +177,34 @@ export class CatalogClient extends api.ApiClient {
     } while (pageToken);
   }
 
+  async createEntry(project: string, location: string, entryGroup: string, 
+                    entryId: string, entry?: Entry): Promise<api.ApiResult<Entry>> {
+    const parent = catalogContainer(project, location, entryGroup);
+    const resourceName = `${parent}/entries`;
+
+    const params: Record<string, any> = { entryId };
+
+    const res = await this._post<Entry>(resourceName, entry, params);
+
+    if (res.status == 200 && res.result) {
+      await _fixEntry(res.result, this.context);
+    }
+
+    return res;
+  }
+
+  async createEntryGroup(project: string, location: string, 
+                         entryGroupId: string, entryGroup?: EntryGroup): Promise<api.ApiResult<EntryGroup>> {
+    const parent = catalogContainer(project, location);
+    const resourceName = `${parent}/entryGroups`;
+
+    const params: Record<string, any> = { entryGroupId };
+
+    const res = await this._post<EntryGroup>(resourceName, entryGroup, params);
+
+    return res;
+  }
+
 }
 
 

toolbox/mdcode/src/libts/sync.ts

@@ -44,6 +44,9 @@ export class CatalogSync {
         const nameParts = entry.name.split('/');
         const res = await this._catalog.lookupEntry(nameParts[1], nameParts[3], entry.name,
                                                     [...this._snapshot.aspectTypes.keys()]);
+        // The server will respond with 403 permission denied for both resource not exist or 
+        // insufficient permission. We cannot tell if a resource not exist or user does not 
+        // have the access. Thus using 200 for an ensured result.
         if (res.status != 200 || !res.result) {
           continue;
         }
@@ -76,6 +79,20 @@ export class CatalogSync {
       const project = nameParts[1];
       const location = nameParts[3];
 
+      const exist = await this._catalog.lookupEntry(project, location, entry.name);
+      if (exist.status != 200 || !exist.result) {
+        console.log(`entry ${name} does not exist, will try to create the entry.`);
+
+        const entryGroup = nameParts[5];
+        const entryId = nameParts[7];
+        const createEntryRes = await this._catalog.createEntry(project, location, entryGroup, entryId, entry);
+        if (createEntryRes.status != 200 || !createEntryRes.result) {
+          console.log(`Failed to push entry ${entry.name}: Failed to create new entry.`);
+        }
+        console.log(`Successfully created and pushed entry ${entry.name}`);
+        continue;
+      }
+
       const updateMask = [];
       const aspectKeys = Object.keys(entry.aspects || {});
       if (aspectKeys.length) {

toolbox/mdcode/tests/libts/mocks.ts

@@ -130,6 +130,15 @@ export class CatalogClientMock extends gcp.CatalogClient {
     }
     return { status: 404, message: 'Not found' };
   }
+
+  async createEntry(project: string, location: string, entryGroup: string, entryId: string, entry?: gcp.Entry): Promise<gcp.ApiResult<gcp.Entry>> {
+    const fakeEntry = entry;
+    if (fakeEntry) {
+      this.mockEntries.push(fakeEntry);
+      return { status: 200, result: entry };
+    }
+    return {status: 404, message: 'Not found' };
+  }
 }
 
 

toolbox/mdcode/tests/libts/scenarios.ts

@@ -145,7 +145,7 @@ function runScenario(scenario: any) {
 
       // Assert expectations - Catalog Service
       if (scenario.assert?.catalog?.entries) {
-        expect(catalog.mockEntries).toEqual(scenario.assert.catalog.entries);
+        expect(JSON.parse(JSON.stringify(catalog.mockEntries))).toEqual(JSON.parse(JSON.stringify(scenario.assert.catalog.entries)));
       }
     });
   });
@@ -236,6 +236,15 @@ function main() {
     }
   );
 
+  spyOn(gcp.CatalogClient.prototype, 'createEntry').mockImplementation(
+    async function(project: string, location: string, entryGroup: string, entryId: string, entry?: gcp.Entry) {
+      if (currentCatalogMock) {
+        return await currentCatalogMock.createEntry(project, location, entryGroup, entryId, entry);
+      }
+      return { status: 404, message: 'Not found' };
+    }
+  );
+
   spyOn(bq.BigQueryClient.prototype, 'getDataset').mockImplementation(
     async function(project: string, dataset: string) {
       if (currentBigQueryMock) {

toolbox/mdcode/tests/scenarios/push_new_entry.yaml

@@ -0,0 +1,49 @@
+name: push_new_entry
+description: Push Custom entry modifications
+setup:
+  catalog:
+    entryGroups:
+      - name: projects/p/locations/us/entryGroups/custom-group
+    entryTypes:
+      - name: projects/p/locations/global/entryTypes/custom-type
+    entries:
+      - name: projects/p/locations/us/entryGroups/custom-group/entries/existing-entry
+        entryType: projects/p/locations/global/entryTypes/custom-type
+        entrySource:
+          description: Old custom description
+
+  fileSystem:
+    catalog.yaml: |
+      scope: entryGroup.p.us.custom-group
+      snapshot:
+        entries:
+          - p.global.custom-type
+
+
+actions:
+  - action: pull
+  - action: createEntry
+    name: custom-entry
+    entry:
+      name: custom-entry
+      type: p.global.custom-type
+      resource:
+        description: Hello world
+  - action: push
+
+assert:
+  fileSystem:
+    catalog/custom-entry.yaml:
+      - contains: "name: custom-entry"
+      - contains: "description: Hello world"
+  catalog:
+    entries:
+      - name: projects/p/locations/us/entryGroups/custom-group/entries/existing-entry
+        entryType: projects/p/locations/global/entryTypes/custom-type
+        entrySource:
+          description: Old custom description
+      - name: projects/p/locations/us/entryGroups/custom-group/entries/custom-entry
+        entryType: projects/p/locations/global/entryTypes/custom-type
+        entrySource:
+          description: "Hello world"
+        aspects: {}