Add EntryLinks support

toolbox/mdcode/.gitignore

@@ -1,3 +1,6 @@
 build/
 dist/
 node_modules/
+catalog.yaml
+catalog/
+tests/scenarios/

toolbox/mdcode/src/libts/gcp/api.ts

@@ -46,6 +46,12 @@ export class ApiClient {
     return this._requestRetry('PATCH', url, queryParams, body);
   }
 
+  async _delete<T>(resourceName: string,
+                   queryParams?: Record<string, any>): Promise<ApiResult<T>> {
+    const url = `${this._endpoint}/${this._pathPrefix}/${resourceName}`;
+    return this._requestRetry('DELETE', url, queryParams);
+  }
+
   private async _requestRetry<T>(method: string,
                                  url: string,
                                  queryParams?: Record<string, any>,

toolbox/mdcode/src/libts/gcp/crm.ts

@@ -48,3 +48,51 @@ export async function fixProject(resource: string, ctx: context.ApiContext): Pro
 
   return resource;
 }
+
+const PROJECT_ID_TO_NUM_CACHE = new Map<string, string>();
+PROJECT_ID_TO_NUM_CACHE.set('dataplex-types', '655216118709');
+
+export function tryGetProjectNumber(projectId: string): string | undefined {
+  return PROJECT_ID_TO_NUM_CACHE.get(projectId);
+}
+
+export async function toProjectNumber(resource: string, ctx: context.ApiContext): Promise<string> {
+  if (!resource.includes('/')) {
+    let num = PROJECT_ID_TO_NUM_CACHE.get(resource);
+    if (!num) {
+      const res = await new ResourceManagerClient(ctx).getProject(resource);
+      if (res.status === 200 && res.result?.name) {
+        const nameParts = res.result.name.split('/');
+        num = nameParts[nameParts.length - 1];
+      }
+    }
+
+    if (num) {
+      PROJECT_ID_TO_NUM_CACHE.set(resource, num);
+      return num;
+    }
+    return resource;
+  }
+
+  const parts = resource.split('/');
+  if (parts.length > 1 && !/^\d+$/.test(parts[1])) {
+    const projectId = parts[1];
+    let num = PROJECT_ID_TO_NUM_CACHE.get(projectId);
+    if (!num) {
+      const res = await new ResourceManagerClient(ctx).getProject(projectId);
+      if (res.status === 200 && res.result?.name) {
+        const nameParts = res.result.name.split('/');
+        num = nameParts[nameParts.length - 1];
+      }
+    }
+
+    if (num) {
+      PROJECT_ID_TO_NUM_CACHE.set(projectId, num);
+      parts[1] = num;
+    }
+    resource = parts.join('/');
+  }
+
+  return resource;
+}
+

toolbox/mdcode/src/libts/gcp/dataplex.ts

@@ -49,6 +49,24 @@ export interface Entry {
   aspects?: Record<string, Aspect>;
 }
 
+export interface EntryReference {
+  name: string;
+  type: string;
+  path?: string;
+}
+
+export interface EntryLink {
+  name: string;
+  entryLinkType: string;
+  entryReferences: EntryReference[];
+  aspects?: Record<string, Aspect>;
+}
+
+export interface LookupEntryLinksResponse {
+  entryLinks: EntryLink[];
+  nextPageToken?: string;
+}
+
 interface EntryList {
   entries: Entry[];
   nextPageToken?: string;
@@ -79,6 +97,16 @@ export class CatalogClient extends api.ApiClient {
     return await this._get(name);
   }
 
+  async getGlossary(project: string, location: string, glossaryId: string): Promise<api.ApiResult<any>> {
+    const name = `projects/${project}/locations/${location}/glossaries/${glossaryId}`;
+    return await this._get(name);
+  }
+
+  async getGlossaryTerm(project: string, location: string, glossaryId: string, termId: string): Promise<api.ApiResult<any>> {
+    const name = `projects/${project}/locations/${location}/glossaries/${glossaryId}/terms/${termId}`;
+    return await this._get(name);
+  }
+
   async getEntry(project: string, location: string, entryGroup: string, entry: string,
                  aspects?: string[]): Promise<api.ApiResult<Entry>> {
     const name = `${catalogContainer(project, location, entryGroup)}/entries/${entry}`;
@@ -177,6 +205,124 @@ export class CatalogClient extends api.ApiClient {
     } while (pageToken);
   }
 
+  async lookupEntryLinks(
+    project: string,
+    location: string,
+    entryName: string,
+    entryLinkTypes?: string[]
+  ): Promise<api.ApiResult<LookupEntryLinksResponse>> {
+    const container = `${catalogContainer(project, location)}:lookupEntryLinks`;
+    const entryLinks: EntryLink[] = [];
+    let pageToken: string | undefined = undefined;
+
+    do {
+      const params: Record<string, any> = {
+        entry: entryName,
+        pageSize: 1000,
+      };
+      if (entryLinkTypes && entryLinkTypes.length) {
+        params.entryLinkTypes = entryLinkTypes;
+      }
+      if (pageToken) {
+        params.pageToken = pageToken;
+      }
+      const res = await this._get<LookupEntryLinksResponse>(container, params);
+      if (res.status !== 200) {
+        return res;
+      }
+      if (res.result?.entryLinks) {
+        for (const link of res.result.entryLinks) {
+          await _fixEntryLink(link, this.context);
+          entryLinks.push(link);
+        }
+      }
+      pageToken = res.result?.nextPageToken;
+    } while (pageToken);
+
+    return {
+      status: 200,
+      result: {
+        entryLinks,
+      },
+    };
+  }
+
+  async createEntryLink(
+    project: string,
+    location: string,
+    entryGroup: string,
+    entryLink: EntryLink
+  ): Promise<api.ApiResult<EntryLink>> {
+    const parent = catalogContainer(project, location, entryGroup);
+    const container = `${parent}/entryLinks`;
+
+    const sortedRefs = [...entryLink.entryReferences].sort((a, b) => a.name.localeCompare(b.name));
+    const source = sortedRefs[0]?.name || '';
+    const target = sortedRefs[1]?.name || '';
+    const type = entryLink.entryLinkType.split('/').pop() || '';
+    const sourcePath = sortedRefs[0]?.path || '';
+    const targetPath = sortedRefs[1]?.path || '';
+
+    const hashInput = `${source}|${target}|${type}|${sourcePath}|${targetPath}`;
+    let hash = 0;
+    for (let i = 0; i < hashInput.length; i++) {
+      hash = (hash << 5) - hash + hashInput.charCodeAt(i);
+      hash |= 0;
+    }
+    const entryLinkId = `link-${Math.abs(hash).toString(36)}`;
+    const params = { entryLinkId };
+
+    return await this._post<EntryLink>(container, entryLink, params);
+  }
+
+  async deleteEntryLink(
+    project: string,
+    location: string,
+    entryGroup: string,
+    entryLinkName: string
+  ): Promise<api.ApiResult<any>> {
+    const parent = catalogContainer(project, location, entryGroup);
+    const name = `${parent}/entryLinks/${entryLinkName}`;
+    return await this._delete<any>(name);
+  }
+
+  async *listEntryLinks(
+    project: string,
+    location: string,
+    entryGroup: string,
+    filter?: string
+  ): AsyncGenerator<EntryLink, void, unknown> {
+    const parent = catalogContainer(project, location, entryGroup);
+    const resourceName = `${parent}/entryLinks`;
+
+    let pageToken: string | undefined = undefined;
+    do {
+      const params: Record<string, string | number> = { pageSize: 1000 };
+      if (filter) {
+        params.filter = filter;
+      }
+      if (pageToken) {
+        params.pageToken = pageToken;
+      }
+
+      const res = await this._get<{ entryLinks: EntryLink[]; nextPageToken?: string }>(
+        resourceName,
+        params
+      );
+      if (res.status !== 200) {
+        throw new Error(`Failed to list entry links: ${res.message || res.status}`);
+      }
+
+      const links = res.result?.entryLinks || [];
+      for (const link of links) {
+        await _fixEntryLink(link, this.context);
+        yield link;
+      }
+
+      pageToken = res.result?.nextPageToken;
+    } while (pageToken);
+  }
+
   async createEntry(project: string, location: string, entryGroup: string, 
                     entryId: string, entry?: Entry): Promise<api.ApiResult<Entry>> {
     const parent = catalogContainer(project, location, entryGroup);
@@ -205,6 +351,36 @@ export class CatalogClient extends api.ApiClient {
     return res;
   }
 
+  async fixEntry(entry: Entry): Promise<void> {
+    await _fixEntry(entry, this.context);
+  }
+
+  async fixEntryLink(link: EntryLink): Promise<void> {
+    await _fixEntryLink(link, this.context);
+  }
+
+}
+
+async function _fixEntryLink(link: EntryLink, ctx: context.ApiContext): Promise<void> {
+  link.name = await crm.fixProject(link.name, ctx);
+  link.entryLinkType = await crm.fixProject(link.entryLinkType, ctx);
+  if (link.entryReferences) {
+    for (const ref of link.entryReferences) {
+      ref.name = await crm.fixProject(ref.name, ctx);
+    }
+  }
+  if (link.aspects) {
+    const fixedAspects: Record<string, Aspect> = {};
+    for (const [aspectKey, aspectValue] of Object.entries(link.aspects)) {
+      let aspectType = aspectValue.aspectType || _typeRefToName(aspectKey, 'aspect');
+      aspectType = await crm.fixProject(aspectType, ctx);
+      fixedAspects[_nameToTypeRef(aspectType)] = {
+        aspectType,
+        data: aspectValue.data ?? {}
+      };
+    }
+    link.aspects = fixedAspects;
+  }
 }
 
 

toolbox/mdcode/src/libts/layouts/standard.ts

@@ -9,6 +9,24 @@ import { CatalogLayout } from '../layout';
 import * as md from '../metadata';
 
 
+async function findYamlFiles(dir: string): Promise<string[]> {
+  const files: string[] = [];
+  try {
+    const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+      const fullPath = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        files.push(...(await findYamlFiles(fullPath)));
+      } else if (entry.isFile() && entry.name.endsWith('.yaml')) {
+        files.push(fullPath);
+      }
+    }
+  } catch (err) {
+    // Ignore folder read errors
+  }
+  return files;
+}
+
 export class StandardLayout implements CatalogLayout {
 
   private readonly _catalogPath: string;
@@ -26,11 +44,7 @@ export class StandardLayout implements CatalogLayout {
       return;
     }
 
-    const matches = await glob.glob('**/*.yaml', {
-      cwd: this._catalogPath,
-      absolute: true,
-      nodir: true,
-    });
+    const matches = await findYamlFiles(this._catalogPath);
 
     for (const localPath of matches) {
       try {