MAT-10217 Import and Organize CQL Libraries from Downloaded IG Packages

src/main/java/gov/cms/madie/cqllibraryservice/controllers/CqlLibraryAdminController.java

@@ -161,6 +161,18 @@ public ResponseEntity<byte[]> exportSharedWith(
         .body(adminService.exportSharedWithLibraries(libraryids, username, accessToken));
   }
 
+  /**
+   * Asynchronous operation that initiates the installation of an IG (Implementation Guide) package
+   * for a given package ID (e.g. {@code hl7.fhir.us.qicore}) and version (e.g. {@code 7.0.2}). It
+   * downloads the IG(including its transitive dependencies) and then imports the CQL Libraries from
+   * it(including its transitive dependencies).
+   *
+   * @param principal the currently authenticated user
+   * @param request the IG package installation request containing the {@code packageId} and {@code
+   *     packageVersion} to install
+   * @return a {@link ResponseEntity} with HTTP status {@code 202 Accepted} and a confirmation
+   *     message indicating that the installation has been started
+   */
   @PostMapping("/ig-packages")
   @PreAuthorize("hasRole('MADIE-ADMIN')")
   public ResponseEntity<String> installIgPackage(
@@ -176,7 +188,7 @@ public ResponseEntity<String> installIgPackage(
     igPackageService.installIgPackage(sanitizedPackageId, sanitizedPackageVersion, username);
     return ResponseEntity.accepted()
         .body(
-            "IG package installation has been started for package"
+            "IG package installation has been started for package "
                 + sanitizedPackageId
                 + "#"
                 + sanitizedPackageVersion);

src/main/java/gov/cms/madie/cqllibraryservice/models/ExternalLibrary.java

@@ -0,0 +1,68 @@
+package gov.cms.madie.cqllibraryservice.models;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.index.CompoundIndex;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+import java.time.Instant;
+
+/**
+ * Represents a External CQL Library imported from a FHIR Implementation Guide (IG) package.
+ *
+ * <p>External CQL Libraries are separate from user-created CQL Libraries. They are not editable by
+ * users and are not included in user library validations, but are available for use by translation,
+ * execution, and the CQL Builder.
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Document
+@CompoundIndex(
+    name = "ns_canonical_library_name_version_idx",
+    def = "{'packageCanonical': 1, 'libraryName': 1, 'version': 1}",
+    unique = true)
+public class ExternalLibrary {
+
+  @Id private String id;
+  private String librarySetId;
+
+  /** Machine-readable library name (e.g. FHIRCommon) */
+  private String libraryName;
+
+  /** Human-readable library title (e.g. "FHIR Common") */
+  private String libraryTitle;
+
+  private String description;
+  private String version;
+
+  /**
+   * The canonical URL of the IG that contains this library (from {@code package.json#canonical}).
+   */
+  private String packageCanonical;
+
+  /** The NPM package name of the IG that contains this library (from {@code package.json#name}). */
+  private String namespacePrefix;
+
+  /** The raw CQL source text. */
+  private String cqlContent;
+
+  /** Always {@code false} – imported common libraries are never in draft state. */
+  private boolean draft;
+
+  /** The system owner for common libraries. */
+  private String publisher;
+
+  /** The system owner for common libraries. */
+  private String createdBy;
+
+  /** Timestamp when this library was first imported into the system. */
+  private Instant dateImported;
+
+  /** The original FHIR resource JSON string */
+  private String fhirResource;
+}

src/main/java/gov/cms/madie/cqllibraryservice/models/PackageStatus.java

@@ -4,6 +4,8 @@ public enum PackageStatus {
   DOWNLOADING,
   DOWNLOADED,
   DOWNLOAD_FAILED,
+  // Common CQL Library import is in progress.
+  PROCESSING,
   // Package was successfully read and no CQL Libraries were found.
   PROCESSED,
   // Package was successfully read, and CQL Libraries found were persisted.

src/main/java/gov/cms/madie/cqllibraryservice/models/PackageTrackingRecord.java

@@ -31,4 +31,16 @@ public class PackageTrackingRecord {
   private Instant lastAttemptedAt;
   private Instant downloadedAt;
   private String initiatedBy;
+
+  /** Number of CQL Libraries discovered across all packages during the last import run. */
+  private Integer discoveredLibraryCount;
+
+  /** Number of CQL Libraries actually persisted during the last import run. */
+  private Integer persistedLibraryCount;
+
+  /** Timestamp when the most recent CQL Library import started. */
+  private Instant importStartedAt;
+
+  /** Timestamp when the most recent CQL Library import completed. */
+  private Instant importCompletedAt;
 }

src/main/java/gov/cms/madie/cqllibraryservice/repositories/ExternalLibraryRepository.java

@@ -0,0 +1,15 @@
+package gov.cms.madie.cqllibraryservice.repositories;
+
+import gov.cms.madie.cqllibraryservice.models.ExternalLibrary;
+import org.springframework.data.mongodb.repository.MongoRepository;
+
+import java.util.Optional;
+
+public interface ExternalLibraryRepository extends MongoRepository<ExternalLibrary, String> {
+
+  Optional<ExternalLibrary> findByPackageCanonicalAndLibraryName(
+      String namespaceCanonical, String libraryName);
+
+  boolean existsByPackageCanonicalAndLibraryNameAndVersion(
+      String canonical, String libraryName, String version);
+}

src/main/java/gov/cms/madie/cqllibraryservice/services/ExternalLibraryDiscoveryService.java

@@ -0,0 +1,255 @@
+package gov.cms.madie.cqllibraryservice.services;
+
+import tools.jackson.databind.JsonNode;
+import tools.jackson.databind.ObjectMapper;
+import tools.jackson.databind.node.ObjectNode;
+import gov.cms.madie.cqllibraryservice.models.ExternalLibrary;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.hl7.fhir.utilities.json.model.JsonObject;
+import org.hl7.fhir.utilities.npm.NpmPackage;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+/**
+ * Discovers valid CQL Libraries from a FHIR NPM package.
+ *
+ * <p>A valid CQL Library must satisfy:
+ *
+ * <ul>
+ *   <li>{@code resourceType == "Library"}
+ *   <li>{@code type.coding[*].code == "logic-library"}
+ *   <li>At least one {@code content} entry with {@code contentType == "text/cql"}
+ * </ul>
+ */
+@Slf4j
+@Service
+@RequiredArgsConstructor
+public class ExternalLibraryDiscoveryService {
+
+  /**
+   * Discovers all valid CQL logic libraries from the given package path.
+   *
+   * @param packagePath the filesystem path of the extracted NPM package
+   * @param igPackageId the IG package ID (e.g. {@code hl7.fhir.us.qicore})
+   * @param igPackageVersion the IG package version (e.g. {@code 6.0.0})
+   * @return candidate {@link ExternalLibrary} objects ready for persistence
+   */
+  public List<ExternalLibrary> discoverLibrariesForPackage(
+      String packagePath, String igPackageId, String igPackageVersion) {
+    List<ExternalLibrary> allLibraries = new ArrayList<>();
+    try {
+      NpmPackage npmPackage = NpmPackage.fromFolder(packagePath);
+      allLibraries.addAll(discoverLibraries(npmPackage, igPackageId, igPackageVersion));
+    } catch (IOException e) {
+      log.error(
+          "Failed to load NpmPackage from path [{}] for package [{}#{}]: {}",
+          packagePath,
+          igPackageId,
+          igPackageVersion,
+          e.getMessage());
+    }
+    return allLibraries;
+  }
+
+  /**
+   * Discovers all valid CQL logic libraries from the given in-memory {@link NpmPackage}.
+   *
+   * @param npmPackage the loaded NPM package
+   * @param igPackageId the IG package ID
+   * @param igPackageVersion the IG package version
+   * @return candidate {@link ExternalLibrary} objects ready for persistence
+   */
+  public List<ExternalLibrary> discoverLibraries(
+      NpmPackage npmPackage, String igPackageId, String igPackageVersion) {
+    List<ExternalLibrary> discoveredLibraries = new ArrayList<>();
+
+    JsonObject npm = npmPackage.getNpm();
+    if (npm == null) {
+      log.warn("Package [{}#{}] has no package.json – skipping", igPackageId, igPackageVersion);
+      return discoveredLibraries;
+    }
+
+    String packageCanonical = npm.asString("canonical");
+    String namespacePrefix = npm.asString("name");
+
+    if (StringUtils.isBlank(packageCanonical)) {
+      log.warn(
+          "Package [{}#{}] package.json is missing 'canonical' – skipping",
+          igPackageId,
+          igPackageVersion);
+      return discoveredLibraries;
+    }
+    if (StringUtils.isBlank(namespacePrefix)) {
+      log.warn(
+          "Package [{}#{}] package.json is missing 'name' – skipping discovery",
+          igPackageId,
+          igPackageVersion);
+    }
+
+    List<String> libraryFiles;
+    try {
+      libraryFiles = npmPackage.listResources("Library");
+    } catch (IOException e) {
+      log.error(
+          "Failed to list Library resources in package [{}#{}]: {}",
+          igPackageId,
+          igPackageVersion,
+          e.getMessage());
+      return discoveredLibraries;
+    }
+
+    log.info(
+        "Found {} Library resource(s) in package [{}#{}]",
+        libraryFiles.size(),
+        igPackageId,
+        igPackageVersion);
+
+    for (String filename : libraryFiles) {
+      try (InputStream resourceStream = npmPackage.load("package", filename)) {
+        ExternalLibrary discoveredLibrary =
+            parseLibraryResource(resourceStream, packageCanonical, namespacePrefix, filename);
+        if (discoveredLibrary != null) {
+          discoveredLibraries.add(discoveredLibrary);
+          log.debug(
+              "Discovered CQL Library [{}] v[{}] from package [{}#{}]",
+              discoveredLibrary.getLibraryName(),
+              discoveredLibrary.getVersion(),
+              igPackageId,
+              igPackageVersion);
+        }
+      } catch (Exception e) {
+        log.warn(
+            "Skipping Library resource [{}] in package [{}#{}] due to error: {}",
+            filename,
+            igPackageId,
+            igPackageVersion,
+            e.getMessage());
+      }
+    }
+
+    return discoveredLibraries;
+  }
+
+  /**
+   * Parses a single FHIR Library JSON resource and returns a {@link ExternalLibrary} candidate if
+   * it meets all criteria, or {@code null} if it should be ignored.
+   */
+  private ExternalLibrary parseLibraryResource(
+      InputStream stream, String packageCanonical, String namespacePrefix, String filename) {
+
+    JsonNode root = new ObjectMapper().readTree(stream);
+
+    // 1. Must be resourceType == "Library"
+    if (!"Library".equals(root.path("resourceType").asString(null))) {
+      log.debug("Ignoring non-Library resource in file [{}]", filename);
+      return null;
+    }
+
+    // 2. Must have type.coding[*].code == "logic-library"
+    if (!hasLogicLibraryType(root)) {
+      log.debug("Ignoring Library resource [{}] – not a logic-library type", filename);
+      return null;
+    }
+
+    // 3. Must have cql(contentType == "text/cql")
+    String cqlContent = extractCqlContent(root);
+    if (cqlContent == null) {
+      log.debug("Ignoring Library resource [{}] – no text/cql content found", filename);
+      return null;
+    }
+
+    String libraryName = root.path("name").asString(null);
+    String version = root.path("version").asString(null);
+    String title = root.path("title").asString(null);
+    String description = root.path("description").asString(null);
+    String publisher = root.path("publisher").asString(null);
+
+    if (StringUtils.isBlank(libraryName)) {
+      log.warn("Library resource [{}] is missing 'name' – skipping", filename);
+      return null;
+    }
+    if (StringUtils.isBlank(version)) {
+      log.warn(
+          "Library resource [{}] (name={}) is missing 'version' – skipping", filename, libraryName);
+      return null;
+    }
+
+    return ExternalLibrary.builder()
+        .libraryName(libraryName)
+        .libraryTitle(title)
+        .version(version)
+        .description(description)
+        .packageCanonical(packageCanonical)
+        .namespacePrefix(namespacePrefix)
+        .publisher(publisher)
+        .cqlContent(cqlContent)
+        .fhirResource(stripContentData(root))
+        .draft(false)
+        .dateImported(Instant.now())
+        .build();
+  }
+
+  /**
+   * Returns a JSON string of the FHIR resource with the entire {@code content} array removed.
+   *
+   * <p>FHIR Library resources embed base64-encoded payloads (CQL, ELM XML, ELM JSON, etc.) inside
+   * the {@code content} array. These can be several MB each and, when stored verbatim, produce
+   * documents large enough to crash MongoDB Compass. The decoded CQL is already persisted in the
+   * dedicated {@code cqlContent} field, so dropping the whole array here is safe.
+   */
+  private String stripContentData(JsonNode root) {
+    ObjectNode copy = root.deepCopy().asObject();
+    copy.remove("content");
+    return copy.toString();
+  }
+
+  /** Returns {@code true} if the resource has {@code type.coding[*].code == "logic-library"}. */
+  private boolean hasLogicLibraryType(JsonNode root) {
+    JsonNode codings = root.path("type").path("coding");
+    if (codings.isMissingNode() || !codings.isArray()) {
+      return false;
+    }
+    for (JsonNode coding : codings) {
+      if ("logic-library".equals(coding.path("code").asString(null))) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Finds the first {@code content} entry whose {@code contentType} is {@code "text/cql"} and
+   * returns the base64-decoded CQL string. Returns {@code null} if no such entry exists.
+   */
+  private String extractCqlContent(JsonNode root) {
+    JsonNode contentArray = root.path("content");
+    if (contentArray.isMissingNode() || !contentArray.isArray()) {
+      return null;
+    }
+    for (JsonNode content : contentArray) {
+      if ("text/cql".equals(content.path("contentType").asString(null))) {
+        String base64Data = content.path("data").asString(null);
+        if (base64Data != null) {
+          try {
+            return new String(Base64.getDecoder().decode(base64Data), StandardCharsets.UTF_8);
+          } catch (IllegalArgumentException e) {
+            log.warn("Failed to base64-decode CQL content: {}", e.getMessage());
+            return null;
+          }
+        }
+        // data field absent but contentType matches – treat as no CQL available
+        return null;
+      }
+    }
+    return null;
+  }
+}