Merge pull request #304 from NASA-PDS/bugfix/342-ldd-conflict-logging

Fix false ERROR log when loading older LDD over existing newer version

Author: jordanpadams

CLAUDE.md

@@ -114,6 +114,16 @@ The library provides a unified `Request` and `Response` interface that abstracts
 - Builds publish to Maven Central via Sonatype Central Portal
 - Requires secrets: `CENTRAL_REPOSITORY_USERNAME`, `CENTRAL_REPOSITORY_TOKEN`, `CODE_SIGNING_KEY`
 
+## Coding Style
+
+All Java source files must follow the [Google Java Style Guide](https://google.github.io/styleguide/javaguide.html). Key rules:
+
+- **Indentation**: 2 spaces (no tabs)
+- **Column limit**: 100 characters
+- **Braces**: always used for blocks, opening brace on same line
+- **Imports**: no wildcard imports; ordered as static imports first, then standard library, then third-party, then project imports, each group separated by a blank line
+- **Naming**: `UpperCamelCase` for classes, `lowerCamelCase` for methods and variables, `UPPER_SNAKE_CASE` for constants
+
 ## Critical Invariants
 
 **Schema field must exist before metadata is loaded:**

src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java

@@ -38,6 +38,9 @@ public class DataLoader {
   private int printProgressSize = 500;
   private int batchSize = 100;
   private int totalRecords;
+  // When true, 409 Conflict on "create" is treated as success (document already exists).
+  // Safe for idempotent LDD loads; leave false for product ingestion where duplicates are errors.
+  private boolean ignoreConflicts = false;
 
   private Logger log;
   private ConnectionFactory conFactory;
@@ -55,9 +58,19 @@ public DataLoader(ConnectionFactory conFactory) throws Exception {
   }
 
 
+  /**
+   * When set to true, 409 Conflict responses on "create" operations are treated as success.
+   * Use for idempotent loads (e.g. LDD ingestion) where a pre-existing document is acceptable.
+   * Default is false; keep false for product ingestion.
+   */
+  public void setIgnoreConflicts(boolean ignoreConflicts) {
+    this.ignoreConflicts = ignoreConflicts;
+  }
+
+
   /**
    * Set data batch size
-   * 
+   *
    * @param size batch size
    */
   public void setBatchSize(int size) {
@@ -184,8 +197,13 @@ private String loadBatch(BufferedReader fileReader, String firstLine, int retrie
       totalRecords += uploaded;
 
       if (uploaded != numRecords) {
+        // When ignoreConflicts=true, 409s are not counted as errors so uploaded==numRecords
+        // and this branch is not reached.  When ignoreConflicts=false (default, product
+        // ingestion), 409s ARE counted as failures, so a shortfall here means real write
+        // failures that should be investigated.
         throw new Exception(
-            "Failed to upload all documents (" + uploaded + "/" + numRecords + ") to -dd");
+            "Failed to upload " + (numRecords - uploaded) + "/" + numRecords + " documents to index '"
+                + conFactory.getIndexName() + "'. Check ERROR lines above for per-document reasons.");
       }
       return line1;
     } catch (UnknownHostException ex) {
@@ -306,9 +324,14 @@ private int processErrors(Response.Bulk resp, Set<String> errorLidvids,
     if (resp.errors()) {
       for (Response.Bulk.Item item : resp.items()) {
         if (item.error()) {
-          if (item.operation().equals("create") && item.status() == 409) { // already exists
+          if (item.operation().equals("create") && item.status() == 409) {
             todo.remove(asKey(item));
-            numErrors++;
+            if (ignoreConflicts) {
+              log.debug("Document '{}' already exists in '{}', skipping (create 409).",
+                  item.id(), conFactory.getIndexName());
+            } else {
+              numErrors++;
+            }
           } else {
             String message = item.reason();
             String sanitizedLidvid = item.id().replace('\r', ' ').replace('\n', ' '); // protect vs

src/main/java/gov/nasa/pds/registry/common/es/dao/dd/LddVersions.java

@@ -13,7 +13,8 @@
  */
 public class LddVersions
 {
-    private static final String DEFAULT_DATE = "1965-01-01T00:00:00.000Z";
+    public static final String DEFAULT_DATE = "1965-01-01T00:00:00.000Z";
+    public static final Instant DEFAULT_LAST_DATE = Instant.parse(DEFAULT_DATE);
 
     public Set<String> files;
     public Instant lastDate;

src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java

@@ -49,6 +49,7 @@ public JsonLddLoader(DataDictionaryDao dao, ConnectionFactory conFact) throws Ex
     dtMap = new Pds2EsDataTypeMap();
 
     loader = new DataLoader(conFact.clone().setIndexName(conFact.getIndexName() + "-dd"));
+    loader.setIgnoreConflicts(true);
     this.dao = dao;
   }
 
@@ -238,6 +239,22 @@ private String createEsDataFile(File lddFile, String lddFileName, String namespa
 
     // If this LDD date is after the last stored in Elasticsearch, overwrite old records
     boolean overwrite = overwriteLdd(lastDate, attrParser.getLddDate());
+    if (!overwrite && !lastDate.equals(LddVersions.DEFAULT_LAST_DATE)) {
+      // Only log "not newer" when the date was parseable — overwriteLdd() returns false on
+      // parse failure too, but in that case it already logged a warn about the bad date string.
+      boolean dateWasParseable;
+      try {
+        LddUtils.lddDateToIsoInstant(attrParser.getLddDate());
+        dateWasParseable = true;
+      } catch (Exception ex) {
+        dateWasParseable = false;
+      }
+      if (dateWasParseable) {
+        log.debug("LDD {} (dated {}) is not newer than the version already loaded in the registry (dated {})."
+            + " Existing field definitions for namespace '{}' will be used.",
+            lddFileName, attrParser.getLddDate(), lastDate, namespace);
+      }
+    }
 
     // Create a writer to save LDD data in Elasticsearch JSON data file
     LddEsJsonWriter writer = null;

src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java

@@ -103,8 +103,8 @@ public void updateSchema(Set<String> fields, Map<String, String> xsds) throws Ex
         } catch (LddException ex) {
           throw ex;
         } catch (Exception ex) {
-          log.error("Could not update LDD for namespace '" + prefix + "' at URI " + uri
-              + ": " + ex.getMessage() + ". Harvesting will continue with available field definitions.");
+          log.warn("Could not update LDD for namespace '{}' at URI {}: {}. Harvesting will continue with available field definitions.",
+              prefix, uri, ex.getMessage(), ex);
         }
       }
     }
@@ -203,17 +203,17 @@ private void updateLdd(String uri, String prefix) throws LddException {
       throw ex;
     } catch (InterruptedException ex) {
       Thread.currentThread().interrupt();
-      log.error("Interrupted while downloading or loading LDD for namespace '" + prefix + "' from " + jsonUrl);
       if (lddInfo.isEmpty()) {
+        log.error("Interrupted while downloading or loading LDD for namespace '{}' from {} and no previously loaded version exists.",
+            prefix, jsonUrl);
         if (!forceLoad) {
           throw new LddException("No previously loaded LDD found for namespace '" + prefix
               + "'. Cannot load products with fields from this namespace.");
         }
-        log.warn("Force mode: no LDD found for namespace '" + prefix
-            + "'. Fields from this namespace will not be indexed.");
+        log.warn("Force mode: no LDD found for namespace '{}'. Fields from this namespace will not be indexed.", prefix);
       } else {
-        log.warn("Will use previously loaded field definitions for namespace '" + prefix
-            + "' from " + lddInfo.files);
+        log.warn("Interrupted while loading LDD {} for namespace '{}'. Will use previously loaded field definitions from {}.",
+            schemaFileName, prefix, lddInfo.files);
       }
       return;
     } catch (Exception ex) {
@@ -236,18 +236,17 @@ private void updateLdd(String uri, String prefix) throws LddException {
         }
         if (mirrorSuccess) return;
       }
-      log.error("Failed to download or load LDD for namespace '" + prefix + "' from " + jsonUrl
-          + ": " + ExceptionUtils.getMessage(ex));
       if (lddInfo.isEmpty()) {
+        log.error("Failed to download or load LDD for namespace '{}' from {}: {}",
+            prefix, jsonUrl, ExceptionUtils.getMessage(ex));
         if (!forceLoad) {
           throw new LddException("No previously loaded LDD found for namespace '" + prefix
               + "'. Cannot load products with fields from this namespace.");
         }
-        log.warn("Force mode: no LDD found for namespace '" + prefix
-            + "'. Fields from this namespace will not be indexed.");
+        log.warn("Force mode: no LDD found for namespace '{}'. Fields from this namespace will not be indexed.", prefix);
       } else {
-        log.warn("Will use previously loaded field definitions for namespace '" + prefix
-            + "' from " + lddInfo.files);
+        log.warn("Failed to load LDD {} for namespace '{}': {}. Will use previously loaded field definitions from {}.",
+            schemaFileName, prefix, ExceptionUtils.getMessage(ex), lddInfo.files);
       }
     } finally {
       lddFile.delete();