Make indexing behaviour if document exists configurable.

That is, if a document with the same persistent identifier (pidField)
is already in the index, do we fail, replace it or skip it?

A pidField must be configured for this, otherwise the document is always
added (so you may get duplicate documents).

Configure the behaviour in blacklab[-server].yaml in the indexing section
with the ifDocumentExists setting (valid values: fail, replace, skip) or
by passing --ifexists <value> to IndexTool.

Author: jan-niestadt

engine/src/main/java/nl/inl/blacklab/config/BLConfigIndexing.java

@@ -1,5 +1,6 @@
 package nl.inl.blacklab.config;
 
+import nl.inl.blacklab.search.BlackLabIndexWriter;
 import nl.inl.util.DownloadCache;
 
 public class BLConfigIndexing {
@@ -36,6 +37,11 @@ public class BLConfigIndexing {
     @Deprecated
     int maxMetadataValuesToStore = 0;
 
+    /** What to do if a document already exists in the index (i.e. has the same persistent identifier):
+     * fail (default), replace (i.e. upsert) or skip?
+     */
+    BlackLabIndexWriter.IfDocumentExists ifDocumentExists = BlackLabIndexWriter.IfDocumentExists.FAIL;
+
     public DownloadCache.Config downloadCacheConfig() {
         return new DownloadCache.Config() {
             @Override
@@ -155,7 +161,11 @@ public void setMaxValueLength(int maxValueLength) {
         this.maxValueLength = maxValueLength;
     }
 
-    public void setMaxMetadataValuesToStore(int maxMetadataValuesToStore) {
-        this.maxMetadataValuesToStore = maxMetadataValuesToStore;
+    public BlackLabIndexWriter.IfDocumentExists getIfDocumentExists() {
+        return this.ifDocumentExists;
+    }
+
+    public void setIfDocumentExists(BlackLabIndexWriter.IfDocumentExists ifDocumentExists) {
+        this.ifDocumentExists = ifDocumentExists;
     }
 }

engine/src/main/java/nl/inl/blacklab/index/BLIndexWriterProxyLucene.java

@@ -92,7 +92,8 @@ public void addDocument(BLInputDocument document) throws IOException {
                     break;
                 case FAIL:
                     throw new ErrorIndexingFile("Document with pid '" + pid +
-                            "' already exists in corpus; cannot add document: " + document);
+                            "' already exists in corpus; cannot add it again " +
+                            "(ifDocumentExists setting set to 'fail'; set to 'replace' to upsert instead)");
                 default:
                     throw new IllegalArgumentException();
                 }

engine/src/main/java/nl/inl/blacklab/search/BlackLabIndexWriter.java

@@ -7,6 +7,9 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
 import nl.inl.blacklab.index.BLIndexObjectFactory;
 import nl.inl.blacklab.index.BLIndexWriterProxy;
 import nl.inl.blacklab.index.BLInputDocument;
@@ -27,12 +30,35 @@ enum IfDocumentExists {
         SKIP,
 
         /** Fail with an error message. */
-        FAIL,
+        FAIL;
+
+        @JsonCreator
+        public static IfDocumentExists forValue(String ifDocumentExists) {
+            switch (ifDocumentExists.toLowerCase()) {
+                case "upsert", "replace", "overwrite" -> {
+                    return UPSERT;
+                }
+                case "skip" -> {
+                    return SKIP;
+                }
+                case "fail" -> {
+                    return FAIL;
+                }
+                default -> throw new IllegalArgumentException("Unknown IfDocumentExists value: " + ifDocumentExists + "(valid values: fail, replace or skip)");
+            }
+        }
+
+        @JsonValue
+        @Override
+        public String toString() {
+            return super.toString().toLowerCase();
+        }
     }
 
     /** What to do if a document with the same persistent identifier (pidField) already exists? */
     default IfDocumentExists getIfDocumentExists() {
-        return IfDocumentExists.UPSERT; // TODO: make configurable (IndexTool cmdline, BLS config)
+        return BlackLab.config().getIndexing().getIfDocumentExists();
+        //return IfDocumentExists.UPSERT; // TODO: make configurable (IndexTool cmdline, BLS config)
     }
 
     /**

site/docs/server/050_configuration.md

@@ -331,13 +331,24 @@ You probably won't need to change these settings, but they are available for adv
 ```yaml
 indexing:
     # (...above settings...)
+
+    # What to do if a document with the same persistent identifier
+    # already exists in the index.
+    # Options: fail (default), replace, skip
+    # (note that this will never apply if no pidField has been configured!)
+    ifDocumentExists: fail
     
     # Should inline tags and relations be indexed case- and accent-sensitively?
     # This used to be the default, but we've switched over to case-insensitive
     # indexing by default. Set to true to revert to the old behavior.
     # (default: false)
     relationsSensitive: false
 
+    # Max. length of a value of an annotation value (i.e. word, lemma)
+    # or 0 for no limit (Lucene has a limit of 32766 characters for a term though;
+    # longer values will cause an error)
+    maxValueLength: 0
+
     # Are http downloads of e.g. metadata allowed?
     # (default: false)
     downloadAllowed: false

tools/src/main/java/nl/inl/blacklab/tools/IndexTool.java

@@ -147,6 +147,23 @@ public static void main(String[] args) throws ErrorOpeningIndex, ParseException,
                             return;
                         }
                     }
+                    case "ifexists" -> {
+                        if (i + 1 == args.length) {
+                            System.err.println("--ifexists option needs argument");
+                            usage();
+                            return;
+                        }
+                        try {
+                            BlackLabIndexWriter.IfDocumentExists ifExists = BlackLabIndexWriter.IfDocumentExists.forValue(
+                                    args[i + 1]);
+                            BlackLab.config().getIndexing().setIfDocumentExists(ifExists);
+                            i++;
+                        } catch (IllegalArgumentException e) {
+                            System.err.println("--ifexists option needs valid argument (fail, replace or skip)");
+                            usage();
+                            return;
+                        }
+                    }
                     case "help" -> {
                         usage();
                         return;
@@ -243,19 +260,11 @@ public static void main(String[] args) throws ErrorOpeningIndex, ParseException,
         // Init log4j
         LogUtil.setupBasicLoggingConfig();
 
-        List<File> dirs = new ArrayList<>(List.of(new File(".")));
         Optional<File> inputDir = indexSource.getAssociatedDirectory();
         File inputDirParent = null;
-        if (inputDir.isPresent()) {
-            dirs.add(inputDir.get());
+        if (inputDir.isPresent())
             inputDirParent = inputDir.get().getAbsoluteFile().getParentFile();
-        }
-        if (inputDirParent != null)
-            dirs.add(inputDirParent);
-        dirs.add(indexDir);
         File indexDirParent = indexDir.getAbsoluteFile().getParentFile();
-        if (indexDirParent != null)
-            dirs.add(indexDirParent);
 
         String op = forceCreateNew ? "Creating new" : "Appending to";
         System.out.println(op + " index in " + indexDir + File.separator + " from " + indexSource +