Skip to content

Commit 73b531c

Browse files
ivakegghgklohr
ivakegg
authored and
hgklohr
committed
fixes #3459: added disableIteratorMostRecentUniqueFields mechanism (#3460)
(true by default pending some addigiona yielding work) to disable using the tserver side most-recent-unique functionality and rely completely on the webserver side filtering
1 parent 3938a04 commit 73b531c

7 files changed

Lines changed: 102 additions & 32 deletions

File tree

‎warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -406,6 +406,7 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
406406
private boolean accrueStats = false;
407407

408408
private boolean disableIteratorUniqueFields = false;
409+
private boolean disableIteratorMostRecentUniqueFields = true;
409410
private UniqueFields uniqueFields = new UniqueFields();
410411
private boolean cacheModel = false;
411412
/**
@@ -797,6 +798,7 @@ public void copyFrom(ShardQueryConfiguration other) {
797798
this.setGroupFieldsBatchSize(other.getGroupFieldsBatchSize());
798799
this.setAccrueStats(other.getAccrueStats());
799800
this.setDisableIteratorUniqueFields(other.isDisableIteratorUniqueFields());
801+
this.setDisableIteratorMostRecentUniqueFields(other.isDisableIteratorMostRecentUniqueFields());
800802
this.setUniqueFields(other.getUniqueFields());
801803
log.info("Checkpointing with " + getUniqueFields());
802804
this.setUniqueCacheBufferSize(other.getUniqueCacheBufferSize());
@@ -2025,6 +2027,14 @@ public void setDisableIteratorUniqueFields(boolean disableIteratorUniqueFields)
20252027
this.disableIteratorUniqueFields = disableIteratorUniqueFields;
20262028
}
20272029

2030+
public boolean isDisableIteratorMostRecentUniqueFields() {
2031+
return disableIteratorMostRecentUniqueFields;
2032+
}
2033+
2034+
public void setDisableIteratorMostRecentUniqueFields(boolean disableIteratorMostRecentUniqueFields) {
2035+
this.disableIteratorMostRecentUniqueFields = disableIteratorMostRecentUniqueFields;
2036+
}
2037+
20282038
public UniqueFields getUniqueFields() {
20292039
return uniqueFields;
20302040
}
@@ -3224,6 +3234,7 @@ public boolean equals(Object o) {
32243234
getMaxLinesToPrint() == that.getMaxLinesToPrint() &&
32253235
getMaxAnyFieldScanTimeMillis() == that.getMaxAnyFieldScanTimeMillis() &&
32263236
isDisableIteratorUniqueFields() == that.isDisableIteratorUniqueFields() &&
3237+
isDisableIteratorMostRecentUniqueFields() == that.isDisableIteratorMostRecentUniqueFields() &&
32273238
isUseShardedIndex() == that.isUseShardedIndex() &&
32283239
getDayIndexThreshold() == that.getDayIndexThreshold() &&
32293240
isUseTruncatedIndex() == that.isUseTruncatedIndex() &&
@@ -3461,6 +3472,7 @@ public int hashCode() {
34613472
getMaxLinesToPrint(),
34623473
getMaxAnyFieldScanTimeMillis(),
34633474
isDisableIteratorUniqueFields(),
3475+
isDisableIteratorMostRecentUniqueFields(),
34643476
isUseShardedIndex(),
34653477
getDayIndexThreshold(),
34663478
isUseTruncatedIndex(),

‎warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -709,7 +709,7 @@ private void configureIterator(ShardQueryConfiguration config, IteratorSetting c
709709
addOption(cfg, QueryOptions.MATCHING_FIELD_SETS, config.getMatchingFieldSetsAsString(), false);
710710
addOption(cfg, QueryOptions.GROUP_FIELDS, config.getGroupFields().toString(), true);
711711
addOption(cfg, QueryOptions.GROUP_FIELDS_BATCH_SIZE, config.getGroupFieldsBatchSizeAsString(), true);
712-
if (!config.isDisableIteratorUniqueFields()) {
712+
if (!config.isDisableIteratorUniqueFields() && !(config.isDisableIteratorMostRecentUniqueFields() && config.getUniqueFields().isMostRecent())) {
713713
addOption(cfg, QueryOptions.UNIQUE_FIELDS, config.getUniqueFields().toString(), true);
714714
if (config.getUniqueFields().isMostRecent()) {
715715
// this may be redundant with the uniqueFields.toString(), but other code relies on this explicitly being set

‎warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1908,6 +1908,14 @@ public void setDisableIteratorUniqueFields(boolean disableIteratorUniqueFields)
19081908
getConfig().setDisableIteratorUniqueFields(disableIteratorUniqueFields);
19091909
}
19101910

1911+
public boolean isDisableIteratorMostRecentUniqueFields() {
1912+
return getConfig().isDisableIteratorMostRecentUniqueFields();
1913+
}
1914+
1915+
public void setDisableIteratorMostRecentUniqueFields(boolean disableIteratorMostRecentUniqueFields) {
1916+
getConfig().setDisableIteratorMostRecentUniqueFields(disableIteratorMostRecentUniqueFields);
1917+
}
1918+
19111919
public UniqueFields getUniqueFields() {
19121920
return getConfig().getUniqueFields();
19131921
}

‎warehouse/query-core/src/main/java/datawave/query/util/sortedmap/BufferedFileBackedSortedMap.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -305,13 +305,13 @@ public void compact(int maxFiles) throws IOException {
305305

306306
// compact it
307307
if (log.isDebugEnabled()) {
308-
log.debug("Starting compaction for " + mapToCompact);
308+
log.debug("Starting compaction for " + mapsPerCompaction + " maps");
309309
}
310310
long start = System.currentTimeMillis();
311311
FileSortedMap<K,V> compaction = compact(mapToCompact);
312312
if (log.isDebugEnabled()) {
313313
long delta = System.currentTimeMillis() - start;
314-
log.debug("Compacted " + mapToCompact + " -> " + compaction + " in " + delta + "ms");
314+
log.debug("Compacted " + mapsPerCompaction + " maps in " + delta + "ms");
315315
}
316316

317317
// add the compacted map to our final multimap

‎warehouse/query-core/src/main/java/datawave/query/util/sortedset/BufferedFileBackedSortedSet.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -344,13 +344,13 @@ public void compact(int maxFiles) throws IOException {
344344

345345
// compact it
346346
if (log.isDebugEnabled()) {
347-
log.debug("Starting compaction for " + setToCompact);
347+
log.debug("Starting compaction for " + setsPerCompaction + " sets");
348348
}
349349
long start = System.currentTimeMillis();
350350
FileSortedSet<E> compaction = compact(setToCompact);
351351
if (log.isDebugEnabled()) {
352352
long delta = System.currentTimeMillis() - start;
353-
log.debug("Compacted " + setToCompact + " -> " + compaction + " in " + delta + "ms");
353+
log.debug("Compacted " + setsPerCompaction + " sets in " + delta + "ms");
354354
}
355355

356356
// add the compacted set to our final multiset

‎warehouse/query-core/src/test/java/datawave/query/UniqueTest.java‎

Lines changed: 75 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -402,39 +402,87 @@ public void testRecentUniquenessWithModelAliases() throws Exception {
402402
}
403403

404404
@Test
405-
public void testMostRecentUniqueness() throws Exception {
406-
Map<String,String> extraParameters = new HashMap<>();
407-
extraParameters.put("include.grouping.context", "true");
408-
extraParameters.put(QueryParameters.MOST_RECENT_UNIQUE, "true");
405+
public void testMostRecentUniquenessWebServerOnly() throws Exception {
406+
boolean originalSetting = logic.isDisableIteratorMostRecentUniqueFields();
407+
try {
408+
logic.setDisableIteratorMostRecentUniqueFields(true);
409+
Map<String,String> extraParameters = new HashMap<>();
410+
extraParameters.put("include.grouping.context", "true");
411+
extraParameters.put(QueryParameters.MOST_RECENT_UNIQUE, "true");
409412

410-
Date startDate = format.parse("20091231");
411-
Date endDate = format.parse("20150101");
413+
Date startDate = format.parse("20091231");
414+
Date endDate = format.parse("20150101");
412415

413-
String queryString = "UUID =~ '^[CS].*'";
416+
String queryString = "UUID =~ '^[CS].*'";
414417

415-
Set<Set<String>> expected = new HashSet<>();
416-
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
417-
extraParameters.put("unique.fields", "DEATH_DATE,$MAGIC");
418-
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
418+
Set<Set<String>> expected = new HashSet<>();
419+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
420+
extraParameters.put("unique.fields", "DEATH_DATE,$MAGIC");
421+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
419422

420-
expected.clear();
421-
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
422-
extraParameters.put("unique.fields", "death_date,$magic");
423-
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
423+
expected.clear();
424+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
425+
extraParameters.put("unique.fields", "death_date,$magic");
426+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
424427

425-
expected.clear();
426-
expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID));
427-
expected.add(Sets.newHashSet(WiseGuysIngest.corleoneUID));
428-
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
429-
extraParameters.put("unique.fields", "$DEATH_DATE,BIRTH_DATE");
430-
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
428+
expected.clear();
429+
expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID));
430+
expected.add(Sets.newHashSet(WiseGuysIngest.corleoneUID));
431+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
432+
extraParameters.put("unique.fields", "$DEATH_DATE,BIRTH_DATE");
433+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
431434

432-
expected.clear();
433-
expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID));
434-
expected.add(Sets.newHashSet(WiseGuysIngest.corleoneUID));
435-
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
436-
extraParameters.put("unique.fields", "death_date,birth_date");
437-
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
435+
expected.clear();
436+
expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID));
437+
expected.add(Sets.newHashSet(WiseGuysIngest.corleoneUID));
438+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
439+
extraParameters.put("unique.fields", "death_date,birth_date");
440+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
441+
} finally {
442+
logic.setDisableIteratorMostRecentUniqueFields(originalSetting);
443+
}
444+
}
445+
446+
@Test
447+
public void testMostRecentUniquenessWithIterator() throws Exception {
448+
boolean originalSetting = logic.isDisableIteratorMostRecentUniqueFields();
449+
try {
450+
logic.setDisableIteratorMostRecentUniqueFields(false);
451+
Map<String,String> extraParameters = new HashMap<>();
452+
extraParameters.put("include.grouping.context", "true");
453+
extraParameters.put(QueryParameters.MOST_RECENT_UNIQUE, "true");
454+
455+
Date startDate = format.parse("20091231");
456+
Date endDate = format.parse("20150101");
457+
458+
String queryString = "UUID =~ '^[CS].*'";
459+
460+
Set<Set<String>> expected = new HashSet<>();
461+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
462+
extraParameters.put("unique.fields", "DEATH_DATE,$MAGIC");
463+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
464+
465+
expected.clear();
466+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
467+
extraParameters.put("unique.fields", "death_date,$magic");
468+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
469+
470+
expected.clear();
471+
expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID));
472+
expected.add(Sets.newHashSet(WiseGuysIngest.corleoneUID));
473+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
474+
extraParameters.put("unique.fields", "$DEATH_DATE,BIRTH_DATE");
475+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
476+
477+
expected.clear();
478+
expected.add(Sets.newHashSet(WiseGuysIngest.sopranoUID));
479+
expected.add(Sets.newHashSet(WiseGuysIngest.corleoneUID));
480+
expected.add(Sets.newHashSet(WiseGuysIngest.caponeUID));
481+
extraParameters.put("unique.fields", "death_date,birth_date");
482+
runTestQueryWithUniqueness(expected, queryString, startDate, endDate, extraParameters);
483+
} finally {
484+
logic.setDisableIteratorMostRecentUniqueFields(originalSetting);
485+
}
438486
}
439487

440488
@Test

‎warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,8 @@ public void setUp() throws Exception {
461461
updatedValues.put("compositeFilterFunctionsEnabled", true);
462462
defaultValues.put("disableIteratorUniqueFields", false);
463463
updatedValues.put("disableIteratorUniqueFields", true);
464+
defaultValues.put("disableIteratorMostRecentUniqueFields", true);
465+
updatedValues.put("disableIteratorMostRecentUniqueFields", false);
464466
defaultValues.put("uniqueFields", new UniqueFields());
465467
updatedValues.put("uniqueFields", UniqueFields.from("FIELD_U,FIELD_V"));
466468
defaultValues.put("uniqueCacheBufferSize", 100);

0 commit comments

Comments
 (0)