Skip to content

Commit 1233026

Browse files
FineAndDandyhgklohr
FineAndDandy
authored and
hgklohr
committed
Restructure AllHits response (#3409)
1 parent 73b531c commit 1233026

6 files changed

Lines changed: 115 additions & 84 deletions

File tree

‎warehouse/query-core/src/main/java/datawave/query/transformer/annotation/AllHitsFactory.java‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
import java.util.Iterator;
77
import java.util.List;
88
import java.util.Map;
9+
import java.util.Set;
910
import java.util.SortedMap;
1011
import java.util.TreeMap;
12+
import java.util.TreeSet;
1113
import java.util.concurrent.TimeUnit;
1214

1315
import datawave.annotation.protobuf.v1.SegmentBoundary;
@@ -24,6 +26,8 @@
2426
* TimeUnit
2527
*/
2628
public class AllHitsFactory {
29+
public static final TermHitComparator TERM_HIT_COMPARATOR = new TermHitComparator();
30+
2731
/**
2832
* Convenience method for MILLIS
2933
*
@@ -143,13 +147,14 @@ private float convertTime(long time, TimeUnit to) {
143147
return (float) timeInNanos / (float) targetInNanos;
144148
}
145149

146-
private void applyHit(AllHit allHit, SegmentBoundary segmentBoundary, SegmentValue hitValue, TimeUnit timeUnit) {
150+
private void applyHit(AllHit allHit, Term term, SegmentValue hitValue, boolean best) {
147151
TermHit th = new TermHit();
148-
th.setTermLabel(hitValue.getValue());
152+
th.setLabel(hitValue.getValue());
149153
th.setConfidence(hitValue.getScore());
150-
th.getTimeRange().setStartTime(convertTime(segmentBoundary.getStart(), timeUnit));
151-
th.getTimeRange().setEndTime(convertTime(segmentBoundary.getEnd(), timeUnit));
152-
allHit.getTermHits().add(th);
154+
th.setOneBest(best);
155+
156+
Set<TermHit> hits = term.getHits();
157+
hits.add(th);
153158

154159
// rollup confidence
155160
if (allHit.getConfidence() < th.getConfidence()) {
@@ -184,7 +189,8 @@ private void applyContextAndHit(AllHit allHit, MergedHit mergedHit, Iterator<Map
184189
t.setConfidence(firstValue.getScore());
185190
t.getTimeRange().setStartTime(convertTime(boundary.getStart(), timeUnit));
186191
t.getTimeRange().setEndTime(convertTime(boundary.getEnd(), timeUnit));
187-
allHit.getOneBestContext().add(t);
192+
t.setHits(new TreeSet<>(TERM_HIT_COMPARATOR));
193+
allHit.getContext().add(t);
188194

189195
// now check if this segment also contains the hit
190196
List<Integer> hitIndexes = mergedHit.getHitIndexes(boundary);
@@ -195,7 +201,8 @@ private void applyContextAndHit(AllHit allHit, MergedHit mergedHit, Iterator<Map
195201
+ " index:" + hitIndex);
196202
}
197203
SegmentValue value = contextEntry.getValue().get(hitIndex);
198-
applyHit(allHit, boundary, value, timeUnit);
204+
boolean best = contextEntry.getValue().size() - 1 == hitIndex;
205+
applyHit(allHit, t, value, best);
199206
}
200207
}
201208
}

‎warehouse/query-core/src/main/java/datawave/query/transformer/annotation/TermHitComparator.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package datawave.query.transformer.annotation;
2+
3+
import java.util.Comparator;
4+
5+
import datawave.query.transformer.annotation.model.TermHit;
6+
7+
/**
8+
* Compares TermHits by confidence, sorting in descending order
9+
*/
10+
public class TermHitComparator implements Comparator<TermHit> {
11+
@Override
12+
public int compare(TermHit o1, TermHit o2) {
13+
return Float.compare(o1.getConfidence(), o2.getConfidence());
14+
}
15+
}

‎warehouse/query-core/src/main/java/datawave/query/transformer/annotation/model/AllHit.java‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,10 @@ public class AllHit {
1010
private float confidence;
1111

1212
@JsonProperty
13-
private List<Term> oneBestContext = new ArrayList<>();
13+
private List<Term> context = new ArrayList<>();
1414

15-
@JsonProperty
16-
private List<TermHit> termHits = new ArrayList<>();
17-
18-
public List<Term> getOneBestContext() {
19-
return oneBestContext;
20-
}
21-
22-
public List<TermHit> getTermHits() {
23-
return termHits;
15+
public List<Term> getContext() {
16+
return context;
2417
}
2518

2619
public float getConfidence() {

‎warehouse/query-core/src/main/java/datawave/query/transformer/annotation/model/Term.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package datawave.query.transformer.annotation.model;
22

3+
import java.util.Set;
4+
35
import com.fasterxml.jackson.annotation.JsonProperty;
46

57
public class Term {
@@ -9,6 +11,9 @@ public class Term {
911
@JsonProperty
1012
private float confidence;
1113

14+
@JsonProperty
15+
private Set<TermHit> hits;
16+
1217
@JsonProperty
1318
private TimeRange timeRange = new TimeRange();
1419

@@ -31,4 +36,12 @@ public float getConfidence() {
3136
public void setConfidence(float confidence) {
3237
this.confidence = confidence;
3338
}
39+
40+
public Set<TermHit> getHits() {
41+
return hits;
42+
}
43+
44+
public void setHits(Set<TermHit> hits) {
45+
this.hits = hits;
46+
}
3447
}

‎warehouse/query-core/src/main/java/datawave/query/transformer/annotation/model/TermHit.java‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,20 @@
44

55
public class TermHit {
66
@JsonProperty
7-
private String termLabel;
7+
private String label;
88

99
@JsonProperty
1010
private float confidence;
1111

1212
@JsonProperty
13-
private TimeRange timeRange = new TimeRange();
13+
private boolean oneBest;
1414

15-
public String getTermLabel() {
16-
return termLabel;
15+
public String getLabel() {
16+
return label;
1717
}
1818

19-
public void setTermLabel(String termLabel) {
20-
this.termLabel = termLabel;
19+
public void setLabel(String label) {
20+
this.label = label;
2121
}
2222

2323
public float getConfidence() {
@@ -28,7 +28,11 @@ public void setConfidence(float confidence) {
2828
this.confidence = confidence;
2929
}
3030

31-
public TimeRange getTimeRange() {
32-
return timeRange;
31+
public boolean isOneBest() {
32+
return oneBest;
33+
}
34+
35+
public void setOneBest(boolean oneBest) {
36+
this.oneBest = oneBest;
3337
}
3438
}

‎warehouse/query-core/src/test/java/datawave/query/transformer/annotation/AllHitsFactoryTest.java‎

Lines changed: 58 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
package datawave.query.transformer.annotation;
22

3+
import static datawave.query.transformer.annotation.AllHitsFactory.TERM_HIT_COMPARATOR;
34
import static org.junit.jupiter.api.Assertions.assertEquals;
45
import static org.junit.jupiter.api.Assertions.assertNull;
56
import static org.junit.jupiter.api.Assertions.assertThrows;
67

78
import java.util.List;
9+
import java.util.Set;
810
import java.util.TreeMap;
11+
import java.util.TreeSet;
912

1013
import org.junit.jupiter.api.BeforeEach;
1114
import org.junit.jupiter.api.Test;
@@ -107,8 +110,7 @@ public void singleHitOnSingleValueTest() throws AllHitsException, JsonProcessing
107110
expected.setMaxTermHitConfidence(.3f);
108111
AllHit hit1 = new AllHit();
109112
hit1.setConfidence(.3f);
110-
hit1.getOneBestContext().add(getTerm("hit", .3f, 3f, 4f));
111-
hit1.getTermHits().add(getTermHit("hit", .3f, 3f, 4f));
113+
hit1.getContext().add(addTermHit(getTerm("hit", .3f, 3f, 4f), getTermHit("hit", .3f, true)));
112114
expected.getKeywordResultList().add(hit1);
113115

114116
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -130,10 +132,11 @@ public void multipleHitOnSameTermTest() throws AllHitsException, JsonProcessingE
130132
expected.setMaxTermHitConfidence(.6f);
131133
AllHit hit1 = new AllHit();
132134
hit1.setConfidence(.6f);
133-
hit1.getOneBestContext().add(getTerm("hit2", .6f, 3f, 4f));
134-
hit1.getTermHits().add(getTermHit("hit1", .3f, 3f, 4f));
135-
hit1.getTermHits().add(getTermHit("hit2", .6f, 3f, 4f));
136-
hit1.getTermHits().add(getTermHit("hit3", .4f, 3f, 4f));
135+
Term t = getTerm("hit2", .6f, 3f, 4f);
136+
addTermHit(t, getTermHit("hit1", .3f, false));
137+
addTermHit(t, getTermHit("hit2", .6f, true));
138+
addTermHit(t, getTermHit("hit3", .4f, false));
139+
hit1.getContext().add(t);
137140
expected.getKeywordResultList().add(hit1);
138141

139142
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -156,12 +159,10 @@ public void doubleHitOnMultipleTermsTest() throws AllHitsException, JsonProcessi
156159
expected.setMaxTermHitConfidence(.6f);
157160
AllHit hit1 = new AllHit();
158161
hit1.setConfidence(.3f);
159-
hit1.getOneBestContext().add(getTerm("term1", .6f, 3f, 4f));
160-
hit1.getTermHits().add(getTermHit("hit1", .3f, 3f, 4f));
162+
hit1.getContext().add(addTermHit(getTerm("term1", .6f, 3f, 4f), getTermHit("hit1", .3f, false)));
161163
AllHit hit2 = new AllHit();
162164
hit2.setConfidence(.6f);
163-
hit2.getOneBestContext().add(getTerm("hit2", .6f, 4f, 5f));
164-
hit2.getTermHits().add(getTermHit("hit2", .6f, 4f, 5f));
165+
hit2.getContext().add(addTermHit(getTerm("hit2", .6f, 4f, 5f), getTermHit("hit2", .6f, true)));
165166
expected.getKeywordResultList().add(hit1);
166167
expected.getKeywordResultList().add(hit2);
167168

@@ -185,12 +186,10 @@ public void doubleHitOnMultipleTermsReverseOrderToVerifyAggregateConfidenceTest(
185186
expected.setMaxTermHitConfidence(.6f);
186187
AllHit hit1 = new AllHit();
187188
hit1.setConfidence(.3f);
188-
hit1.getOneBestContext().add(getTerm("term1", .6f, 4f, 5f));
189-
hit1.getTermHits().add(getTermHit("hit1", .3f, 4f, 5f));
189+
hit1.getContext().add(addTermHit(getTerm("term1", .6f, 4f, 5f), getTermHit("hit1", .3f, false)));
190190
AllHit hit2 = new AllHit();
191191
hit2.setConfidence(.6f);
192-
hit2.getOneBestContext().add(getTerm("hit2", .6f, 3f, 4f));
193-
hit2.getTermHits().add(getTermHit("hit2", .6f, 3f, 4f));
192+
hit2.getContext().add(addTermHit(getTerm("hit2", .6f, 3f, 4f), getTermHit("hit2", .6f, true)));
194193
expected.getKeywordResultList().add(hit2);
195194
expected.getKeywordResultList().add(hit1);
196195

@@ -210,13 +209,12 @@ public void hitWithContextTest() throws AllHitsException, JsonProcessingExceptio
210209
expected.setMaxTermHitConfidence(.3f);
211210
AllHit hit1 = new AllHit();
212211
hit1.setConfidence(.3f);
213-
hit1.getOneBestContext().add(getTerm("a2", .5f, 1f, 2f));
214-
hit1.getOneBestContext().add(getTerm("b2", .6f, 2f, 3f));
215-
hit1.getOneBestContext().add(getTerm("max", .7f, 3f, 4f));
216-
hit1.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
217-
hit1.getOneBestContext().add(getTerm("d1", .2f, 5f, 6f));
212+
hit1.getContext().add(getTerm("a2", .5f, 1f, 2f));
213+
hit1.getContext().add(getTerm("b2", .6f, 2f, 3f));
214+
hit1.getContext().add(addTermHit(getTerm("max", .7f, 3f, 4f), getTermHit("hit", .3f, false)));
215+
hit1.getContext().add(getTerm("c2", .8f, 4f, 5f));
216+
hit1.getContext().add(getTerm("d1", .2f, 5f, 6f));
218217

219-
hit1.getTermHits().add(getTermHit("hit", .3f, 3f, 4f));
220218
expected.getKeywordResultList().add(hit1);
221219

222220
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -238,14 +236,12 @@ public void overlappingHitContextTest() throws AllHitsException, JsonProcessingE
238236
expected.setMaxTermHitConfidence(.8f);
239237
AllHit allHit1 = new AllHit();
240238
allHit1.setConfidence(.8f);
241-
allHit1.getOneBestContext().add(getTerm("a2", .5f, 1f, 2f));
242-
allHit1.getOneBestContext().add(getTerm("b2", .6f, 2f, 3f));
243-
allHit1.getOneBestContext().add(getTerm("max", .7f, 3f, 4f));
244-
allHit1.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
245-
allHit1.getOneBestContext().add(getTerm("d1", .2f, 5f, 6f));
246-
247-
allHit1.getTermHits().add(getTermHit("hit", .3f, 3f, 4f));
248-
allHit1.getTermHits().add(getTermHit("c2", .8f, 4f, 5f));
239+
allHit1.getContext().add(getTerm("a2", .5f, 1f, 2f));
240+
allHit1.getContext().add(getTerm("b2", .6f, 2f, 3f));
241+
allHit1.getContext().add(addTermHit(getTerm("max", .7f, 3f, 4f), getTermHit("hit", .3f, false)));
242+
allHit1.getContext().add(addTermHit(getTerm("c2", .8f, 4f, 5f), getTermHit("c2", .8f, true)));
243+
allHit1.getContext().add(getTerm("d1", .2f, 5f, 6f));
244+
249245
expected.getKeywordResultList().add(allHit1);
250246

251247
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -267,14 +263,12 @@ public void overlappingBoundaryHitContextTest() throws AllHitsException, JsonPro
267263
expected.setMaxTermHitConfidence(.8f);
268264
AllHit allHit1 = new AllHit();
269265
allHit1.setConfidence(.8f);
270-
allHit1.getOneBestContext().add(getTerm("a2", .5f, 1f, 2f));
271-
allHit1.getOneBestContext().add(getTerm("b2", .6f, 2f, 3f));
272-
allHit1.getOneBestContext().add(getTerm("max", .7f, 3f, 4f));
273-
allHit1.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
274-
allHit1.getOneBestContext().add(getTerm("d1", .2f, 5f, 6f));
275-
276-
allHit1.getTermHits().add(getTermHit("hit", .3f, 3f, 4f));
277-
allHit1.getTermHits().add(getTermHit("c2", .8f, 4f, 5f));
266+
allHit1.getContext().add(getTerm("a2", .5f, 1f, 2f));
267+
allHit1.getContext().add(getTerm("b2", .6f, 2f, 3f));
268+
allHit1.getContext().add(addTermHit(getTerm("max", .7f, 3f, 4f), getTermHit("hit", .3f, false)));
269+
allHit1.getContext().add(addTermHit(getTerm("c2", .8f, 4f, 5f), getTermHit("c2", .8f, true)));
270+
allHit1.getContext().add(getTerm("d1", .2f, 5f, 6f));
271+
278272
expected.getKeywordResultList().add(allHit1);
279273

280274
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -297,21 +291,18 @@ public void overlappingContextButNotHitTest() throws AllHitsException, JsonProce
297291

298292
AllHit allHit1 = new AllHit();
299293
allHit1.setConfidence(.3f);
300-
allHit1.getOneBestContext().add(getTerm("a2", .5f, 1f, 2f));
301-
allHit1.getOneBestContext().add(getTerm("b2", .6f, 2f, 3f));
302-
allHit1.getOneBestContext().add(getTerm("max", .7f, 3f, 4f));
303-
allHit1.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
304-
305-
allHit1.getTermHits().add(getTermHit("hit", .3f, 3f, 4f));
294+
allHit1.getContext().add(getTerm("a2", .5f, 1f, 2f));
295+
allHit1.getContext().add(getTerm("b2", .6f, 2f, 3f));
296+
allHit1.getContext().add(addTermHit(getTerm("max", .7f, 3f, 4f), getTermHit("hit", .3f, false)));
297+
allHit1.getContext().add(getTerm("c2", .8f, 4f, 5f));
306298

307299
expected.getKeywordResultList().add(allHit1);
308300

309301
AllHit allHit2 = new AllHit();
310302
allHit2.setConfidence(.2f);
311-
allHit2.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
312-
allHit2.getOneBestContext().add(getTerm("d1", .2f, 5f, 6f));
303+
allHit2.getContext().add(getTerm("c2", .8f, 4f, 5f));
304+
allHit2.getContext().add(addTermHit(getTerm("d1", .2f, 5f, 6f), getTermHit("d1", .2f, true)));
313305

314-
allHit2.getTermHits().add(getTermHit("d1", .2f, 5f, 6f));
315306
expected.getKeywordResultList().add(allHit2);
316307

317308
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -338,22 +329,18 @@ public void overlappingContextAddedAtMostOnceTest() throws AllHitsException, Jso
338329

339330
AllHit allHit1 = new AllHit();
340331
allHit1.setConfidence(.3f);
341-
allHit1.getOneBestContext().add(getTerm("a2", .5f, 1f, 2f));
342-
allHit1.getOneBestContext().add(getTerm("b2", .6f, 2f, 3f));
343-
allHit1.getOneBestContext().add(getTerm("max", .7f, 3f, 4f));
344-
allHit1.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
345-
346-
allHit1.getTermHits().add(getTermHit("a1", .1f, 1f, 2f));
347-
allHit1.getTermHits().add(getTermHit("hit", .3f, 3f, 4f));
332+
allHit1.getContext().add(addTermHit(getTerm("a2", .5f, 1f, 2f), getTermHit("a1", .1f, false)));
333+
allHit1.getContext().add(getTerm("b2", .6f, 2f, 3f));
334+
allHit1.getContext().add(addTermHit(getTerm("max", .7f, 3f, 4f), getTermHit("hit", .3f, false)));
335+
allHit1.getContext().add(getTerm("c2", .8f, 4f, 5f));
348336

349337
expected.getKeywordResultList().add(allHit1);
350338

351339
AllHit allHit2 = new AllHit();
352340
allHit2.setConfidence(.2f);
353-
allHit2.getOneBestContext().add(getTerm("c2", .8f, 4f, 5f));
354-
allHit2.getOneBestContext().add(getTerm("d1", .2f, 5f, 6f));
341+
allHit2.getContext().add(getTerm("c2", .8f, 4f, 5f));
342+
allHit2.getContext().add(addTermHit(getTerm("d1", .2f, 5f, 6f), getTermHit("d1", .2f, true)));
355343

356-
allHit2.getTermHits().add(getTermHit("d1", .2f, 5f, 6f));
357344
expected.getKeywordResultList().add(allHit2);
358345

359346
assertEquals(objectMapper.writeValueAsString(expected), objectMapper.writeValueAsString(result));
@@ -382,17 +369,29 @@ private Term getTerm(String label, float confidence, float start, float end) {
382369
Term term = new Term();
383370
term.setLabel(label);
384371
term.setConfidence(confidence);
372+
term.setHits(new TreeSet<>(TERM_HIT_COMPARATOR));
385373
term.getTimeRange().setStartTime(start);
386374
term.getTimeRange().setEndTime(end);
387375
return term;
388376
}
389377

390-
private TermHit getTermHit(String label, float confidence, float start, float end) {
378+
private Term addTermHit(Term term, TermHit termHit) {
379+
Set<TermHit> hits = term.getHits();
380+
if (hits == null) {
381+
hits = new TreeSet<>(TERM_HIT_COMPARATOR);
382+
term.setHits(hits);
383+
}
384+
385+
hits.add(termHit);
386+
387+
return term;
388+
}
389+
390+
private TermHit getTermHit(String label, float confidence, boolean oneBest) {
391391
TermHit termHit = new TermHit();
392392
termHit.setConfidence(confidence);
393-
termHit.setTermLabel(label);
394-
termHit.getTimeRange().setStartTime(start);
395-
termHit.getTimeRange().setEndTime(end);
393+
termHit.setLabel(label);
394+
termHit.setOneBest(oneBest);
396395

397396
return termHit;
398397
}

0 commit comments

Comments
 (0)