Skip to content

Commit ca3bde8

Browse files
stonesmistonesmi
authored
Improved sort performance (#2087)
1 parent e9b9bef commit ca3bde8

10 files changed

Lines changed: 298 additions & 558 deletions

File tree

‎benchmark/src/main/java/org/finos/vuu/benchmark/BenchmarkHelper.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,10 @@ public void addTableData(DataTable dataTable, int offset, int size) {
7777
rowBuilder.setKey(ric);
7878
rowBuilder.setString(ricColumn, ric);
7979
rowBuilder.setString(exchangeColumn, "exchange-" + i);
80-
rowBuilder.setDouble(bidColumn, 101);
81-
rowBuilder.setDouble(askColumn, 100);
82-
rowBuilder.setDouble(lastColumn, 105);
83-
rowBuilder.setDouble(closeColumn, 106);
80+
rowBuilder.setDouble(bidColumn, i + 1.0);
81+
rowBuilder.setDouble(askColumn, i + 2.0);
82+
rowBuilder.setDouble(lastColumn, i + 3.0);
83+
rowBuilder.setDouble(closeColumn, i + 4.0);
8484
dataTable.processUpdate(rowBuilder.build());
8585
}
8686
}

‎benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmark.java‎

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package org.finos.vuu.benchmark.sort;
22

33
import org.finos.vuu.benchmark.BenchmarkHelper;
4-
import org.finos.vuu.core.sort.GenericSort2;
54
import org.finos.vuu.core.sort.Sort;
65
import org.finos.vuu.core.sort.SortDirection;
6+
import org.finos.vuu.core.table.Column;
77
import org.finos.vuu.core.table.DataTable;
88
import org.finos.vuu.core.table.ViewPortColumnCreator;
99
import org.finos.vuu.net.SortDef;
@@ -12,25 +12,42 @@
1212

1313
import java.util.List;
1414

15+
import static java.util.Arrays.stream;
1516
import static org.finos.vuu.util.ScalaCollectionConverter.toScala;
1617

1718
public class SortBenchmark {
1819

1920
private final DataTable inMemDataTable;
2021
private final ViewPortColumns viewPortColumns;
21-
private final Sort sort;
22+
private final Sort singleSort;
23+
private final Sort multiSort;
2224

2325
public SortBenchmark(BenchmarkHelper benchmarkHelper, int size) {
2426
inMemDataTable = benchmarkHelper.buildTable();
25-
viewPortColumns = ViewPortColumnCreator.create(inMemDataTable, toScala(List.of("exchange")));
26-
sort = new GenericSort2(
27-
SortSpec.apply(toScala(List.of(SortDef.apply("exchange", SortDirection.ASCENDING().external())))),
28-
toScala(List.of(inMemDataTable.getTableDef().columnForName("exchange"))));
27+
viewPortColumns = ViewPortColumnCreator.create(inMemDataTable,
28+
toScala(stream(inMemDataTable.getTableDef().getColumns()).map(Column::name).toList()));
29+
var closeColumn = inMemDataTable.getTableDef().columnForName("close");
30+
var exchangeColumn = inMemDataTable.getTableDef().columnForName("exchange");
31+
singleSort = Sort.apply(
32+
SortSpec.apply(toScala(List.of(
33+
SortDef.apply(exchangeColumn.name(), SortDirection.ASCENDING().external())
34+
))),
35+
toScala(List.of(exchangeColumn)));
36+
multiSort = Sort.apply(
37+
SortSpec.apply(toScala(List.of(
38+
SortDef.apply(exchangeColumn.name(), SortDirection.DESCENDING().external()),
39+
SortDef.apply(closeColumn.name(), SortDirection.ASCENDING().external())
40+
))),
41+
toScala(List.of(exchangeColumn, closeColumn)));
2942
benchmarkHelper.addTableData(inMemDataTable, size);
3043
}
3144

32-
void sortLargeTable() {
33-
sort.doSort(inMemDataTable, inMemDataTable.primaryKeys(), viewPortColumns);
45+
void sortLargeTableSingleColumn() {
46+
singleSort.doSort(inMemDataTable, inMemDataTable.primaryKeys(), viewPortColumns);
47+
}
48+
49+
void sortLargeTableMultiColumn() {
50+
multiSort.doSort(inMemDataTable, inMemDataTable.primaryKeys(), viewPortColumns);
3451
}
3552

3653
}

‎benchmark/src/main/java/org/finos/vuu/benchmark/sort/SortBenchmarkRunner.java‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import org.openjdk.jmh.annotations.State;
1515
import org.openjdk.jmh.annotations.Warmup;
1616

17-
import java.io.IOException;
1817
import java.util.concurrent.TimeUnit;
1918

2019
@State(Scope.Benchmark)
@@ -23,7 +22,7 @@ public class SortBenchmarkRunner {
2322
private final BenchmarkHelper benchmarkHelper = new BenchmarkHelper();
2423
private SortBenchmark benchmark;
2524

26-
@Param({ "10000", "100000", "500000", "1000000" })
25+
@Param({ "10000", "100000", "1000000" })
2726
public int tableSize;
2827

2928
@Setup(Level.Trial)
@@ -37,8 +36,18 @@ public void setup() {
3736
@Measurement(iterations = 5)
3837
@Fork(1)
3938
@BenchmarkMode(Mode.SampleTime)
40-
public void sortLargeTable() throws IOException {
41-
benchmark.sortLargeTable();
39+
public void sortLargeTableSingle() {
40+
benchmark.sortLargeTableSingleColumn();
41+
}
42+
43+
@Benchmark
44+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
45+
@Warmup(iterations = 5)
46+
@Measurement(iterations = 5)
47+
@Fork(1)
48+
@BenchmarkMode(Mode.SampleTime)
49+
public void sortLargeTableMulti() {
50+
benchmark.sortLargeTableMultiColumn();
4251
}
4352

4453
}

‎toolbox/src/main/scala/org/finos/toolbox/collection/array/VectorImmutableArray.scala‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,10 @@ private class VectorImmutableArrayImpl[T <: Object : ClassTag](private val data:
9595
doRemove(logicalIndex)
9696
}
9797

98-
override def iterator: Iterator[T] = {
99-
val iterator = new Iterator[Int] {
100-
private val it = activeIndices.getIntIterator
101-
def hasNext: Boolean = it.hasNext
102-
def next(): Int = it.next()
103-
}
104-
iterator.map(data(_))
98+
override def iterator: Iterator[T] = new Iterator[T] {
99+
private val it = activeIndices.getIntIterator
100+
override def hasNext: Boolean = it.hasNext
101+
override def next(): T = data(it.next())
105102
}
106103

107104
override def foreach[U](f: T => U): Unit = {

‎vuu/src/main/scala/org/finos/vuu/core/sort/SortCompares.scala‎

Lines changed: 0 additions & 108 deletions
This file was deleted.

‎vuu/src/main/scala/org/finos/vuu/core/sort/SortProjection.scala‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
package org.finos.vuu.core.sort
2+
3+
import com.typesafe.scalalogging.StrictLogging
4+
import org.finos.vuu.core.sort.SortDirection.Ascending
5+
import org.finos.vuu.core.table.{Column, DataType}
6+
7+
trait SortProjectionComparator extends java.util.Comparator[Array[AnyRef]]
8+
9+
object SortProjectionComparator extends StrictLogging {
10+
11+
def apply(columns: Array[Column], sortDirections: Array[SortDirection]): SortProjectionComparator = {
12+
val comparators = columns.indices.map { i =>
13+
val col = columns(i)
14+
val dir = sortDirections(i)
15+
buildColumnComparator(col, i + 1, dir == Ascending)
16+
}.toArray
17+
18+
comparators.length match {
19+
case 1 => SingleColumnComparatorImpl(comparators.head)
20+
case _ => MultiColumnComparatorImpl(comparators)
21+
}
22+
}
23+
24+
private def buildColumnComparator(column: Column, index: Int, isAscending: Boolean): ColumnSort = {
25+
column.dataType match {
26+
case DataType.StringDataType =>
27+
if (isAscending) StringColumnSortAsc(index) else StringColumnSortDesc(index)
28+
case DataType.LongDataType | DataType.IntegerDataType | DataType.DoubleDataType |
29+
DataType.BooleanDataType | DataType.CharDataType | DataType.EpochTimestampType |
30+
DataType.ScaledDecimal2Type | DataType.ScaledDecimal4Type |
31+
DataType.ScaledDecimal6Type | DataType.ScaledDecimal8Type =>
32+
if (isAscending) ComparableColumnSortAsc(index) else ComparableColumnSortDesc(index)
33+
case _ =>
34+
logger.warn(s"Unable to sort datatype ${column.dataType}")
35+
NoColumnSort
36+
}
37+
}
38+
39+
}
40+
41+
case class SingleColumnComparatorImpl(columnSort: ColumnSort) extends SortProjectionComparator {
42+
43+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = {
44+
columnSort.compare(o1, o2)
45+
}
46+
47+
}
48+
49+
case class MultiColumnComparatorImpl(comparators: Array[ColumnSort]) extends SortProjectionComparator {
50+
51+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = {
52+
var i = 0
53+
val len = comparators.length
54+
while (i < len) {
55+
val res = comparators(i).compare(o1, o2)
56+
if (res != 0) return res
57+
i += 1
58+
}
59+
0
60+
}
61+
}
62+
63+
sealed trait ColumnSort {
64+
def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int
65+
}
66+
67+
object NoColumnSort extends ColumnSort {
68+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = 0
69+
}
70+
71+
case class StringColumnSortAsc(index: Int) extends ColumnSort {
72+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = {
73+
val v1 = o1(index).asInstanceOf[String]
74+
val v2 = o2(index).asInstanceOf[String]
75+
76+
if (v1 eq v2) 0
77+
else if (v1 == null) 1
78+
else if (v2 == null) -1
79+
else v1.compareToIgnoreCase(v2)
80+
}
81+
}
82+
83+
case class StringColumnSortDesc(index: Int) extends ColumnSort {
84+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = {
85+
val v1 = o1(index).asInstanceOf[String]
86+
val v2 = o2(index).asInstanceOf[String]
87+
88+
if (v1 eq v2) 0
89+
else if (v1 == null) -1
90+
else if (v2 == null) 1
91+
else v2.compareToIgnoreCase(v1)
92+
}
93+
}
94+
95+
case class ComparableColumnSortAsc(index: Int) extends ColumnSort {
96+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = {
97+
val v1 = o1(index).asInstanceOf[Comparable[AnyRef]]
98+
val v2 = o2(index).asInstanceOf[Comparable[AnyRef]]
99+
100+
if (v1 eq v2) 0
101+
else if (v1 == null) 1
102+
else if (v2 == null) -1
103+
else v1.compareTo(v2)
104+
}
105+
}
106+
107+
case class ComparableColumnSortDesc(index: Int) extends ColumnSort {
108+
override def compare(o1: Array[AnyRef], o2: Array[AnyRef]): Int = {
109+
val v1 = o1(index).asInstanceOf[Comparable[AnyRef]]
110+
val v2 = o2(index).asInstanceOf[Comparable[AnyRef]]
111+
112+
if (v1 eq v2) 0
113+
else if (v1 == null) -1
114+
else if (v2 == null) 1
115+
else v2.compareTo(v1)
116+
}
117+
}

0 commit comments

Comments
 (0)