Skip to content

Commit 31adebb

Browse files
adsharmaadsharma
committed
Support secondary ART indexes
1 parent 44c670a commit 31adebb

18 files changed

Lines changed: 950 additions & 132 deletions

File tree

‎src/binder/bind/bind_ddl.cpp‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include "parser/ddl/drop.h"
3636
#include "parser/expression/parsed_function_expression.h"
3737
#include "parser/expression/parsed_literal_expression.h"
38+
#include "storage/index/art_index.h"
3839
#include "storage/index/hash_index.h"
3940
#include "storage/storage_manager.h"
4041
#include "transaction/transaction.h"
@@ -422,8 +423,11 @@ std::unique_ptr<BoundStatement> Binder::bindCreateIndex(const Statement& stateme
422423
if (!nodeTableEntry->getStorage().empty()) {
423424
throw BinderException("CREATE INDEX is only supported on native node tables.");
424425
}
425-
if (!StringUtils::caseInsensitiveEquals(nodeTableEntry->getPrimaryKeyName(),
426-
info.propertyName)) {
426+
const auto isPrimaryIndex =
427+
StringUtils::caseInsensitiveEquals(nodeTableEntry->getPrimaryKeyName(), info.propertyName);
428+
const auto isArtIndex = StringUtils::caseInsensitiveEquals(indexType,
429+
storage::ArtPrimaryKeyIndex::getIndexType().typeName);
430+
if (!isPrimaryIndex && !isArtIndex) {
427431
throw BinderException(std::format(
428432
"{} indexes are currently supported only on node primary keys.", indexType));
429433
}
@@ -434,7 +438,9 @@ std::unique_ptr<BoundStatement> Binder::bindCreateIndex(const Statement& stateme
434438
auto& property = tableEntry->getProperty(info.propertyName);
435439
std::vector<PropertyDefinition> propertyDefinitions;
436440
propertyDefinitions.push_back(property.copy());
437-
validatePrimaryKey(property.getName(), propertyDefinitions);
441+
if (isPrimaryIndex) {
442+
validatePrimaryKey(property.getName(), propertyDefinitions);
443+
}
438444
auto indexName = info.indexName.empty() ? std::string(storage::PrimaryKeyIndex::DEFAULT_NAME) :
439445
info.indexName;
440446
if (info.onConflict == ConflictAction::ON_CONFLICT_THROW) {
@@ -454,7 +460,7 @@ std::unique_ptr<BoundStatement> Binder::bindCreateIndex(const Statement& stateme
454460
BoundCreateIndexInfo boundInfo{indexType, std::move(indexName), info.tableName,
455461
tableEntry->getTableID(), property.getName(), tableEntry->getPropertyID(property.getName()),
456462
tableEntry->getColumnID(property.getName()), property.getType().getPhysicalType(),
457-
info.onConflict};
463+
isPrimaryIndex, info.onConflict};
458464
return std::make_unique<BoundCreateIndex>(std::move(boundInfo));
459465
}
460466

‎src/common/file_system/local_file_system.cpp‎

Lines changed: 50 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -421,35 +421,59 @@ void LocalFileSystem::readFromFile(FileInfo& fileInfo, void* buffer, uint64_t nu
421421
uint64_t position) const {
422422
auto localFileInfo = fileInfo.constPtrCast<LocalFileInfo>();
423423
DASSERT(localFileInfo->getFileSize() >= position + numBytes);
424+
auto outputBuffer = static_cast<uint8_t*>(buffer);
425+
uint64_t remainingNumBytesToRead = numBytes;
426+
uint64_t bufferOffset = 0;
427+
// Keep reads below common OS syscall transfer limits.
428+
constexpr uint64_t maxBytesToReadAtOnce = 1ull << 30;
429+
while (remainingNumBytesToRead > 0) {
430+
const auto numBytesToRead = (std::min)(remainingNumBytesToRead, maxBytesToReadAtOnce);
424431
#if defined(_WIN32)
425-
DWORD numBytesRead;
426-
OVERLAPPED overlapped = {};
427-
overlapped.Offset = position & 0xffffffff;
428-
overlapped.OffsetHigh = position >> 32;
429-
if (!ReadFile((HANDLE)localFileInfo->handle, buffer, numBytes, &numBytesRead, &overlapped)) {
430-
auto error = GetLastError();
431-
throw IOException(
432-
std::format("Cannot read from file: {} handle: {} "
433-
"numBytesRead: {} numBytesToRead: {} position: {}. Error {}: {}",
434-
fileInfo.path, (intptr_t)localFileInfo->handle, numBytesRead, numBytes, position,
435-
error, std::system_category().message(error)));
436-
}
437-
if (numBytesRead != numBytes && fileInfo.getFileSize() != position + numBytesRead) {
438-
throw IOException(std::format("Cannot read from file: {} handle: {} "
439-
"numBytesRead: {} numBytesToRead: {} position: {}",
440-
fileInfo.path, (intptr_t)localFileInfo->handle, numBytesRead, numBytes, position));
441-
}
432+
DWORD numBytesRead;
433+
OVERLAPPED overlapped = {};
434+
overlapped.Offset = position & 0xffffffff;
435+
overlapped.OffsetHigh = position >> 32;
436+
if (!ReadFile((HANDLE)localFileInfo->handle, outputBuffer + bufferOffset, numBytesToRead,
437+
&numBytesRead, &overlapped)) {
438+
auto error = GetLastError();
439+
throw IOException(
440+
std::format("Cannot read from file: {} handle: {} "
441+
"numBytesRead: {} numBytesToRead: {} position: {}. Error {}: {}",
442+
fileInfo.path, (intptr_t)localFileInfo->handle, numBytesRead, numBytesToRead,
443+
position, error, std::system_category().message(error)));
444+
}
445+
if (numBytesRead != numBytesToRead && fileInfo.getFileSize() != position + numBytesRead) {
446+
throw IOException(std::format("Cannot read from file: {} handle: {} "
447+
"numBytesRead: {} numBytesToRead: {} position: {}",
448+
fileInfo.path, (intptr_t)localFileInfo->handle, numBytesRead, numBytesToRead,
449+
position));
450+
}
442451
#else
443-
auto numBytesRead = pread(localFileInfo->fd, buffer, numBytes, position);
444-
if (static_cast<uint64_t>(numBytesRead) != numBytes &&
445-
localFileInfo->getFileSize() != position + numBytesRead) {
446-
// LCOV_EXCL_START
447-
throw IOException(std::format("Cannot read from file: {} fileDescriptor: {} "
448-
"numBytesRead: {} numBytesToRead: {} position: {}",
449-
fileInfo.path, localFileInfo->fd, numBytesRead, numBytes, position));
450-
// LCOV_EXCL_STOP
451-
}
452+
auto numBytesRead =
453+
pread(localFileInfo->fd, outputBuffer + bufferOffset, numBytesToRead, position);
454+
if (numBytesRead < 0) {
455+
// LCOV_EXCL_START
456+
throw IOException(std::format("Cannot read from file: {} fileDescriptor: {} "
457+
"numBytesRead: {} numBytesToRead: {} position: {}",
458+
fileInfo.path, localFileInfo->fd, numBytesRead, numBytesToRead, position));
459+
// LCOV_EXCL_STOP
460+
}
461+
if (static_cast<uint64_t>(numBytesRead) != numBytesToRead &&
462+
localFileInfo->getFileSize() != position + numBytesRead) {
463+
// LCOV_EXCL_START
464+
throw IOException(std::format("Cannot read from file: {} fileDescriptor: {} "
465+
"numBytesRead: {} numBytesToRead: {} position: {}",
466+
fileInfo.path, localFileInfo->fd, numBytesRead, numBytesToRead, position));
467+
// LCOV_EXCL_STOP
468+
}
452469
#endif
470+
if (numBytesRead == 0) {
471+
break;
472+
}
473+
remainingNumBytesToRead -= numBytesRead;
474+
position += numBytesRead;
475+
bufferOffset += numBytesRead;
476+
}
453477
}
454478

455479
int64_t LocalFileSystem::readFile(FileInfo& fileInfo, void* buf, size_t nbyte) const {

‎src/include/binder/ddl/bound_create_index.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ struct BoundCreateIndexInfo {
1616
common::property_id_t propertyID;
1717
common::column_id_t columnID;
1818
common::PhysicalTypeID keyDataType;
19+
bool isPrimary;
1920
common::ConflictAction onConflict;
2021

2122
BoundCreateIndexInfo copy() const { return *this; }

‎src/include/planner/operator/scan/logical_scan_node_table.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ namespace planner {
1010
enum class LogicalScanNodeTableType : uint8_t {
1111
SCAN = 0,
1212
PRIMARY_KEY_SCAN = 1,
13+
SECONDARY_INDEX_SCAN = 2,
1314
};
1415

1516
struct ExtraScanNodeTableInfo {
@@ -45,6 +46,18 @@ struct PrimaryKeyScanInfo final : ExtraScanNodeTableInfo {
4546
}
4647
};
4748

49+
struct SecondaryIndexScanInfo final : ExtraScanNodeTableInfo {
50+
std::string indexName;
51+
std::shared_ptr<binder::Expression> key;
52+
53+
SecondaryIndexScanInfo(std::string indexName, std::shared_ptr<binder::Expression> key)
54+
: indexName{std::move(indexName)}, key{std::move(key)} {}
55+
56+
std::unique_ptr<ExtraScanNodeTableInfo> copy() const override {
57+
return std::make_unique<SecondaryIndexScanInfo>(indexName, key);
58+
}
59+
};
60+
4861
struct LogicalScanNodeTablePrintInfo final : OPPrintInfo {
4962
std::shared_ptr<binder::Expression> nodeID;
5063
binder::expression_vector properties;

‎src/include/processor/operator/scan/primary_key_scan_node_table.h‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,15 @@ class PrimaryKeyScanNodeTable : public ScanTable {
4747
PrimaryKeyScanNodeTable(ScanOpInfo opInfo, std::vector<ScanNodeTableInfo> tableInfos,
4848
std::unique_ptr<evaluator::ExpressionEvaluator> indexEvaluator,
4949
std::unique_ptr<evaluator::ExpressionEvaluator> upperBoundEvaluator, bool isRange,
50-
bool lowerInclusive, bool upperInclusive,
50+
bool isIndexEquality, bool lowerInclusive, bool upperInclusive, std::string indexName,
5151
std::shared_ptr<PrimaryKeyScanSharedState> sharedState, physical_op_id id,
5252
std::unique_ptr<OPPrintInfo> printInfo)
5353
: ScanTable{type_, std::move(opInfo), id, std::move(printInfo)}, scanState{nullptr},
5454
tableInfos{std::move(tableInfos)}, indexEvaluator{std::move(indexEvaluator)},
5555
upperBoundEvaluator{std::move(upperBoundEvaluator)}, sharedState{std::move(sharedState)},
56-
isRange{isRange}, lowerInclusive{lowerInclusive}, upperInclusive{upperInclusive},
57-
currentRangeTableIdx{0}, rangeOffsetCursor{0} {}
56+
isRange{isRange}, isIndexEquality{isIndexEquality}, lowerInclusive{lowerInclusive},
57+
upperInclusive{upperInclusive}, indexName{std::move(indexName)}, currentRangeTableIdx{0},
58+
rangeOffsetCursor{0} {}
5859

5960
bool isSource() const override { return true; }
6061

@@ -68,7 +69,8 @@ class PrimaryKeyScanNodeTable : public ScanTable {
6869
return std::make_unique<PrimaryKeyScanNodeTable>(opInfo.copy(), copyVector(tableInfos),
6970
indexEvaluator == nullptr ? nullptr : indexEvaluator->copy(),
7071
upperBoundEvaluator == nullptr ? nullptr : upperBoundEvaluator->copy(), isRange,
71-
lowerInclusive, upperInclusive, sharedState, id, printInfo->copy());
72+
isIndexEquality, lowerInclusive, upperInclusive, indexName, sharedState, id,
73+
printInfo->copy());
7274
}
7375

7476
private:
@@ -81,8 +83,10 @@ class PrimaryKeyScanNodeTable : public ScanTable {
8183
std::unique_ptr<evaluator::ExpressionEvaluator> upperBoundEvaluator;
8284
std::shared_ptr<PrimaryKeyScanSharedState> sharedState;
8385
bool isRange;
86+
bool isIndexEquality;
8487
bool lowerInclusive;
8588
bool upperInclusive;
89+
std::string indexName;
8690
common::idx_t currentRangeTableIdx;
8791
std::vector<common::offset_t> rangeOffsets;
8892
common::idx_t rangeOffsetCursor;

‎src/include/storage/index/art_index.h‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,25 +67,32 @@ class ArtPrimaryKeyIndex final : public Index {
6767
const common::ValueVector& nodeIDVector,
6868
const std::vector<common::ValueVector*>& indexVectors,
6969
Index::InsertState& insertState) override;
70+
std::unique_ptr<UpdateState> initUpdateState(main::ClientContext* context,
71+
common::column_id_t columnID, visible_func isVisible) override;
72+
void update(transaction::Transaction* transaction, const common::ValueVector& nodeIDVector,
73+
common::ValueVector& propertyVector, UpdateState& updateState) override;
7074

7175
std::unique_ptr<DeleteState> initDeleteState(const transaction::Transaction*, MemoryManager*,
7276
visible_func) override {
7377
return std::make_unique<DeleteState>();
7478
}
75-
void delete_(transaction::Transaction*, const common::ValueVector&, DeleteState&) override {
76-
// Visibility rules filter deleted rows. Physical removal is used only for rollback cleanup.
77-
}
79+
void delete_(transaction::Transaction*, const common::ValueVector& nodeIDVector,
80+
DeleteState&) override;
7881

7982
bool lookupPrimaryKey(const transaction::Transaction* transaction,
8083
common::ValueVector* keyVector, uint64_t vectorPos, common::offset_t& result,
8184
visible_func isVisible) override;
85+
bool lookupAll(const transaction::Transaction* transaction, common::ValueVector* keyVector,
86+
uint64_t vectorPos, std::vector<common::offset_t>& results,
87+
visible_func isVisible) override;
8288
bool scanPrimaryKeyRange(common::ValueVector* lowerBoundVector, uint64_t lowerBoundPos,
8389
bool lowerInclusive, common::ValueVector* upperBoundVector, uint64_t upperBoundPos,
8490
bool upperInclusive, common::idx_t maxResults, std::vector<common::offset_t>& results,
8591
visible_func isVisible) override;
8692
void discardPrimaryKey(common::ValueVector* keyVector) override;
8793

8894
void checkpoint(main::ClientContext*, PageAllocator&) override;
95+
void serialize(common::Serializer& ser) const override;
8996

9097
static LBUG_API std::unique_ptr<Index> load(main::ClientContext* context,
9198
StorageManager* storageManager, IndexInfo indexInfo, std::span<uint8_t> storageInfoBuffer);
@@ -114,19 +121,26 @@ class ArtPrimaryKeyIndex final : public Index {
114121
};
115122

116123
std::optional<common::offset_t> offset;
124+
std::unique_ptr<std::vector<common::offset_t>> overflowOffsets;
125+
std::vector<uint8_t> prefix;
117126
Kind kind = Kind::NODE4;
118127
uint16_t count = 0;
119-
union {
120-
SmallChildren small;
121-
Node48Children node48;
122-
Node256Children node256;
123-
};
128+
SmallChildren small;
129+
std::unique_ptr<Node48Children> node48;
130+
std::unique_ptr<Node256Children> node256;
124131

125132
Node();
126133
Node* getChild(uint8_t byte) const;
127134
Node* getOrInsertChild(ArtPrimaryKeyIndex& index, uint8_t byte);
135+
void insertChild(ArtPrimaryKeyIndex& index, uint8_t byte, Node* child);
128136
void removeChild(uint8_t byte);
129-
bool empty() const { return !offset.has_value() && count == 0; }
137+
bool hasOffsets() const {
138+
return offset.has_value() || (overflowOffsets && !overflowOffsets->empty());
139+
}
140+
bool empty() const {
141+
return !offset.has_value() && (!overflowOffsets || overflowOffsets->empty()) &&
142+
count == 0;
143+
}
130144
};
131145

132146
static constexpr uint64_t NODE_BLOCK_CAPACITY = 16 * 1024;
@@ -144,19 +158,31 @@ class ArtPrimaryKeyIndex final : public Index {
144158
};
145159

146160
bool insertInternal(const ArtKey& key, common::offset_t offset, visible_func isVisible);
161+
void insertSecondaryInternal(const ArtKey& key, common::offset_t offset);
162+
Node* findOrCreateLeaf(const std::vector<uint8_t>& key);
147163
bool lookup(const ArtKey& key, common::offset_t& result, visible_func isVisible) const;
164+
const Node* findLeaf(const ArtKey& key) const;
165+
void appendVisibleOffsets(const Node& node, std::vector<common::offset_t>& results,
166+
visible_func isVisible) const;
148167
bool eraseInternal(Node& node, const std::vector<uint8_t>& key, uint64_t depth);
149168
void erase(const ArtKey& key);
169+
static void eraseOffsetFromLeaf(Node& node, common::offset_t offset);
170+
static void resetNodePayload(Node& node);
171+
bool eraseOffsetInternal(Node& node, common::offset_t offset);
150172
Node* allocateNode();
151173
void recordKindChange(Node& node, Node::Kind newKind);
152174
void collectRange(const Node& node, std::vector<uint8_t>& key, const ArtKey* lowerBound,
153175
bool lowerInclusive, const ArtKey* upperBound, bool upperInclusive,
154176
common::idx_t maxResults, std::vector<common::offset_t>& results,
155177
visible_func isVisible) const;
156178
void clear();
179+
uint64_t calculateSerializedTreeSize(const Node& node) const;
180+
void serializeTree(const Node& node, common::Serializer& serializer) const;
181+
void loadTree(common::BufferReader& reader, Node& node);
157182
void collectEntries(const Node& node, std::vector<uint8_t>& key,
158183
std::vector<std::pair<std::vector<uint8_t>, common::offset_t>>& entries) const;
159184
void loadEntries(const ArtPrimaryKeyIndexStorageInfo& storageInfo);
185+
void loadEntries(common::BufferReader& reader);
160186

161187
private:
162188
Node root;

‎src/include/storage/index/index.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,11 @@ class LBUG_API Index {
151151
visible_func /*isVisible*/) {
152152
return false;
153153
}
154+
virtual bool lookupAll(const transaction::Transaction* /*transaction*/,
155+
common::ValueVector* /*keyVector*/, uint64_t /*vectorPos*/,
156+
std::vector<common::offset_t>& /*results*/, visible_func /*isVisible*/) {
157+
return false;
158+
}
154159
virtual bool scanPrimaryKeyRange(common::ValueVector* /*lowerBoundVector*/,
155160
uint64_t /*lowerBoundPos*/, bool /*lowerInclusive*/,
156161
common::ValueVector* /*upperBoundVector*/, uint64_t /*upperBoundPos*/,
@@ -206,7 +211,7 @@ class IndexHolder {
206211
public:
207212
explicit IndexHolder(std::unique_ptr<Index> loadedIndex);
208213
IndexHolder(IndexInfo indexInfo, std::unique_ptr<uint8_t[]> storageInfoBuffer,
209-
uint32_t storageInfoBufferSize);
214+
uint64_t storageInfoBufferSize);
210215

211216
std::string getName() const { return indexInfo.name; }
212217
bool isLoaded() const { return loaded; }

‎src/include/storage/table/node_table.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,13 @@ class LBUG_API NodeTable : public Table {
158158
common::ValueVector* lowerBoundVector, uint64_t lowerBoundPos, bool lowerInclusive,
159159
common::ValueVector* upperBoundVector, uint64_t upperBoundPos, bool upperInclusive,
160160
common::idx_t maxResults, std::vector<common::offset_t>& results) const;
161+
bool lookupIndexRange(const transaction::Transaction* transaction, const std::string& indexName,
162+
common::ValueVector* lowerBoundVector, uint64_t lowerBoundPos, bool lowerInclusive,
163+
common::ValueVector* upperBoundVector, uint64_t upperBoundPos, bool upperInclusive,
164+
common::idx_t maxResults, std::vector<common::offset_t>& results) const;
165+
bool lookupIndex(const transaction::Transaction* transaction, const std::string& indexName,
166+
common::ValueVector* keyVector, uint64_t keyPos,
167+
std::vector<common::offset_t>& results) const;
161168

162169
void addIndex(std::unique_ptr<Index> index);
163170
void buildIndexAndAdd(main::ClientContext* context, std::unique_ptr<Index> index);

0 commit comments

Comments
 (0)