Multiple changes to optimize and cleanup typed_buffers, BasicXorEncryptor, and Sequencer

src/client/dbps_api_client_test.cpp

@@ -23,19 +23,14 @@
 #include "tcb/span.hpp"
 #include "dbps_api_client.h"
 #include "http_client_base.h"
+#include "../common/bytes_utils.h"
 #include "../common/enums.h"
 #include <nlohmann/json.hpp>
 #include <gtest/gtest.h>
 
 using namespace dbps::external;
 using namespace dbps::enum_utils;
 
-// TODO: Move this to a common test utility file.
-// Helper function to convert string to binary data
-std::vector<uint8_t> StringToBytes(const std::string& str) {
-    return std::vector<uint8_t>(str.begin(), str.end());
-}
-
 // Utility function to compare JSON strings, ignoring specified fields
 bool CompareJsonStrings(const std::string& json1, const std::string& json2, const std::vector<std::string>& ignore_fields = {}) {
     try {

src/common/bytes_utils.h

@@ -81,6 +81,13 @@ inline void write_u32_le_at(std::vector<uint8_t>& buf, size_t offset, uint32_t v
     buf[offset + 3] = static_cast<uint8_t>((v >> 24) & 0xFF);
 }
 
+inline void write_u32_le(uint8_t* p, uint32_t v) {
+    p[0] = static_cast<uint8_t>(v);
+    p[1] = static_cast<uint8_t>(v >> 8);
+    p[2] = static_cast<uint8_t>(v >> 16);
+    p[3] = static_cast<uint8_t>(v >> 24);
+}
+
 inline uint32_t read_u32_le(const std::vector<uint8_t>& in, size_t offset) {
     return static_cast<uint32_t>(in[offset]) |
         (static_cast<uint32_t>(in[offset + 1]) << 8) |
@@ -95,6 +102,12 @@ inline uint32_t read_u32_le(tcb::span<const uint8_t> in, size_t offset) {
         (static_cast<uint32_t>(in[offset + 3]) << 24);
 }
 
+inline uint32_t read_u32_le(const uint8_t* p) {
+    uint32_t v;
+    std::memcpy(&v, p, sizeof(v));
+    return v;
+}
+
 // Utility functions for splitting and joining byte vectors.
 
 struct BytesPair {
@@ -289,3 +302,8 @@ inline std::string AddStringAttribute(
     out[key] = value;
     return value;
 }
+
+// Helper function to convert string to binary data
+inline std::vector<uint8_t> StringToBytes(const std::string& str) {
+    return std::vector<uint8_t>(str.begin(), str.end());
+}

src/common/dbpa_local_test.cpp

@@ -50,7 +50,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryption) {
                                Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
 
     std::vector<uint8_t> test_data = BuildByteArrayValueBytesForTesting("test_ABC");
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -74,7 +74,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulEncryptionCompressedDictiona
         0x03, 0x32, 0x92, 0x12, 0xF3, 0x80, 0x10, 0x00, 0xC7, 0xB8,
         0x50, 0xFC, 0x13, 0x00, 0x00, 0x00
     };
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data_gzip, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -93,7 +93,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, SuccessfulDecryption) {
                                Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, DBPS_ENCRYPTION_METADATA));
 
     std::vector<uint8_t> test_data = BuildByteArrayValueBytesForTesting("test_EFG");
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Decrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -113,7 +113,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, RoundTripEncryptDecrypt) {
 
     // Original data to encrypt
     std::vector<uint8_t> original_data = BuildByteArrayValueBytesForTesting("roundtrip_XYZ");
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
 
     // Encrypt the data
     auto encrypt_result = encrypt_agent.Encrypt(original_data, encoding_attributes);
@@ -156,7 +156,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, EncryptWithoutInit) {
     LocalDataBatchProtectionAgent agent;
 
     std::vector<uint8_t> test_data = {1, 2, 3, 4};
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -171,7 +171,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, DecryptWithoutInit) {
     LocalDataBatchProtectionAgent agent;
 
     std::vector<uint8_t> test_data = {1, 2, 3, 4};
-    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_encoding", "PLAIN"}, {"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Decrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);
@@ -203,7 +203,7 @@ TEST_F(LocalDataBatchProtectionAgentTest, MissingPageEncoding) {
                                Type::BYTE_ARRAY, std::nullopt, CompressionCodec::UNCOMPRESSED, std::nullopt));
 
     std::vector<uint8_t> test_data = {1, 2, 3, 4};
-    std::map<std::string, std::string> encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}};
+    std::map<std::string, std::string> encoding_attributes = {{"page_type", "DICTIONARY_PAGE"}, {"dict_page_num_values", "1"}};
     auto result = agent.Encrypt(test_data, encoding_attributes);
 
     ASSERT_NE(result, nullptr);

src/processing/encryption_sequencer.cpp

@@ -23,7 +23,6 @@
 #include "../common/exceptions.h"
 #include "encryptors/basic_xor_encryptor.h"
 #include <functional>
-#include <iomanip>
 #include <iostream>
 #include <sstream>
 #include <optional>
@@ -138,11 +137,12 @@ bool DataBatchEncryptionSequencer::DecodeAndEncrypt(tcb::span<const uint8_t> pla
      */
     try {
         // Decompress and split plaintext into level and value bytes
-        auto [level_bytes, value_bytes] = DecompressAndSplit(
+        auto [level_bytes, value_bytes, num_elements] = DecompressAndSplit(
             plaintext, compression_, encoding_attributes_converted_);
 
         // Parse value bytes into typed values buffer
-        auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer(value_bytes, datatype_, datatype_length_, encoding_);
+        auto typed_buffer = ReinterpretValueBytesAsTypedValuesBuffer(
+            value_bytes, num_elements, datatype_, datatype_length_, encoding_);
 
         // Encrypt the typed values buffer and level bytes, then join them into a single encrypted byte vector.
         auto encrypted_value_bytes = encryptor_->EncryptValueList(typed_buffer);
@@ -294,7 +294,7 @@ bool DataBatchEncryptionSequencer::ConvertEncodingAttributesToValues() {
             add_int("page_v2_num_nulls");
             add_bool("page_v2_is_compressed");
         } else if (page_type == "DICTIONARY_PAGE") {
-            // DICTIONARY_PAGE has no specific encoding attributes
+            add_int("dict_page_num_values");
         } else {
             throw InvalidInputException("Unexpected page type: " + page_type);
         }