Experimental support for the JSON data type

README.md

@@ -23,6 +23,8 @@ C++ client for [ClickHouse](https://clickhouse.com/).
 * UUID
 * Map
 * Point, Ring, Polygon, MultiPolygon
+* JSON - experimental support; requires output_format_native_write_json_as_string=1; data is passed as strings
+
 
 ## Dependencies
 In the most basic case one needs only:

clickhouse/CMakeLists.txt

@@ -16,6 +16,7 @@ SET ( clickhouse-cpp-lib-src
     columns/geo.cpp
     columns/ip4.cpp
     columns/ip6.cpp
+    columns/json.cpp
     columns/lowcardinality.cpp
     columns/nullable.cpp
     columns/numeric.cpp
@@ -60,6 +61,7 @@ SET ( clickhouse-cpp-lib-src
     columns/geo.h
     columns/ip4.h
     columns/ip6.h
+    columns/json.h
     columns/itemview.h
     columns/lowcardinality.h
     columns/lowcardinalityadaptor.h
@@ -221,6 +223,7 @@ INSTALL(FILES columns/factory.h DESTINATION include/clickhouse/columns/)
 INSTALL(FILES columns/geo.h DESTINATION include/clickhouse/columns/)
 INSTALL(FILES columns/ip4.h DESTINATION include/clickhouse/columns/)
 INSTALL(FILES columns/ip6.h DESTINATION include/clickhouse/columns/)
+INSTALL(FILES columns/json.h DESTINATION include/clickhouse/columns/)
 INSTALL(FILES columns/itemview.h DESTINATION include/clickhouse/columns/)
 INSTALL(FILES columns/lowcardinality.h DESTINATION include/clickhouse/columns/)
 INSTALL(FILES columns/nothing.h DESTINATION include/clickhouse/columns/)

clickhouse/client.h

@@ -10,6 +10,7 @@
 #include "columns/geo.h"
 #include "columns/ip4.h"
 #include "columns/ip6.h"
+#include "columns/json.h"
 #include "columns/lowcardinality.h"
 #include "columns/nothing.h"
 #include "columns/nullable.h"

clickhouse/columns/factory.cpp

@@ -7,6 +7,7 @@
 #include "geo.h"
 #include "ip4.h"
 #include "ip6.h"
+#include "json.h"
 #include "lowcardinality.h"
 #include "lowcardinalityadaptor.h"
 #include "map.h"
@@ -136,6 +137,8 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) {
             return nullptr;
         }
         return std::make_shared<ColumnTime64>(GetASTChildElement(ast, 0).value);
+    case Type::JSON:
+        return std::make_shared<ColumnJSON>();
     default:
         return nullptr;
     }

clickhouse/columns/itemview.cpp

@@ -72,6 +72,7 @@ void ItemView::ValidateData(Type::Code type, DataType data) {
 
         case Type::Code::String:
         case Type::Code::FixedString:
+        case Type::Code::JSON:
             // value can be of any size
             return;
 

clickhouse/columns/json.cpp

@@ -0,0 +1,102 @@
+#include "json.h"
+#include "../base/wire_format.h"
+
+namespace clickhouse {
+
+enum class JSONSerializationVersion : uint64_t {
+  // String is the only currently supported serialization of JSON.
+  // it should be enabled with output_format_native_write_json_as_string=1
+  String = 1,
+};
+
+ColumnJSON::ColumnJSON()
+    : Column(Type::CreateJSON())
+    , data_(std::make_shared<ColumnString>())
+{}
+
+ColumnJSON::ColumnJSON(std::vector<std::string> data)
+    : Column(Type::CreateJSON())
+    , data_(std::make_shared<ColumnString>(std::move(data)))
+{}
+
+void ColumnJSON::Append(std::string_view str) {
+    data_->Append(str);
+}
+
+void ColumnJSON::Append(const char* str) {
+    data_->Append(str);
+}
+void ColumnJSON::Append(std::string&& str) {
+    data_->Append(std::move(str));
+}
+
+std::string_view ColumnJSON::At(size_t n) const {
+    return data_->At(n);
+}
+
+void ColumnJSON::Append(ColumnRef column) {
+    if (auto col = column->As<ColumnJSON>()) {
+        data_->Append(col->data_);
+    }
+}
+
+void ColumnJSON::Reserve(size_t new_cap) {
+    data_->Reserve(new_cap);
+}
+
+bool ColumnJSON::LoadPrefix(InputStream* input, size_t) {
+    uint64_t v;
+    if (!WireFormat::ReadFixed(*input, &v)) {
+        return false;
+    }
+    if (v != static_cast<uint64_t>(JSONSerializationVersion::String)) {
+        // Hard stop: the library can only parse JSON when `output_format_native_write_json_as_string` is enabled.
+        // Further processing is meaningless after this error and the user must be notified immediately.
+        throw ProtocolError("Unsupported JSON serialization version. "
+                            "Make sure output_format_native_write_json_as_string=1 is set.");
+    }
+    return true;
+}
+
+bool ColumnJSON::LoadBody(InputStream* input, size_t rows) {
+    return data_->LoadBody(input, rows);
+}
+
+void ColumnJSON::SavePrefix(OutputStream* output) {
+    WireFormat::WriteFixed(*output, static_cast<uint64_t>(JSONSerializationVersion::String));
+}
+
+void ColumnJSON::SaveBody(OutputStream* output) {
+    data_->SaveBody(output);
+}
+
+void ColumnJSON::Clear() {
+    data_->Clear();
+}
+
+size_t ColumnJSON::Size() const {
+    return data_->Size();
+}
+
+ColumnRef ColumnJSON::Slice(size_t begin, size_t len) const {
+    auto ret = std::make_shared<ColumnJSON>();
+    auto sliced_data = data_->Slice(begin, len)->As<ColumnString>();
+    ret->data_->Swap(*sliced_data);
+    return ret;
+}
+
+ColumnRef ColumnJSON::CloneEmpty() const
+{
+    return std::make_shared<ColumnJSON>();
+}
+
+void ColumnJSON::Swap(Column& other) {
+    auto & col = dynamic_cast<ColumnJSON &>(other);
+    data_.swap(col.data_);
+}
+
+ItemView ColumnJSON::GetItem(size_t index) const {
+    return ItemView{Type::JSON, data_->GetItem(index)};
+}
+
+}

clickhouse/columns/json.h

@@ -0,0 +1,82 @@
+#pragma once
+
+#include "column.h"
+#include "string.h"
+#include "nullable.h"
+
+namespace clickhouse {
+
+/**
+ * JSON Column: Represents JSON values as strings.
+ * Works only when ClickHouse outputs JSON as strings and requires the setting
+ * output_format_native_write_json_as_string to be set to 1 for selecting data.
+ * Inserting JSON data does not require setting this setting.
+ * 
+ * WARNING: THIS IS AN EXPERIMENTAL IMPLEMENTATION.
+ * The API may change in the future as we continue working on full support for JSON columns.
+ *
+ * ClickHouse does not accept empty strings as JSON; it requires an empty object ({}).
+ * For nullable columns, each row marked a NULL must contain {}.
+ * For convenience `clickhouse::ColumnNullableT<ColumnJSON>` automatically inserts {} for NULL rows.
+ */
+class ColumnJSON : public Column {
+public:
+
+    ColumnJSON();
+    explicit ColumnJSON(std::vector<std::string> data);
+
+    /// Appends one element to the column.
+    void Append(std::string_view str);
+
+    void Append(const char* str);
+    void Append(std::string&& str);
+
+    std::string_view At(size_t n) const;
+    inline std::string_view operator [] (size_t n) const { return At(n); }
+
+    /// Appends content of given column to the end of current one.
+    void Append(ColumnRef column) override;
+
+    /// Increase the capacity of the column for large block insertion.
+    void Reserve(size_t new_cap) override;
+
+    /// Loads column prefix from input stream.
+    bool LoadPrefix(InputStream* input, size_t rows) override;
+
+    /// Loads column data from input stream.
+    bool LoadBody(InputStream* input, size_t rows) override;
+
+    /// Saves column prefix to output stream. Column types with prefixes must implement it.
+    void SavePrefix(OutputStream* output) override;
+
+    /// Saves column data to output stream.
+    void SaveBody(OutputStream* output) override;
+
+    /// Clear column data .
+    void Clear() override;
+
+    /// Returns count of rows in the column.
+    size_t Size() const override;
+
+    /// Makes slice of the current column.
+    ColumnRef Slice(size_t begin, size_t len) const override;
+    ColumnRef CloneEmpty() const override;
+    void Swap(Column& other) override;
+
+    ItemView GetItem(size_t index) const override;
+
+private:
+    std::shared_ptr<ColumnString> data_;
+};
+
+template <>
+inline void ColumnNullableT<ColumnJSON>::Append(std::optional<std::string_view> value) {
+    ColumnNullable::Append(!value.has_value());
+    if (value.has_value()) {
+        typed_nested_data_->Append(*value);
+    } else {
+        typed_nested_data_->Append(std::string_view("{}"));
+    }
+}
+
+}

clickhouse/types/type_parser.cpp

@@ -67,6 +67,7 @@ static const std::unordered_map<std::string, Type::Code> kTypeCode = {
     { "MultiPolygon", Type::MultiPolygon },
     { "Time",        Type::Time },
     { "Time64",      Type::Time64 },
+    { "JSON",        Type::JSON },
 };
 
 template <typename L, typename R>

clickhouse/types/types.cpp

@@ -54,6 +54,7 @@ const char* Type::TypeName(Type::Code code) {
         case Type::Code::MultiPolygon:   return "MultiPolygon";
         case Type::Code::Time:           return "Time";
         case Type::Code::Time64:         return "Time64";
+        case Type::Code::JSON:           return "JSON";
     }
 
     return "Unknown type";
@@ -85,6 +86,7 @@ std::string Type::GetName() const {
         case Ring:
         case Polygon:
         case MultiPolygon:
+        case JSON:
             return TypeName(code_);
         case Time64:
             return As<Time64Type>()->GetName();
@@ -138,6 +140,7 @@ uint64_t Type::GetTypeUniqueId() const {
         case Float32:
         case Float64:
         case String:
+        case JSON:
         case IPv4:
         case IPv6:
         case Date:
@@ -279,6 +282,10 @@ TypeRef Type::CreateMultiPolygon() {
     return TypeRef(new Type(Type::MultiPolygon));
 }
 
+TypeRef Type::CreateJSON() {
+    return TypeRef(new Type(Type::JSON));
+}
+
 /// class ArrayType
 
 ArrayType::ArrayType(TypeRef item_type) : Type(Array), item_type_(item_type) {

clickhouse/types/types.h

@@ -59,6 +59,7 @@ class Type {
         MultiPolygon,
         Time,
         Time64,
+        JSON,
     };
 
     using EnumItem = std::pair<std::string /* name */, int16_t /* value */>;
@@ -148,6 +149,8 @@ class Type {
 
     static TypeRef CreateTime64(size_t precision);
 
+    static TypeRef CreateJSON();
+
 private:
     uint64_t GetTypeUniqueId() const;
 

ut/Column_ut.cpp

@@ -195,6 +195,7 @@ using TestCases = ::testing::Types<
 
     GenericColumnTestCase<ColumnString, &makeColumn<ColumnString>, std::string, &MakeStrings>,
     GenericColumnTestCase<ColumnFixedString, &makeColumn<ColumnFixedString, 12>, std::string, &MakeFixedStrings<12>>,
+    GenericColumnTestCase<ColumnJSON, &makeColumn<ColumnJSON>, std::string, &MakeJSONs>,
 
     GenericColumnTestCase<ColumnDate, &makeColumn<ColumnDate>, time_t, &MakeDates<time_t>>,
     GenericColumnTestCase<ColumnDate32, &makeColumn<ColumnDate32>, time_t, &MakeDates<time_t>>,

ut/CreateColumnByType_ut.cpp

@@ -2,6 +2,7 @@
 #include <clickhouse/columns/date.h>
 #include <clickhouse/columns/numeric.h>
 #include <clickhouse/columns/string.h>
+#include <clickhouse/columns/json.h>
 
 #include <gtest/gtest.h>
 

ut/column_array_ut.cpp

@@ -227,6 +227,29 @@ TEST(ColumnArrayT, SimpleFixedString) {
     EXPECT_EQ("world\0"sv, (*array)[0][1]);
 }
 
+TEST(ColumnArrayT, JSON) {
+    using namespace std::literals;
+    auto i1 = R"({"item": 1})"sv;
+    auto i2 = R"({"item": 2})"sv;
+    auto i3 = R"({"item": 3})"sv;
+    auto array = std::make_shared<ColumnArrayT<ColumnJSON>>();
+    array->Append({i1});
+    array->Append({i2, i3});
+
+    EXPECT_EQ(i1, array->At(0).At(0));
+    EXPECT_EQ(i2, array->At(1).At(0));
+    EXPECT_EQ(i3, array->At(1).At(1));
+
+    auto r1 = array->At(0);
+    EXPECT_EQ(1u, r1.Size());
+    EXPECT_EQ(i1, r1.At(0));
+
+    auto r2 = array->At(1);
+    EXPECT_EQ(2u, r2.Size());
+    EXPECT_EQ(i2, r2.At(0));
+    EXPECT_EQ(i3, r2.At(1));
+}
+
 TEST(ColumnArrayT, SimpleUInt64_2D) {
     // Nested 2D-arrays are supported too:
     auto array = std::make_shared<ColumnArrayT<ColumnArrayT<ColumnUInt64>>>();

ut/columns_ut.cpp

@@ -137,6 +137,30 @@ TEST(ColumnsCase, StringAppend) {
     ASSERT_EQ(col->At(2), "11");
 }
 
+TEST(ColumnsCase, JSONInit) {
+    auto values = MakeJSONs();
+    auto col = std::make_shared<ColumnJSON>(values);
+
+    ASSERT_EQ(col->Size(), values.size());
+    ASSERT_EQ(col->At(1), values[1]);
+    ASSERT_EQ(col->At(2), values[2]);
+    ASSERT_EQ(col->At(3), values[3]);
+}
+
+TEST(ColumnsCase, JSONAppend) {
+    auto col = std::make_shared<ColumnJSON>();
+    const char* expected = "\"ufiudhf3493fyiudferyer3yrifhdflkdjfeuroe\"";
+    std::string data(expected);
+    col->Append(data);
+    col->Append(std::move(data));
+    col->Append("11");
+
+    ASSERT_EQ(col->Size(), 3u);
+    ASSERT_EQ(col->At(0), expected);
+    ASSERT_EQ(col->At(1), expected);
+    ASSERT_EQ(col->At(2), "11");
+}
+
 TEST(ColumnsCase, TupleAppend){
     auto tuple1 = std::make_shared<ColumnTuple>(std::vector<ColumnRef>({
                                 std::make_shared<ColumnUInt64>(),

ut/itemview_ut.cpp

@@ -70,6 +70,8 @@ TEST(ItemView, StorableTypes) {
 
     TEST_ITEMVIEW_TYPE_VALUE(Type::Code::FixedString, std::string_view, "");
     TEST_ITEMVIEW_TYPE_VALUE(Type::Code::FixedString, std::string_view, "here is a string");
+    TEST_ITEMVIEW_TYPE_VALUE(Type::Code::JSON, std::string_view, "{}");
+    TEST_ITEMVIEW_TYPE_VALUE(Type::Code::JSON, std::string_view, R"({"key": "value"})");
 }
 
 #define EXPECT_ITEMVIEW_ERROR(TypeCode, NativeType) \