Add AnyUtf8: one logical type for any UTF-8 encoding (#10)

* Add `AnyUtf8`: one logical type for any UTF-8 encoding

The string sibling of `AnyBinary`/`AnyList`: `Column<AnyUtf8>` accepts any of
arrow's UTF-8 encodings — `Utf8`, `LargeUtf8`, or `Utf8View` — and reads them
all uniformly as `&str`. Parse-only: implements `LogicalType` (and `RefType`)
but not `ConcreteType`, so no `from_values`/`Default`/schema; build a concrete
encoding such as `Column<Utf8>`.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* Improve readme

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: emilk

README.md

@@ -195,6 +195,7 @@ The supported logical types:
 |----------------------------------------------|------------------------------|---------------------------|
 | `bool`, `i8`–`i64`, `u8`–`u64`, `f16`–`f64`  | The same                     | By value                  |
 | `Utf8`, `LargeUtf8`, `Utf8View`              | The same                     | `&str`                    |
+| `AnyUtf8`                                    | *any* UTF-8 encoding above   | `&str`                    |
 | `FixedSizeBinary<N>`                         | `FixedSizeBinary(N)`         | `&[u8; N]`                |
 | `Binary`, `LargeBinary`, `BinaryView`        | The same                     | `&[u8]`                   |
 | `AnyBinary`                                  | *any* binary encoding (incl. `FixedSizeBinary`) | `&[u8]`        |
@@ -211,7 +212,7 @@ The supported logical types:
 | `Map<K, V>`                                  | `Map(…)`, recursively        | An iterator over `(key, value)` pairs |
 | `Option<L>`                                  | Nullable at this level       | `Option<…>`               |
 
-### `AnyList<L>`: one logical type for any list encoding
+### Semi-dynamic logical types
 
 Arrow has five physically different ways to store the same logical thing — a
 column of lists of `L`: `List`, `LargeList`, `ListView`, `LargeListView`, and
@@ -243,7 +244,8 @@ pick a concrete encoding such as `Column<List<L>>`.
 
 `AnyBinary` is the same idea for byte strings: it accepts any of `Binary`,
 `LargeBinary`, `BinaryView`, or `FixedSizeBinary` (any size) and reads them all
-as `&[u8]`. Also parse-only.
+as `&[u8]`. `AnyUtf8` likewise accepts any of `Utf8`, `LargeUtf8`, or `Utf8View`
+and reads them as `&str`. Both are also parse-only.
 
 ### What is *not* supported
 

crates/quiver/tests/column.rs

@@ -674,6 +674,45 @@ fn any_binary_columns() {
     ));
 }
 
+#[test]
+fn any_utf8_columns() {
+    use quiver::arrow::array::{LargeStringArray, StringViewArray};
+    use quiver::{AnyUtf8, LargeUtf8, Utf8View};
+
+    // `try_from` accepts every string encoding, read uniformly as `&str`:
+    let encodings = [
+        Column::<Utf8>::from_values(["alice", "bob"]).into_arrow(),
+        Column::<LargeUtf8>::from_values(["alice", "bob"]).into_arrow(),
+        Column::<Utf8View>::from_values(["alice", "bob"]).into_arrow(),
+    ];
+    for array in encodings {
+        let column = Column::<AnyUtf8>::try_from(array).unwrap();
+        assert_eq!(column.value(0), "alice");
+        assert_eq!(&column[1], "bob"); // `RefType` indexing
+        assert_eq!(column.to_vec(), ["alice", "bob"]);
+    }
+
+    // A non-string array is rejected:
+    let ints = Column::<i64>::from_values([1, 2]).into_arrow();
+    assert!(matches!(
+        Column::<AnyUtf8>::try_from(ints),
+        Err(ColumnError::WrongDatatype { .. })
+    ));
+
+    // Nullable rows via the column-level `Option`:
+    let array = LargeStringArray::from(vec![Some("x"), None]);
+    let column = Column::<Option<AnyUtf8>>::try_from(Arc::new(array) as ArrayRef).unwrap();
+    let values: Vec<Option<&str>> = column.iter().collect();
+    assert_eq!(values, [Some("x"), None]);
+
+    // A null at a non-nullable level is rejected:
+    let array = StringViewArray::from(vec![Some("x"), None]);
+    assert!(matches!(
+        Column::<AnyUtf8>::try_from(Arc::new(array) as ArrayRef),
+        Err(ColumnError::UnexpectedNulls { null_count: 1 })
+    ));
+}
+
 #[test]
 fn f16_column() {
     use quiver::half::f16;

crates/quiver_types/src/lib.rs

@@ -52,7 +52,7 @@ pub use self::list_view::{LargeListView, ListView, TypedLargeListView, TypedList
 pub use self::map::{Map, MapValue, TypedMap};
 pub use self::newtype::As;
 pub use self::run::{Run, RunEndType, TypedRun};
-pub use self::string::{LargeUtf8, Utf8, Utf8View};
+pub use self::string::{AnyTypedUtf8, AnyUtf8, LargeUtf8, Utf8, Utf8View};
 pub use self::time::{Time32Millisecond, Time32Second, Time64Microsecond, Time64Nanosecond};
 pub use self::timestamp::{
     Microsecond, Millisecond, Nanosecond, NoTimezone, Second, TimeUnitSpec, Timestamp,

crates/quiver_types/src/string.rs

@@ -11,11 +11,14 @@
 //!
 //! All three are markers: the owned values are `String`s, and reading is
 //! zero-copy — the element values are `&str` borrows into the array.
+//!
+//! [`AnyUtf8`] accepts *any* of those three encodings (they all read as `&str`),
+//! for when the encoding is decided at runtime.
 
 use arrow::array::{Array, ArrayRef};
 use arrow::datatypes::DataType;
 
-use crate::datatype::{ColumnError, LogicalType, RefType, impl_marker_datatype};
+use crate::datatype::{ColumnError, LogicalType, RefType, downcast_array, impl_marker_datatype};
 
 /// UTF-8 text: an arrow [`DataType::Utf8`] column with `String` values.
 ///
@@ -103,3 +106,89 @@ impl RefType for Utf8View {
         typed.value(index)
     }
 }
+
+/// Marker for a UTF-8 column in *any* of arrow's string encodings.
+///
+/// Accepts [`Utf8`], [`LargeUtf8`], or [`Utf8View`] — they all read as `&str`.
+///
+/// Like [`AnyList`](crate::AnyList), this is a quiver-only logical type with no
+/// single arrow datatype: `Column<AnyUtf8>` accepts whichever encoding it is
+/// handed and reads them all uniformly. It is *parse-only* — it implements
+/// [`LogicalType`] (so `try_from`/reading work) but not
+/// [`ConcreteType`](crate::ConcreteType), so there is no `from_values`/`Default`/
+/// schema; to build, pick a concrete encoding such as `Column<Utf8>`.
+///
+/// ```
+/// use quiver::{AnyUtf8, Column};
+/// use quiver::arrow::array::{ArrayRef, LargeStringArray};
+/// # use std::sync::Arc;
+///
+/// // `array` may be a Utf8 / LargeUtf8 / Utf8View:
+/// let array: ArrayRef = Arc::new(LargeStringArray::from(vec!["alice", "bob"]));
+/// let column = Column::<AnyUtf8>::try_from(array).unwrap();
+/// assert_eq!(column.value(0), "alice");
+/// ```
+///
+/// This type is never instantiated — it only appears as a type parameter.
+pub struct AnyUtf8;
+
+/// The validated representation of an [`AnyUtf8`] column: one of the
+/// per-encoding string arrays.
+#[derive(Clone)]
+pub enum AnyTypedUtf8 {
+    Utf8(arrow::array::StringArray),
+    LargeUtf8(arrow::array::LargeStringArray),
+    Utf8View(arrow::array::StringViewArray),
+}
+
+impl LogicalType for AnyUtf8 {
+    type Typed = AnyTypedUtf8;
+    type Value<'a> = &'a str;
+    type Owned = String;
+
+    fn downcast(array: &dyn Array) -> Result<Self::Typed, ColumnError> {
+        match array.data_type() {
+            DataType::Utf8 => Ok(AnyTypedUtf8::Utf8(downcast_array(array, || {
+                "Utf8".to_owned()
+            })?)),
+            DataType::LargeUtf8 => Ok(AnyTypedUtf8::LargeUtf8(downcast_array(array, || {
+                "LargeUtf8".to_owned()
+            })?)),
+            DataType::Utf8View => Ok(AnyTypedUtf8::Utf8View(downcast_array(array, || {
+                "Utf8View".to_owned()
+            })?)),
+            actual => Err(ColumnError::WrongDatatype {
+                expected: "a string array (Utf8/LargeUtf8/Utf8View)".to_owned(),
+                actual: actual.clone(),
+            }),
+        }
+    }
+
+    fn is_null(typed: &Self::Typed, index: usize) -> bool {
+        match typed {
+            AnyTypedUtf8::Utf8(array) => array.is_null(index),
+            AnyTypedUtf8::LargeUtf8(array) => array.is_null(index),
+            AnyTypedUtf8::Utf8View(array) => array.is_null(index),
+        }
+    }
+
+    fn value(typed: &Self::Typed, index: usize) -> Self::Value<'_> {
+        match typed {
+            AnyTypedUtf8::Utf8(array) => array.value(index),
+            AnyTypedUtf8::LargeUtf8(array) => array.value(index),
+            AnyTypedUtf8::Utf8View(array) => array.value(index),
+        }
+    }
+
+    fn to_owned_value(value: Self::Value<'_>) -> Self::Owned {
+        value.to_owned()
+    }
+}
+
+impl RefType for AnyUtf8 {
+    type Ref = str;
+
+    fn value_ref(typed: &Self::Typed, index: usize) -> &str {
+        Self::value(typed, index)
+    }
+}