json.rs

static USAGE: &str = r#"
Convert JSON to CSV.

The JSON data is expected to be non-empty and non-nested as either:

1. An array of objects where:
   A. All objects are non-empty, have non-empty and unique keys, and the same keys are in each object.
   B. Values are not objects or arrays.
2. An object where values are not objects or arrays and the object is as described above.

Objects with duplicate keys are not recommended as only one key and its values may be used.

If your JSON data is not in the expected format and/or is nested or complex, try using
the --jaq option to pass a jq-like filter before parsing with the above constraints.
Learn more about jaqhere: https://github.com/01mf02/jaq

As an example, say we have the following JSON data in a file fruits.json:

[
    {
        "fruit": "apple",
        "price": 2.50,
        "calories": 95
    },
    {
        "fruit": "banana",
        "price": 3.00,
        "calories": 105
    }
]

To convert it to CSV format run:

  $ qsv json fruits.json

And the following is printed to the terminal:

fruit,price,calories
apple,2.5,95
banana,3.0,105

IMPORTANT:
  The order of the columns in the CSV file will be the same as the order of the keys in the first JSON object.
  The order of the rows in the CSV file will be the same as the order of the objects in the JSON array.

  Additional keys not present in the first JSON object will be appended as additional columns in the
  output CSV in the order they appear.

For example, say we have the following JSON data in a file fruits2.json:

[
    {
        "fruit": "apple",
        "cost": 1.75,
        "price": 2.50,
        "calories": 95
    },
    {
        "fruit": "mangosteen",
        "price": 5.00,
        "calories": 56
    },
    {
        "fruit": "starapple",
        "rating": 9,
        "price": 4.50,
        "calories": 95,
    },
    {
        "fruit": "banana",
        "price": 3.00,
        "calories": 105
    }
]

If we run the following command:

  $ qsv json fruits2.json | qsv table

The output CSV will have the following columns:

fruit       cost  price  calories  rating
apple       1.75  2.5    95
mangosteen        5.0    56
starapple         4.5    95        9
banana            3.0    105

Note that the "rating" column is added as an additional column in the output CSV,
though it appears as the 2nd column in the third JSON object for "starapple".

If you want to select/reorder/drop columns in the output CSV, use the --select option, for example:

  $ qsv json fruits.json --select price,fruit

The following is printed to the terminal:

price,fruit
2.5,apple
3.0,banana

Note: Trailing zeroes in decimal numbers from the input data are truncated (2.50 becomes 2.5)
because JSON decimals (numbers with a fractional part or exponent) round-trip through f64.
Plain integer tokens are preserved exactly (serde_json parses them as i64/u64). With --jaq,
integers that don't fit i64/u64 are emitted verbatim as strings to avoid silent precision
loss, and decimal literals written inside the --jaq filter expression itself are passed
through verbatim (2.50 stays 2.50, scientific notation is kept as written).

If the JSON data was provided using stdin then either use - or do not provide a file path.
For example you may copy the JSON data above to your clipboard then run:

  $ qsv clipboard | qsv json

Again, when JSON data is nested or complex, try using the --jaq option and provide a filter value.

For example we have a .json file with a "data" key and the value being the same array as before:

{
    "data": [...]
}

We may run the following to select the JSON file and convert the nested array to CSV:

  $ qsv prompt -F json | qsv json --jaq .data

For more examples, see https://github.com/dathere/qsv/blob/master/tests/test_json.rs.
See also https://github.com/dathere/qsv/wiki/Conversion-and-IO#json

Usage:
    qsv json [options] [<input>]
    qsv json --help

json options:
    --jaq <filter>         Filter JSON data using jaq syntax (https://github.com/01mf02/jaq),
                           which is identical to the popular JSON command-line tool - jq.
                           https://jqlang.github.io/jq/
                           Note that the filter is applied BEFORE converting JSON to CSV
    -s, --select <cols>    Select, reorder or drop columns for output.
                           Otherwise, all the columns will be output in the same order as
                           the first object's keys in the JSON data.
                           See 'qsv select --help' for the full syntax.
                           Note however that <cols> NEED to be a comma-delimited list
                           of column NAMES and NOT column INDICES.
                           [default: 1- ]

Common options:
    -h, --help             Display this message
    -o, --output <file>    Write output to <file> instead of stdout.
"#;

use std::{
    collections::HashMap,
    env,
    io::{Read, Write},
};

use jaq_core::{Compiler, Ctx, Vars, data, load, unwrap_valr};
use jaq_json::{Num, Val};
use log::warn;
use serde::Deserialize;
use serde_json::Value;

use crate::{CliError, CliResult, config, select::SelectColumns, util};

#[derive(Deserialize)]
struct Args {
    arg_input:   Option<String>,
    flag_jaq:    Option<String>,
    flag_select: Option<SelectColumns>,
    flag_output: Option<String>,
}

pub fn run(argv: &[&str]) -> CliResult<()> {
    fn get_value_from_stdin() -> CliResult<serde_json::Value> {
        // Create a buffer in memory for stdin
        let mut buffer: Vec<u8> = Vec::new();
        let stdin = std::io::stdin();
        let mut stdin_handle = stdin.lock();
        stdin_handle.read_to_end(&mut buffer)?;
        drop(stdin_handle);

        // Return the JSON contents of the buffer as serde_json::Value
        match serde_json::from_slice(&buffer) {
            Ok(value) => Ok(value),
            Err(err) => fail_clierror!("Failed to parse JSON from stdin: {err}"),
        }
    }

    fn get_value_from_path(path: String) -> CliResult<serde_json::Value> {
        // Open the file in read-only mode with buffer.
        let file = std::fs::File::open(path)?;
        let reader = std::io::BufReader::with_capacity(config::DEFAULT_RDR_BUFFER_CAPACITY, file);

        // Return the JSON contents of the file as serde_json::Value
        match serde_json::from_reader(reader) {
            Ok(value) => Ok(value),
            Err(err) => fail_clierror!("Failed to parse JSON from file: {err}"),
        }
    }

    let args: Args = util::get_args(USAGE, argv)?;

    let mut value = match args.arg_input {
        Some(path) if path == "-" => get_value_from_stdin()?,
        Some(path) => get_value_from_path(path)?,
        None => get_value_from_stdin()?,
    };

    if value.is_null() {
        return fail_clierror!("No JSON data found.");
    }

    if let Some(filter) = args.flag_jaq {
        // Parse jaq filter based on JSON input

        // Create the program from filter string
        let program = load::File {
            code: filter.as_str(),
            path: (),
        };

        // Setup loader and arena
        let loader = load::Loader::new(
            jaq_core::defs()
                .chain(jaq_std::defs())
                .chain(jaq_json::defs()),
        );
        let arena = load::Arena::default();

        // Parse the filter
        let modules = loader
            .load(&arena, program)
            .map_err(|e| CliError::Other(format!("Failed to parse jaq query: {e:?}")))?;

        // Compile the filter
        let jaq_filter = Compiler::default()
            .with_funs(
                jaq_core::funs()
                    .chain(jaq_std::funs())
                    .chain(jaq_json::funs()),
            )
            .compile(modules)
            .map_err(|e| CliError::Other(format!("Failed to compile jaq query: {e:?}")))?;

        // Convert serde_json::Value to jaq Val
        let input: Val = serde_json::from_value(value.clone())
            .map_err(|e| CliError::Other(format!("Failed to convert JSON to jaq value: {e}")))?;

        // Run the filter and convert jaq output back to serde_json::Value.
        // `first_dropped_err` captures the first error encountered while
        // dropping values — either a jaq filter runtime error or a
        // Val-to-JSON conversion error — so the user-facing message can
        // surface the underlying cause when no values survive.
        let ctx = Ctx::<data::JustLut<Val>>::new(&jaq_filter.lut, Vars::new([]));
        let mut first_dropped_err: Option<String> = None;
        let jaq_values: Vec<serde_json::Value> = jaq_filter
            .id
            .run((ctx, input))
            .map(unwrap_valr)
            .filter_map(|r| {
                let msg = match r {
                    Ok(v) => match val_to_json_value(v) {
                        Ok(val) => return Some(val),
                        Err(e) => format!("jaq Val could not be converted to JSON: {e}"),
                    },
                    Err(e) => format!("jaq filter runtime error: {e}"),
                };
                warn!("{msg} (value dropped)");
                first_dropped_err.get_or_insert(msg);
                None
            })
            .collect();

        if jaq_values.is_empty() {
            if let Some(msg) = first_dropped_err {
                return fail_clierror!("jaq query returned no results: {msg}");
            }
            return fail_clierror!("jaq query returned no results.");
        }

        let jaq_value = if jaq_values.len() == 1 {
            jaq_values.into_iter().next().unwrap()
        } else {
            serde_json::Value::Array(jaq_values)
        };

        // If the result is an array wrapping another array, unwrap it.
        // This allows the user to filter with '.data' for {"data": [...]} instead of not being able
        // to use '.data'. Both '.data' and '.data[]' should work with this implementation.
        value = if jaq_value
            .as_array()
            .is_some_and(|arr| arr.first().is_some_and(serde_json::Value::is_array))
        {
            jaq_value.as_array().unwrap().first().unwrap().to_owned()
        } else {
            jaq_value
        };
    }

    if value.is_null() {
        return fail_clierror!("All JSON data filtered.");
    }

    let first_dict = if value.is_array() {
        value
            .as_array()
            .and_then(|arr| arr.first())
            .and_then(|val| val.as_object())
            .ok_or_else(|| CliError::Other("Expected an array of objects in JSON".to_string()))?
    } else {
        value
            .as_object()
            .ok_or_else(|| CliError::Other("Expected a JSON object".to_string()))?
    };
    if first_dict.is_empty() {
        return Err(CliError::Other(
            "Expected a non-empty JSON object".to_string(),
        ));
    }
    let mut first_dict_headers: Vec<&str> = Vec::new();
    for key in first_dict.keys() {
        if key.is_empty() {
            return Err(CliError::Other("Expected a non-empty JSON key".to_string()));
        }
        if first_dict_headers.contains(&key.as_str()) {
            return Err(CliError::Other(format!(
                "Expected non-duplicate keys, found key: {key}"
            )));
        }
        first_dict_headers.push(key.as_str());
    }

    // STEP 1: create an intermediate CSV tempfile from the JSON data
    // we need to do this so we can use qsv select to reorder headers to first dict's keys order
    // as the order of the headers in the CSV file is not guaranteed to be the same as the order of
    // the keys in the JSON object
    let temp_dir = env::temp_dir();
    let intermediate_csv = tempfile::Builder::new()
        .suffix(".csv")
        .tempfile_in(&temp_dir)?
        .into_temp_path()
        .to_string_lossy()
        .into_owned();

    // convert JSON to CSV
    // its inside a block so all the unneeded resources are freed & flushed after the block ends
    {
        // Borrow either the existing array's slice or a 1-element slice over
        // `value` itself. `slice::from_ref` avoids cloning the value — for
        // an array input we never touch it, and for a single object we just
        // wrap the existing reference.
        let values: &[serde_json::Value] = match value.as_array() {
            Some(arr) => arr.as_slice(),
            None => std::slice::from_ref(&value),
        };

        let intermediate_csv_writer = csv::WriterBuilder::new()
            .has_headers(true)
            .from_path(intermediate_csv.clone())?;
        json_objects_to_csv(values, intermediate_csv_writer)?;
    }

    // STEP 2: select the columns to use in the final output
    // Read the intermediate CSV to get the actual headers (which, since we preserve key order,
    // appear in the order the keys were first encountered in the JSON data)
    let sel_rconfig = config::Config::new(Some(intermediate_csv).as_ref()).no_headers(false);
    let mut intermediate_csv_rdr = sel_rconfig.reader()?;
    let byteheaders = intermediate_csv_rdr.byte_headers()?;

    // Convert byte headers to string headers for easier processing
    let actual_headers: Vec<String> = byteheaders
        .iter()
        .map(|h| util::bytes_to_cow_str(h).into_owned())
        .collect();

    // If --select is not specified, reorder the actual headers to match the first dict's key order
    let sel_cols = if let Some(select_cols) = args.flag_select {
        select_cols
    } else {
        // If all expected headers exist, use the original order
        if first_dict_headers
            .iter()
            .all(|&h| actual_headers.contains(&h.to_string()))
        {
            SelectColumns::parse(&first_dict_headers.join(",")).map_err(|e| {
                CliError::Other(format!(
                    "Failed to parse select columns in order of the first JSON dict: {e}"
                ))
            })?
        } else {
            // Otherwise, just use the headers as they appear in the CSV
            SelectColumns::parse(&actual_headers.join(","))
                .map_err(|e| CliError::Other(format!("Failed to parse select columns: {e}")))?
        }
    };

    // and write the selected columns to the final CSV file
    let sel = sel_rconfig.select(sel_cols).selection(byteheaders)?;
    let mut read_record = csv::ByteRecord::new();
    let mut write_record = csv::ByteRecord::new();
    let mut final_csv_wtr = config::Config::new(args.flag_output.as_ref())
        .no_headers(false)
        .writer()?;
    final_csv_wtr.write_record(sel.iter().map(|&i| &byteheaders[i]))?;
    while intermediate_csv_rdr.read_byte_record(&mut read_record)? {
        write_record.clear();
        write_record.extend(sel.iter().map(|&i| &read_record[i]));
        final_csv_wtr.write_byte_record(&write_record)?;
    }

    Ok(final_csv_wtr.flush()?)
}

/// User-facing key separator for flattened nested keys (e.g. `parent.child`, `arr.0`).
const FLATTEN_SEP: &str = ".";

/// Flatten a JSON `Value` into `out` as `(path_segments, leaf_value)` pairs, in document order,
/// mirroring the default behavior of the (now-removed, unmaintained) `json-objects-to-csv` /
/// `flatten-json-object` crates:
/// - each nesting level contributes one path segment (an object key, or an array index),
/// - empty objects and arrays are dropped (they contribute no leaf),
/// - empty-string keys and JSON `null` values are preserved,
/// - the top-level value must be an object.
///
/// Keeping the path as a `Vec<String>` of raw segments (rather than a pre-joined string with a
/// magic separator) lets a later join with `FLATTEN_SEP` produce the display header while still
/// distinguishing structurally-different paths that happen to join to the same header (e.g. a
/// genuinely nested `a`→`b` vs. a literal key `"a.b"`), with no risk of corrupting keys that
/// literally contain the separator.
fn flatten_json_value(
    current: &Value,
    path: &mut Vec<String>,
    depth: u32,
    out: &mut Vec<(Vec<String>, Value)>,
) -> CliResult<()> {
    if depth == 0 && !current.is_object() {
        return fail_clierror!("Flattening error: top-level JSON value must be an object");
    }

    if let Some(obj) = current.as_object() {
        // empty objects are not preserved (nothing to iterate)
        for (k, v) in obj {
            path.push(k.clone());
            flatten_json_value(v, path, depth + 1, out)?;
            path.pop();
        }
    } else if let Some(arr) = current.as_array() {
        // empty arrays are not preserved (nothing to iterate)
        for (i, v) in arr.iter().enumerate() {
            path.push(i.to_string());
            flatten_json_value(v, path, depth + 1, out)?;
            path.pop();
        }
    } else {
        out.push((path.clone(), current.clone()));
    }
    Ok(())
}

/// Convert a slice of JSON `Value`s (each expected to be an object) into CSV, writing to
/// `csv_writer`. Each object becomes one CSV row; the header row is the union of all flattened
/// keys, in the order they are first encountered (key order is preserved). Missing keys are
/// written as empty fields, and `null`/empty values become empty fields.
///
/// This is a focused reimplementation of the parts of the unmaintained `json-objects-to-csv`
/// crate that qsv relied on, so we no longer depend on it (see issue #3523). qsv builds
/// `serde_json` with the `preserve_order` feature, so object keys are already in insertion order.
fn json_objects_to_csv<W: Write>(
    objects: &[Value],
    mut csv_writer: csv::Writer<W>,
) -> CliResult<()> {
    if objects.is_empty() {
        return Ok(());
    }

    // First pass: collect the header union in first-seen order, mapping each display header to the
    // structural path that produced it. If a second, structurally-different path normalizes to an
    // already-seen header, that's a collision (e.g. nested `a`→`b` vs. a literal `"a.b"` key).
    let mut headers: Vec<String> = Vec::new();
    let mut header_paths: HashMap<String, Vec<String>> = HashMap::new();
    let mut leaves: Vec<(Vec<String>, Value)> = Vec::new();
    let mut path: Vec<String> = Vec::new();
    for obj in objects {
        leaves.clear();
        path.clear();
        flatten_json_value(obj, &mut path, 0, &mut leaves)?;
        for (segments, _) in &leaves {
            let header = segments.join(FLATTEN_SEP);
            match header_paths.get(&header) {
                Some(existing) if existing != segments => {
                    return fail_clierror!(
                        "Flattening Key Collision error: two different JSON keys collide after \
                         flattening to '{header}'"
                    );
                },
                Some(_) => {},
                None => {
                    header_paths.insert(header.clone(), segments.clone());
                    headers.push(header);
                },
            }
        }
    }

    // nothing to write if no headers were produced (e.g. all-empty objects)
    if headers.is_empty() {
        return Ok(());
    }

    csv_writer.write_record(&headers)?;

    // Second pass: write one CSV row per object.
    let mut record: Vec<String> = Vec::with_capacity(headers.len());
    let mut row: HashMap<String, Value> = HashMap::new();
    for obj in objects {
        leaves.clear();
        path.clear();
        flatten_json_value(obj, &mut path, 0, &mut leaves)?;

        row.clear();
        for (segments, val) in leaves.drain(..) {
            row.insert(segments.join(FLATTEN_SEP), val);
        }

        record.clear();
        for header in &headers {
            match row.get(header) {
                Some(Value::String(s)) => record.push(s.clone()),
                Some(v @ (Value::Bool(_) | Value::Number(_))) => record.push(v.to_string()),
                // Null, plus any empty Array/Object that slipped through, become empty fields.
                _ => record.push(String::new()),
            }
        }
        csv_writer.write_record(&record)?;
    }

    csv_writer.flush()?;
    Ok(())
}

/// Convert a jaq `Val` to a `serde_json::Value` without going through Display.
///
/// This avoids the `format!("{v}")` → `serde_json::from_str` round-trip which can
/// produce non-JSON output for NaN, Infinity, byte strings, and objects with non-string keys.
///
/// Note on numeric precision (CSV-sink-specific): `serde_json` is built without
/// the `arbitrary_precision` feature in this project, so `serde_json::Number` is
/// internally `i64`/`u64`/`f64`. To avoid silently rounding, `BigInts` that don't
/// fit `i64`/`u64` and `Num::Dec` values (which jaq uses precisely to preserve
/// decimals that did not parse as integers) are emitted as `Value::String` so
/// their original digits round-trip verbatim into the CSV output.
///
/// Caveat: this means the returned `serde_json::Value` is **not** a faithful
/// re-encoding of the input JSON — re-serializing it to JSON would produce
/// quoted strings where the original had numeric literals. This helper is
/// intended to feed `qsv json`'s CSV writer (where every cell is text, so
/// quoting is invisible). Do not lift it to a JSON sink without revisiting
/// these branches.
fn val_to_json_value(v: Val) -> Result<serde_json::Value, String> {
    match v {
        Val::Null => Ok(serde_json::Value::Null),
        Val::Bool(b) => Ok(serde_json::Value::Bool(b)),
        Val::Num(ref n) => match n {
            Num::Int(i) => Ok(serde_json::Value::Number((*i as i64).into())),
            Num::Float(f) => {
                if f.is_finite() {
                    serde_json::Number::from_f64(*f)
                        .map(serde_json::Value::Number)
                        .ok_or_else(|| format!("cannot represent {f} as JSON number"))
                } else {
                    // NaN and Infinity have no JSON representation — map to null like jq does
                    warn!("non-finite float {f} converted to null");
                    Ok(serde_json::Value::Null)
                }
            },
            Num::BigInt(bi) => {
                // Use TryFrom (prelude) to avoid an unconditional to_string allocation
                // in the in-range case. We only stringify when the value overflows u64
                // and we need a verbatim digit-preserving fallback for the CSV cell.
                // This also avoids serde_json's f64 round-trip, which would silently
                // lose precision for values outside ±2^53.
                if let Ok(i) = i64::try_from(&**bi) {
                    Ok(serde_json::Value::Number(i.into()))
                } else if let Ok(u) = u64::try_from(&**bi) {
                    Ok(serde_json::Value::Number(u.into()))
                } else {
                    warn!("BigInt {bi} out of range for i64/u64 JSON number, converting to string");
                    Ok(serde_json::Value::String(bi.to_string()))
                }
            },
            Num::Dec(s) => {
                // jaq's Dec preserves decimals that didn't parse as integer. Without
                // serde_json's arbitrary_precision feature, parse::<Number>() would
                // round to f64 and silently drop precision. The downstream sink is
                // CSV (text), so pass the original string through verbatim.
                Ok(serde_json::Value::String(s.to_string()))
            },
        },
        Val::TStr(ref s) => Ok(serde_json::Value::String(
            util::bytes_to_cow_str(s).into_owned(),
        )),
        Val::BStr(ref s) => {
            warn!("byte string converted to lossy UTF-8 string for JSON");
            Ok(serde_json::Value::String(
                util::bytes_to_cow_str(s).into_owned(),
            ))
        },
        Val::Arr(a) => {
            let items: Result<Vec<_>, _> = a.iter().cloned().map(val_to_json_value).collect();
            Ok(serde_json::Value::Array(items?))
        },
        Val::Obj(o) => {
            let mut map = serde_json::Map::new();
            for (k, v) in o.iter() {
                let key = match k {
                    Val::TStr(s) => util::bytes_to_cow_str(s).into_owned(),
                    other => {
                        warn!("non-string object key converted to string: {other}");
                        format!("{other}")
                    },
                };
                map.insert(key, val_to_json_value(v.clone())?);
            }
            Ok(serde_json::Value::Object(map))
        },
    }
}