stdlib-tests.rs

//! Execute unit tests in the Noir standard library.

use clap::Parser;
use fm::FileManager;
use nargo::foreign_calls::DefaultForeignCallBuilder;
use noirc_driver::{CompileOptions, check_crate, file_manager_with_stdlib};
use noirc_frontend::error_reporting::report_one;
use noirc_frontend::hir::{FunctionNameMatch, ParsedFiles};
use std::io::Write;
use std::{collections::BTreeMap, path::PathBuf};

use nargo::{
    ops::{TestStatus, report_errors, run_test, test_status_comptime_interpret_result},
    package::{Package, PackageType},
    parse_all, prepare_package,
};
use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
use test_case::test_matrix;

#[derive(Parser, Debug)]
#[command(ignore_errors = true)]
pub struct Options {
    /// Test name to filter for.
    ///
    /// First is assumed to be `run_stdlib_tests` and the second the of the stdlib tests, e.g.:
    ///
    /// ```text
    /// cargo test -p nargo_cli --test stdlib-tests -- run_stdlib_tests sha256
    /// ```
    args: Vec<String>,
}

impl Options {
    pub fn function_name_match(&self) -> FunctionNameMatch {
        match self.args.as_slice() {
            [_test_name, lib] => FunctionNameMatch::Contains(vec![lib.clone()]),
            _ => FunctionNameMatch::Anything,
        }
    }
}

/// Controls which execution mode is forced during stdlib testing.
#[derive(Clone, Copy)]
enum Force {
    /// Standard ACIR compilation with no overrides.
    Nothing,
    /// Force all functions to compile as Brillig (unconstrained).
    Brillig,
    /// Execute tests directly in the comptime interpreter, bypassing SSA compilation.
    Comptime,
}

/// Inliner aggressiveness results in different SSA.
/// Inlining happens if `inline_cost - retain_cost < aggressiveness` (see `inlining.rs`).
/// NB the CLI uses maximum aggressiveness.
///
/// Even with the same inlining aggressiveness, forcing Brillig or using the comptime
/// interpreter can trigger different behavior.
#[test_matrix(
    [Force::Nothing, Force::Brillig, Force::Comptime],
    [i64::MIN, 0, i64::MAX]
)]
fn run_stdlib_tests(force: Force, inliner_aggressiveness: i64) {
    let opts = Options::parse();

    let mut file_manager = file_manager_with_stdlib(&PathBuf::from("."));
    file_manager.add_file_with_source_canonical_path(&PathBuf::from("main.nr"), "".to_owned());
    let parsed_files = parse_all(&file_manager);

    // We need a dummy package as we cannot compile the stdlib on its own.
    let dummy_package = Package {
        version: None,
        compiler_required_version: None,
        compiler_required_unstable_features: Vec::new(),
        root_dir: PathBuf::from("."),
        package_type: PackageType::Binary,
        entry_path: PathBuf::from("main.nr"),
        name: "stdlib".parse().unwrap(),
        dependencies: BTreeMap::new(),
    };

    let (mut context, dummy_crate_id) =
        prepare_package(&file_manager, &parsed_files, &dummy_package);

    let result = check_crate(&mut context, dummy_crate_id, &Default::default());
    report_errors(result, &context.file_manager, &context.parsed_files, true, false)
        .expect("Error encountered while compiling standard library");

    // We can now search within the stdlib for any test functions to compile.

    let test_functions = context.get_all_test_functions_in_crate_matching(
        context.stdlib_crate_id(),
        &opts.function_name_match(),
    );

    let context = std::sync::Mutex::new(context);

    let test_report: Vec<(String, TestStatus)> = test_functions
        .into_iter()
        .map(|(test_name, test_function)| {
            let mut context = match context.lock() {
                Ok(guard) => guard,
                Err(poisoned) => poisoned.into_inner(), // Ignore, it happened during execution.
            };
            let status = match force {
                Force::Nothing | Force::Brillig => {
                    let force_brillig = matches!(force, Force::Brillig);
                    let result = std::panic::catch_unwind(move || {
                        run_test(
                            &bn254_blackbox_solver::Bn254BlackBoxSolver,
                            &mut context,
                            &test_function,
                            std::io::stdout(),
                            &CompileOptions {
                                force_brillig,
                                inliner_aggressiveness,
                                ..Default::default()
                            },
                            |output, base| {
                                DefaultForeignCallBuilder::default()
                                    .with_output(output)
                                    .build_with_base(base)
                            },
                        )
                    });
                    match result {
                        Ok(status) => status,
                        Err(_panic_cause) => TestStatus::Fail {
                            message: "panicked; see details in the end summary".to_string(),
                            error_diagnostic: None,
                        },
                    }
                }
                Force::Comptime => {
                    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                        let result = context.interpret_function(test_function.id, Vec::new());
                        test_status_comptime_interpret_result(result, &test_function)
                    }));
                    match result {
                        Ok(status) => status,
                        Err(_panic_cause) => TestStatus::Fail {
                            message: "panicked; see details in the end summary".to_string(),
                            error_diagnostic: None,
                        },
                    }
                }
            };
            (test_name, status)
        })
        .collect();

    assert!(!test_report.is_empty(), "Could not find any tests within the stdlib");
    display_test_report(
        &file_manager,
        &parsed_files,
        &dummy_package,
        &CompileOptions::default(),
        &test_report,
    );
    assert!(test_report.iter().all(|(_, status)| !status.failed()));
}

// This code is copied from `src/cli/test_cmd.rs`.
// This should be abstracted into a proper test runner at some point.
fn display_test_report(
    file_manager: &FileManager,
    parsed_files: &ParsedFiles,
    package: &Package,
    compile_options: &CompileOptions,
    test_report: &[(String, TestStatus)],
) {
    let writer = StandardStream::stderr(ColorChoice::Always);
    let mut writer = writer.lock();

    for (test_name, test_status) in test_report {
        write!(writer, "[{}] Testing {test_name}... ", package.name)
            .expect("Failed to write to stderr");
        writer.flush().expect("Failed to flush writer");

        match &test_status {
            TestStatus::Pass => {
                writer
                    .set_color(ColorSpec::new().set_fg(Some(Color::Green)))
                    .expect("Failed to set color");
                writeln!(writer, "ok").expect("Failed to write to stderr");
            }
            TestStatus::Fail { message, error_diagnostic } => {
                writer
                    .set_color(ColorSpec::new().set_fg(Some(Color::Red)))
                    .expect("Failed to set color");
                writeln!(writer, "FAIL\n{message}\n").expect("Failed to write to stderr");
                if let Some(diag) = error_diagnostic {
                    report_one(
                        diag,
                        file_manager,
                        parsed_files,
                        compile_options.deny_warnings,
                        compile_options.silence_warnings,
                    );
                }
            }
            TestStatus::Skipped => {
                writer
                    .set_color(ColorSpec::new().set_fg(Some(Color::Yellow)))
                    .expect("Failed to set color");
                writeln!(writer, "skipped").expect("Failed to write to stderr");
            }
            TestStatus::CompileError(err) => {
                report_one(
                    err,
                    file_manager,
                    parsed_files,
                    compile_options.deny_warnings,
                    compile_options.silence_warnings,
                );
            }
        }
        writer.reset().expect("Failed to reset writer");
    }

    write!(writer, "[{}] ", package.name).expect("Failed to write to stderr");

    let count_all = test_report.len();
    let count_failed = test_report.iter().filter(|(_, status)| status.failed()).count();
    let plural = if count_all == 1 { "" } else { "s" };
    if count_failed == 0 {
        writer.set_color(ColorSpec::new().set_fg(Some(Color::Green))).expect("Failed to set color");
        write!(writer, "{count_all} test{plural} passed").expect("Failed to write to stderr");
        writer.reset().expect("Failed to reset writer");
        writeln!(writer).expect("Failed to write to stderr");
    } else {
        let count_passed = count_all - count_failed;
        let plural_failed = if count_failed == 1 { "" } else { "s" };
        let plural_passed = if count_passed == 1 { "" } else { "s" };

        if count_passed != 0 {
            writer
                .set_color(ColorSpec::new().set_fg(Some(Color::Green)))
                .expect("Failed to set color");
            write!(writer, "{count_passed} test{plural_passed} passed, ",)
                .expect("Failed to write to stderr");
        }

        writer.set_color(ColorSpec::new().set_fg(Some(Color::Red))).expect("Failed to set color");
        writeln!(writer, "{count_failed} test{plural_failed} failed")
            .expect("Failed to write to stderr");
        writer.reset().expect("Failed to reset writer");
    }
}