cargo: exclude all test code from coverage instrumentation

.cargo/config.toml

@@ -1,3 +1,21 @@
+[alias]
+# Measure code coverage with cargo-llvm-cov and nextest. Requires a nightly
+# toolchain.
+#
+# The default test-exclusion heuristics aren't quite enough for us, so we get
+# rid of a few extra things below.
+cov = [
+    "llvm-cov",
+    "--branch",
+    "nextest",
+    "--exclude-from-report",
+    "testutils",
+    "--exclude-from-report",
+    "gen-protos",
+    "--ignore-filename-regex",
+    "(cli/testing/|lib/src/(secret_backend|test_signing_backend)\\.rs|cli/build\\.rs)",
+]
+
 [target.'cfg(all(target_family = "windows", target_env = "msvc"))']
 linker = "rust-lld.exe"
 rustflags = [

Cargo.toml

@@ -182,6 +182,10 @@ while_let_loop = "warn"
 [workspace.lints.rust]
 let_underscore_drop = "warn"
 redundant_imports = "warn"
+# set by cargo-llvm-cov, see .cargo/config.toml
+unexpected_cfgs = { level = "warn", check-cfg = [
+    'cfg(coverage, coverage_nightly)',
+] }
 
 [profile.dev.package."*"]
 # Compile all dependencies with opt-level=3 to help speed up tests and

cli/src/cli_util.rs

@@ -4820,6 +4820,7 @@ pub fn shell_quote(s: &str) -> Cow<'_, str> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use clap::CommandFactory as _;
 

cli/src/commands/arrange.rs

@@ -669,6 +669,7 @@ fn render(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use indoc::indoc;
     use maplit::hashset;

cli/src/commands/fix.rs

@@ -591,6 +591,7 @@ fn get_tools_config(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

cli/src/commands/gerrit/upload.rs

@@ -730,6 +730,7 @@ pub async fn cmd_gerrit_upload(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

cli/src/commands/mod.rs

@@ -233,6 +233,7 @@ pub async fn run_command(ui: &mut Ui, command_helper: &CommandHelper) -> Result<
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

cli/src/commands/status.rs

@@ -352,6 +352,7 @@ async fn visit_collapsed_untracked_files(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod test {
     use pollster::FutureExt as _;
     use testutils::TestRepo;

cli/src/commit_ref_list.rs

@@ -298,6 +298,7 @@ fn sort_inner(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use jj_lib::backend::ChangeId;
     use jj_lib::backend::MillisSinceEpoch;

cli/src/commit_templater.rs

@@ -3040,6 +3040,7 @@ fn builtin_trailer_list_methods<'repo>() -> CommitTemplateBuildMethodFnMap<'repo
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::path::Component;
 

cli/src/complete.rs

@@ -1407,6 +1407,7 @@ mod parse {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

cli/src/config.rs

@@ -1175,6 +1175,7 @@ impl TryFrom<Vec<String>> for NonEmptyCommandArgsVec {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::env::join_paths;
     use std::fmt::Write as _;

cli/src/description_util.rs

@@ -457,6 +457,7 @@ pub fn description_template(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use indexmap::indexmap;
     use indoc::indoc;

cli/src/formatter.rs

@@ -798,6 +798,7 @@ fn write_sanitized(output: &mut impl Write, buf: &[u8]) -> Result<(), Error> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::error::Error as _;
 

cli/src/git_util.rs

@@ -561,6 +561,7 @@ pub fn print_push_stats(ui: &Ui, stats: &GitPushStats) -> io::Result<()> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::path::MAIN_SEPARATOR;
 

cli/src/lib.rs

@@ -12,6 +12,13 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Lets items opt out of coverage instrumentation via
+// `#[cfg_attr(coverage_nightly, coverage(off))]`, which we put on every
+// `#[cfg(test)]` module so that reported numbers describe the code we ship
+// rather than the tests exercising it. `coverage_nightly` is set only by
+// cargo-llvm-cov on nightly (see `cargo cov` in .cargo/config.toml), so stable
+// builds never see the unstable feature.
+#![cfg_attr(coverage_nightly, feature(coverage_attribute))]
 #![deny(unused_must_use)]
 
 pub mod cleanup_guard;

cli/src/merge_tools/builtin.rs

@@ -763,6 +763,7 @@ async fn apply_merge_builtin(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::collections::BTreeSet;
 

cli/src/merge_tools/external.rs

@@ -527,6 +527,7 @@ pub fn invoke_external_diff(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

cli/src/merge_tools/mod.rs

@@ -494,6 +494,7 @@ async fn pick_conflict_side(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use jj_lib::config::ConfigLayer;
     use jj_lib::config::ConfigSource;

cli/src/template_builder.rs

@@ -3097,6 +3097,7 @@ fn expect_expression_of_type<'a, L: TemplateLanguage<'a> + ?Sized, T>(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::path::Component;
 

cli/src/template_parser.rs

@@ -828,6 +828,7 @@ pub fn lookup_method<'a, V>(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use assert_matches::assert_matches;
     use jj_lib::dsl_util::KeywordArgument;

cli/src/text_util.rs

@@ -602,6 +602,7 @@ pub fn parse_author(author: &str) -> Result<(String, String), &'static str> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::io::Write as _;
 

core/src/conflict_labels.rs

@@ -133,6 +133,7 @@ impl fmt::Debug for ConflictLabels {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

core/src/content_hash.rs

@@ -171,6 +171,7 @@ where
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::collections::BTreeMap;
     use std::collections::HashMap;

core/src/dag_walk.rs

@@ -249,6 +249,7 @@ fn to_infallible_iter<T>(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use assert_matches::assert_matches;
     use maplit::hashmap;

core/src/dag_walk_async.rs

@@ -601,6 +601,7 @@ where
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::pin::pin;
 

core/src/diff.rs

@@ -1023,6 +1023,7 @@ pub fn diff<'a, T: AsRef<[u8]> + ?Sized + 'a>(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

core/src/file_util.rs

@@ -481,6 +481,7 @@ mod fallback {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use futures::io::Cursor;
     use itertools::Itertools as _;

core/src/graph.rs

@@ -346,6 +346,7 @@ where
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::cell::RefCell;
     use std::convert::Infallible;

core/src/hex_util.rs

@@ -113,6 +113,7 @@ pub fn common_hex_len(bytes_a: &[u8], bytes_b: &[u8]) -> usize {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

core/src/lib.rs

@@ -44,6 +44,7 @@ pub mod str_util;
 pub mod symbol_util;
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use tempfile::TempDir;
 

core/src/matchers.rs

@@ -596,6 +596,7 @@ impl<M1: Matcher, M2: Matcher> Matcher for IntersectionMatcher<M1, M2> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use maplit::hashset;
 

core/src/merge.rs

@@ -711,6 +711,7 @@ impl<T> Merge<Merge<T>> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use test_case::test_case;
 

core/src/object_id.rs

@@ -303,6 +303,7 @@ impl<T: Clone> PrefixResolution<T> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

core/src/ref_name.rs

@@ -441,6 +441,7 @@ fn is_safe_identifier(symbol: &str) -> bool {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

core/src/repo_path.rs

@@ -720,6 +720,7 @@ impl<V: Debug> Debug for RepoPathTree<V> {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use std::panic;
 

core/src/str_util.rs

@@ -586,6 +586,7 @@ impl Debug for StringMatcher {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use assert_matches::assert_matches;
     use itertools::Itertools as _;

docs/contributing.md

@@ -258,6 +258,31 @@ These are listed roughly in order of decreasing importance.
    On Linux, you may be able to speed up `nextest` even further by using
    the `mold` linker, as explained below.
 
+### Measuring code coverage
+
+Install [`cargo-llvm-cov`](https://github.com/taiki-e/cargo-llvm-cov) and
+`cargo-nextest` (see above), then run:
+
+```shell
+cargo +nightly cov          # add --html to get a browsable report
+```
+
+`cov` is an alias defined in `.cargo/config.toml` which enables a consistent
+set of options for all contributors. The nightly Rust toolchain is required (the
+compiler instrumentation feature itself is nightly only).
+
+When you are writing new test modules, make sure to turn coverage off for
+that module so it doesn't skew the resultant accounting; this can be done by
+copy/pasting the following `#[cfg_attr]` after your `#[cfg(test)]` stanza:
+
+```rust
+#[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
+mod tests {
+    // ...
+}
+```
+
 ### Configuring `jj fix` to run `rustfmt`
 
 Run this in the jj repo:

lib/src/absorb.rs

@@ -387,6 +387,7 @@ fn to_file_value(value: MaterializedTreeValue) -> Result<Option<MaterializedFile
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use maplit::hashmap;
 

lib/src/annotate.rs

@@ -478,6 +478,7 @@ async fn get_file_contents(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

lib/src/backend.rs

@@ -890,6 +890,7 @@ impl dyn Backend {
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use super::*;
 

lib/src/config.rs

@@ -913,6 +913,7 @@ static DEFAULT_CONFIG_LAYERS: LazyLock<[Arc<ConfigLayer>; 1]> = LazyLock::new(||
 });
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use assert_matches::assert_matches;
     use indoc::indoc;

lib/src/config_resolver.rs

@@ -455,6 +455,7 @@ fn migrate_layer(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     use assert_matches::assert_matches;
     use indoc::indoc;

lib/src/conflicts.rs

@@ -1133,6 +1133,7 @@ pub async fn update_from_content(
 }
 
 #[cfg(test)]
+#[cfg_attr(coverage_nightly, coverage(off))]
 mod tests {
     #![expect(clippy::too_many_arguments)]