Integrate with Mull for mutation testing

.gitignore

@@ -86,14 +86,16 @@ RunCoverage.cmake
 # Generated coverage runner
 scripts/coverage.sh
 
-
 # CTest / coverage
 CTestTestfile.cmake
 Testing/
 coverage/
 coverage.info
 coverage.cleaned.info
 
+# Mutation testing
+mutation/
+
 # IDEs (CodeBlocks/CLion/VSCode)
 *.cbp
 .idea
@@ -112,3 +114,5 @@ KML/res*centers.kml
 src/h3lib/include/h3api.h
 src/apps/benchmarks/benchmarkCountries.c
 h3-prof.trace
+
+h3.pc

CMakeLists.txt

@@ -61,6 +61,11 @@ option(BUILD_GENERATORS "Build code generation applications." ON)
 # If ON, libfuzzer settings are used to build the fuzzer harnesses. If OFF, a
 # frontend for afl++ is provided instead.
 option(ENABLE_LIBFUZZER "Build fuzzers with libFuzzer support." OFF)
+option(ENABLE_MUTATION "Build library and tests with mutation testing support." OFF)
+option(ENABLE_MUTATION_SLOW "Run some slower tests through mutation testing." OFF)
+
+set(MULL_ROOT "/opt/homebrew/opt/mull@19" CACHE STRING "Root of Mull installation")
+set(MULL_VERSION "19" CACHE STRING "Mull LLVM version")
 
 # These options exist for integration with OSS-Fuzz, so that the fuzzer options
 # can be passed through only to the fuzzer executables but not the H3 library,
@@ -130,6 +135,10 @@ if(NOT MSVC)
         list(APPEND H3_COMPILE_FLAGS -fsanitize=fuzzer,address,undefined)
         list(APPEND H3_LINK_FLAGS -fsanitize=fuzzer,address,undefined)
     endif()
+    if(ENABLE_MUTATION)
+        list(APPEND H3_COMPILE_FLAGS $<$<CONFIG:Debug>:-fpass-plugin=${MULL_ROOT}/lib/mull-ir-frontend-${MULL_VERSION} -grecord-command-line -fprofile-instr-generate -fcoverage-mapping>)
+        list(APPEND H3_LINK_FLAGS $<$<CONFIG:Debug>:-fprofile-instr-generate >)
+    endif()
 endif()
 
 option(WARNINGS_AS_ERRORS "Warnings are treated as errors" OFF)
@@ -384,9 +393,12 @@ function(add_h3_library name h3_alloc_prefix_override)
         set_target_properties(${name} PROPERTIES SOVERSION ${H3_SOVERSION})
         target_compile_definitions(${name} PRIVATE BUILD_SHARED_LIBS=1)
     endif()
-    if(ENABLE_COVERAGE)
+    if(ENABLE_COVERAGE OR ENABLE_MUTATION)
         target_compile_definitions(${name} PRIVATE H3_COVERAGE_TEST=1)
     endif()
+    if(ENABLE_MUTATION)
+        target_compile_definitions(${name} PRIVATE H3_MUTATION_TEST=1)
+    endif()
 
     target_compile_definitions(${name} PUBLIC H3_PREFIX=${H3_PREFIX})
     target_compile_definitions(${name} PRIVATE BUILDING_H3=1)
@@ -525,9 +537,12 @@ macro(add_h3_executable name)
         )
         target_compile_options(${name} PRIVATE ${H3_COMPILE_FLAGS})
         target_link_libraries(${name} PRIVATE ${H3_LINK_FLAGS})
-        if(ENABLE_COVERAGE)
+        if(ENABLE_COVERAGE OR ENABLE_MUTATION)
             target_compile_definitions(${name} PRIVATE H3_COVERAGE_TEST=1)
         endif()
+        if(ENABLE_MUTATION)
+            target_compile_definitions(${name} PRIVATE H3_MUTATION_TEST=1)
+        endif()
     endif()
 endmacro()
 

CMakeTests.cmake

@@ -54,6 +54,28 @@ if(ENABLE_COVERAGE)
         COMMENT "Zeroing counters")
 endif()
 
+if(ENABLE_MUTATION)
+    set(mutation_report_dir "${CMAKE_CURRENT_BINARY_DIR}/mutation")
+    add_custom_target(
+            clean-mutation
+            # Before running mutation, clear all counters
+            # TODO: Use ADDITIONAL_MAKE_CLEAN_FILES or BYPRODUCYS?
+            COMMAND ${CMAKE_COMMAND} -E rm -rf '${mutation_report_dir}'
+            COMMENT "Deleting mutation reports")
+    set(mutation_runner
+            "H3_MULL_SKIP_DB=${mutation_report_dir}/h3-report.sqlite"
+            "MULL_ENV=${CMAKE_CURRENT_SOURCE_DIR}/mull.yml"
+            "${MULL_ROOT}/bin/mull-runner-${MULL_VERSION}"
+            --allow-surviving
+            -reporters IDE
+            -reporters SQLite
+            -report-dir '${mutation_report_dir}'
+            -report-name h3-report
+            --test-program "${CMAKE_CURRENT_SOURCE_DIR}/scripts/mull_wrapper.sh"
+    )
+    set_property(GLOBAL PROPERTY H3_MUTATION_DONE_MARKERS "")
+endif()
+
 macro(add_h3_memory_test name srcfile)
     # Like other test code, but these need to be linked against a different copy
     # of the H3 library which has known intercepted allocator functions.
@@ -110,19 +132,42 @@ macro(add_h3_test name srcfile)
         add_dependencies(coverage ${name}_coverage${test_number})
         add_dependencies(${name}_coverage${test_number} clean-coverage)
     endif()
+
+    if(ENABLE_MUTATION
+            AND (ENABLE_MUTATION_SLOW OR (
+                    # Slow but doable <= 1hr
+                    NOT "${name}" STREQUAL "testCellToLocalIjInternal"
+                    AND NOT "${name}" STREQUAL "testCompactCells"
+                    AND NOT "${name}" STREQUAL "testCellToChildPos"
+                    # Too slow
+                    AND NOT "${name}" STREQUAL "testPolygonToCells"
+                    AND NOT "${name}" STREQUAL "testPolygonToCellsExperimental"
+                    AND NOT "${name}" STREQUAL "testPolygonToCellsReportedExperimental"
+            ))
+            # Too slow
+            AND NOT "${name}" MATCHES "Exhaustive$"
+            # Too slow on startup
+            AND NOT "${name}" STREQUAL "testGosperIter"
+    )
+        set(mutation_done "${mutation_report_dir}/${name}_${test_number}.done")
+        add_custom_command(
+            OUTPUT "${mutation_done}"
+            COMMAND "H3_MULL_REAL_EXEC=$<TARGET_FILE:${name}>" ${mutation_runner} "$<TARGET_FILE:${name}>"
+            COMMAND ${CMAKE_COMMAND} -E touch "${mutation_done}"
+            DEPENDS ${name}
+            WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+            COMMENT "Running ${name}_mutation${test_number}")
+        set_property(GLOBAL APPEND PROPERTY H3_MUTATION_DONE_MARKERS "${mutation_done}")
+    endif()
 endmacro()
 
 macro(add_h3_test_with_file name srcfile argfile)
     add_h3_test_common(${name} ${srcfile})
-    # add a special command (so we don't need to read the test file from the
-    # test program)
-    set(dump_command "cat")
 
     add_test(
         NAME ${name}_test${test_number}
         COMMAND
-            ${SHELL}
-            "${dump_command} ${argfile} | ${TEST_WRAPPER_STR} $<TARGET_FILE:${name}>"
+            ${TEST_WRAPPER} $<TARGET_FILE:${name}> ${argfile}
     )
 
     if(PRINT_TEST_FILES)
@@ -132,12 +177,24 @@ macro(add_h3_test_with_file name srcfile argfile)
     if(ENABLE_COVERAGE)
         add_custom_target(
             ${name}_coverage${test_number}
-            COMMAND ${name} < ${argfile} > /dev/null
+            COMMAND ${name} ${argfile}
             COMMENT "Running ${name}_coverage${test_number}")
 
         add_dependencies(coverage ${name}_coverage${test_number})
         add_dependencies(${name}_coverage${test_number} clean-coverage)
     endif()
+
+    if(ENABLE_MUTATION)
+        set(mutation_done "${mutation_report_dir}/${name}_${test_number}.done")
+        add_custom_command(
+                OUTPUT "${mutation_done}"
+                COMMAND "H3_MULL_REAL_EXEC=$<TARGET_FILE:${name}>" ${mutation_runner} "$<TARGET_FILE:${name}>" ${argfile}
+                COMMAND ${CMAKE_COMMAND} -E touch "${mutation_done}"
+                DEPENDS ${name}
+                WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+                COMMENT "Running ${name}_mutation${test_number}")
+        set_property(GLOBAL APPEND PROPERTY H3_MUTATION_DONE_MARKERS "${mutation_done}")
+    endif()
 endmacro()
 
 macro(add_h3_cli_test name h3_args expect_string)
@@ -172,6 +229,19 @@ macro(add_h3_test_with_arg name srcfile arg)
         add_dependencies(coverage ${name}_coverage${test_number})
         add_dependencies(${name}_coverage${test_number} clean-coverage)
     endif()
+
+    # h3NeighborRotations tests run but are slow
+    if(ENABLE_MUTATION AND ENABLE_MUTATION_SLOW)
+        set(mutation_done "${mutation_report_dir}/${name}_${test_number}.done")
+        add_custom_command(
+                OUTPUT "${mutation_done}"
+                COMMAND "H3_MULL_REAL_EXEC=$<TARGET_FILE:${name}>" ${mutation_runner} "$<TARGET_FILE:${name}>" ${arg}
+                COMMAND ${CMAKE_COMMAND} -E touch "${mutation_done}"
+                DEPENDS ${name}
+                WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+                COMMENT "Running ${name}_mutation${test_number}")
+        set_property(GLOBAL APPEND PROPERTY H3_MUTATION_DONE_MARKERS "${mutation_done}")
+    endif()
 endmacro()
 
 # Add each individual test
@@ -299,3 +369,12 @@ if(BUILD_ALLOC_TESTS)
 endif()
 
 add_custom_target(test-fast COMMAND ctest -E Exhaustive)
+
+if(ENABLE_MUTATION)
+    get_property(mutation_done_markers GLOBAL PROPERTY H3_MUTATION_DONE_MARKERS)
+    add_custom_target(mutation
+        DEPENDS ${mutation_done_markers}
+        COMMAND
+        MULL_ENV="${CMAKE_CURRENT_SOURCE_DIR}/mull.yml" "${MULL_ROOT}/bin/mull-reporter-${MULL_VERSION}" "${mutation_report_dir}/h3-report.sqlite" -reporters Elements -reporters IDE
+        COMMENT "Aggregating mutation results")
+endif()

cmake/TestWrapValgrind.cmake

@@ -31,7 +31,4 @@ if(WRAP_VALGRIND)
         ${VALGRIND} --track-origins=yes --leak-check=full --error-exitcode=99
         CACHE STRING "Wrapper executable for tests and benchmarks")
     mark_as_advanced(TEST_WRAPPER)
-    # Convert from semicolon separated list of values to a form that can be used
-    # by a shell.
-    string(REPLACE ";" " " TEST_WRAPPER_STR "${TEST_WRAPPER}")
 endif()

dev-docs/mutation.md

@@ -0,0 +1,28 @@
+Version number must match your LLVM version:
+
+```
+brew trust mull-project/mull
+brew install mull-project/mull/mull@19
+```
+
+Then, compile with that version of LLVM, enabling the mutation build support here:
+
+```
+mkdir build
+cd build
+CC=/opt/homebrew/Cellar/llvm@19/19.1.7/bin/clang-19 cmake .. -DENABLE_MUTATION=ON -DMULL_ROOT=/opt/homebrew/Cellar/mull@19/0.34.0/ -DMULL_VERSION=19 -DCMAKE_BUILD_TYPE=Debug -DBUILD_GENERATORS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DENABLE_DOCS=OFF
+```
+
+Then, run the mutation test suite:
+
+```
+make mutation
+```
+
+This should produce a final report with a mutation score. Note that not all tests are run through
+the mutation testing pipeline, specifically tests that use data files.
+
+Note: You will want to run mutation in serial. Mull already runs each mutation in
+parallel, and the H3 test scripts help exclude mutations that have already been tested.
+The intention is then you would switch on ENABLE_MUTATION_SLOW=1 and run `make mutation`
+once more.

mull.yml

@@ -0,0 +1,23 @@
+mutators:
+# - cxx_all
+ - cxx_arithmetic
+ - cxx_calls
+ - cxx_comparison
+ - cxx_logical
+ - cxx_arithmetic_assignment
+ - cxx_boundary
+
+includePaths:
+ - src/h3lib/.*$
+
+excludePaths:
+ - src/apps/.*
+
+# 5 minutes
+timeout: 300000
+
+compilationDatabasePath: compile_commands.json
+
+# Modify this to control resources used for testing:
+# parallelization:
+#   executionWorkers: 4

scripts/mull_wrapper.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+# Copyright 2026 Uber Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a wrapper that applies https://github.com/mull-project/mull/issues/1133
+# It skips mutations that have already been tested by other tests, and instead
+# just reports them as having been excluded (exit 1). This makes the test report
+# cumulative, and more importantly avoids spending time testing cases that have
+# already been excluded by quicker tests.
+
+set -u
+db=${H3_MULL_SKIP_DB-}
+real=${H3_MULL_REAL_EXEC-}
+mutant=$(env | awk -F= '$2=="1" && $1 ~ /^(cxx_|negate_)/ {print $1; exit}')
+
+if [ -n "$mutant" ] && [ -s "$db" ]; then
+    # 1=failed, 3=timeout, 4=crashed, 5=abnormal exit
+    hit=$(sqlite3 -readonly "$db" \
+    "SELECT 1 FROM mutant WHERE mutant_id='$mutant' \
+    AND execution_status in (1,3,4,5) LIMIT 1;" 2>/dev/null)
+    if [ "$hit" = 1 ]; then
+      # echo "$real $mutant" >> /tmp/mull_squash.txt
+      exit 1
+    # else
+    #   echo "$real $mutant" >> /tmp/mull_continue.txt
+    fi
+fi
+
+exec "$real" "$@"

src/apps/applib/include/test.h

@@ -29,6 +29,12 @@ extern int globalTestCount;
 extern const char *currentSuiteName;
 extern const char *currentTestName;
 
+#ifdef H3_MUTATION_TEST
+#define T_ASSERT_PRINT
+#else
+#define T_ASSERT_PRINT printf(".")
+#endif  // H3_MUTATION_TEST
+
 #define t_assert(condition, msg)                                           \
     do {                                                                   \
         if (!(condition)) {                                                \
@@ -38,7 +44,7 @@ extern const char *currentTestName;
             exit(1);                                                       \
         }                                                                  \
         globalTestCount++;                                                 \
-        printf(".");                                                       \
+        T_ASSERT_PRINT;                                                    \
     } while (0)
 
 #define t_assertSuccess(condition) t_assert(!(condition), "expected E_SUCCESS")

src/apps/testapps/testCellToBoundary.c

@@ -94,28 +94,35 @@ int readBoundary(FILE *f, CellBoundary *b) {
 
 int main(int argc, char *argv[]) {
     // check command line args
-    if (argc > 1) {
+    if (argc > 2) {
         fprintf(stderr, "usage: %s\n", argv[0]);
         exit(1);
     }
 
+    FILE *in = stdin;
+    if (argc == 2) {
+        in = fopen(argv[1], "r");
+    }
+
     // process the indexes on stdin
     char buff[BUFF_SIZE];
     while (1) {
         // get an index from stdin
-        if (!fgets(buff, BUFF_SIZE, stdin)) {
-            if (feof(stdin))
+        if (!fgets(buff, BUFF_SIZE, in)) {
+            if (feof(in))
                 break;
             else
-                error("reading input H3 index from stdin");
+                error("reading input H3 index from in");
         }
 
         H3Index h3;
         t_assertSuccess(H3_EXPORT(stringToH3)(buff, &h3));
 
         CellBoundary b;
-        readBoundary(stdin, &b);
+        readBoundary(in, &b);
 
         t_assertBoundary(h3, &b);
     }
+
+    fclose(in);
 }