test/fuzz: fuzzing for httpfast and tpleval

Author: sqllqq

.gitignore

@@ -25,3 +25,9 @@ CTestTestfile.cmake
 .idea/
 build/
 packpack/
+./corpus
+.vscode
+*.dSYM
+*.profdata
+*.profraw
+*.log

test/fuzz/README.md

@@ -0,0 +1,33 @@
+## Fuzzing
+
+Clone to home dir
+```bash
+git clone https://github.com/llvm-mirror/compiler-rt.git
+```
+
+Build fuzzers
+```bash
+clang -fsanitize=fuzzer,fuzzer-no-link -g -I../../http tplevah_fuzzer.c -o tplevah_fuzzer
+clang -fsanitize=fuzzer,fuzzer-no-link -g -I../../http httpfast_fuzzer.c -o httpfast_fuzzer
+```
+
+Run fuzzers
+```bash
+python3 test/fuzz/fuzz_until.py tpleval --libfuzzer-log tpleval.log --corpus-dir ./corpus/tpleval &
+python3 test/fuzz/fuzz_until.py httpfast --libfuzzer-log httpfast.log --corpus-dir ./corpus/httpfast &
+```
+
+Build for reports
+```bash
+clang -fprofile-instr-generate -fcoverage-mapping -o httpfast_fuzzer -g -I../../http httpfast_fuzzer.c ~/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c
+clang -fprofile-instr-generate -fcoverage-mapping -o tplevah_fuzzer -g -I../../http tplevah_fuzzer.c ~/compiler-rt/lib/fuzzer/standalone/StandaloneFuzzTargetMain.c
+```
+
+Generate reports
+```bash
+llvm-profdata merge -sparse httpfast.profraw -o httpfast.profdata
+llvm-profdata merge -sparse tpleval.profraw -o tpleval.profdata
+
+llvm-cov show ./test/fuzz/tpleval_fuzzer -instr-profile=tpleval.profdata -format=html -output-dir=coverage_tpleval
+llvm-cov show ./test/fuzz/httpfast_fuzzer -instr-profile=httpfast.profdata -format=html -output-dir=coverage_tpleval
+```

test/fuzz/fuzz_until.py

@@ -0,0 +1,201 @@
+import time
+import subprocess
+from argparse import ArgumentParser
+from enum import Enum, auto
+from dataclasses import dataclass
+from pathlib import Path
+
+
+class MsgType(Enum):
+    """Type of libfuzzer log message"""
+
+    READ = auto()
+    INITED = auto()
+    NEW = auto()
+    REDUCE = auto()
+    pulse = auto()
+    DONE = auto()
+    RELOAD = auto()
+
+
+@dataclass(repr=True)
+class Msg:
+    """Libfuzzer log message contents"""
+
+    ty: MsgType
+    n_inputs: int
+    cov: int | None
+    ft: int | None
+
+
+# See libfuzzer output format: https://llvm.org/docs/LibFuzzer.html#output
+def parse_line(line: str) -> Msg | None:
+    """Parse libfuzzer log message"""
+
+    line = line.split()
+
+    msg_ty = None
+    for ty in MsgType:
+        if ty.name in line:
+            msg_ty = ty
+            break
+    if not msg_ty:
+        return None
+
+    n_inputs = 0
+    if line[0][0] == "#":
+        n_inputs = int(line[0][1:])
+
+    cov = None
+    if "cov:" in line:
+        cov_i = line.index("cov:")
+        cov = int(line[cov_i + 1])
+
+    ft = None
+    if "ft:" in line:
+        ft_i = line.index("ft:")
+        ft = int(line[ft_i + 1])
+
+    return Msg(msg_ty, n_inputs, cov, ft)
+
+
+class Supervisor:
+    """A wrapper to run the fuzzer until stopping criteria are satisfied
+    or an error is encountered"""
+
+    def __init__(
+        self, fuzzer_name: str, libfuzzer_log: Path, corpus_dir: Path, wait_2x_cov: bool
+    ):
+        self.fuzzer_name = fuzzer_name
+        self.libfuzzer_log = libfuzzer_log
+        self.corpus_dir = corpus_dir
+        self.wait_2x_cov = wait_2x_cov
+        self.start_time = time.monotonic()
+        self.init_cov = None
+        self.latest_cov = None
+        self.init_ft = None
+        self.latest_ft = None
+        self.latest_n_inputs = 0
+        self.latest_cov_inc = self.start_time
+
+    def criteria_satisfied(self) -> bool:
+        """Indicates whether stopping criteria of the corresponding fuzzer
+        are satisfied. E.g. it was running long enough and covered enough paths."""
+
+        # Coverage increased at least twice in comparison with corpus
+        cov = False
+        if self.init_cov:
+            cov = self.latest_cov >= self.init_cov * 2
+
+        # Number of unique paths (features) increased at least twice
+        # in comparison with corpus
+        ft = False
+        if self.init_ft:
+            ft = self.latest_ft >= self.init_ft * 2
+
+        # At least 100_000 fuzzer test runs
+        n_inputs = self.latest_n_inputs > 100_000
+
+        # Last coverage increase was detected 2 hours ago
+        cov_inc = (time.monotonic() - self.latest_cov_inc) > 2 * 60 * 60
+
+        return (cov or ft or not self.wait_2x_cov) and n_inputs and cov_inc
+
+    def update_stats(self, msg: Msg):
+        self.latest_n_inputs = msg.n_inputs
+        if msg.cov:
+            if self.latest_cov and msg.cov > self.latest_cov:
+                self.latest_cov_inc = time.monotonic()
+            self.latest_cov = msg.cov
+        if msg.ft:
+            self.latest_ft = msg.ft
+
+        if msg.ty == MsgType.INITED:
+            self.init_cov = msg.cov
+            self.init_ft = msg.ft
+
+    def run(self):
+        """Run the corresponding fuzzer until either stopping criteria are
+        satisfied or fuzzing fails with an error indicating that some problem
+        was detected."""
+
+        print(f"Running {self.fuzzer_name}")
+        proc = subprocess.Popen(
+            [
+                f"./test/fuzz/{self.fuzzer_name}_fuzzer",
+                # Read from both corpus folders, write only to the first
+                self.corpus_dir.resolve(),
+                f"test/static/corpus/{self.fuzzer_name}",
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        
+        with self.libfuzzer_log.open("w") as libfuzzer_log:
+            for line in proc.stderr:
+                msg = parse_line(line)
+                # Add timestamp in seconds relative to the starting time of a fuzzer
+                line = f"{time.monotonic() - self.start_time}s {line}"
+                libfuzzer_log.write(line)
+                # if it is not a conventional log message print and continue
+                if not msg:
+                    print(line.strip("\n"), end="\r\n")
+                    continue
+
+                # On pulse print the current state
+                if msg.ty == MsgType.pulse:
+                    print(vars(self), end="\r\n")
+
+                self.update_stats(msg)
+                if self.criteria_satisfied():
+                    print(f"Fuzzer {self.fuzzer_name} satisfied criteria")
+                    proc.terminate()
+                    try:
+                        proc.wait(timeout=15)
+                    except subprocess.TimeoutExpired:
+                        proc.kill()
+                    return
+
+        outs, errs = proc.communicate(timeout=15)
+        print(outs)
+        print(errs)
+        code = proc.poll()
+        if code != 0 and (not self.criteria_satisfied()):
+            raise Exception(
+                f"Fuzzer {self.fuzzer_name} stopped without satisfying criteria. Exit code: {code}"
+            )
+
+
+# The script takes the fuzzing target name as the first argument.
+# Then it runs the fuzzing target until either stopping criteria are satisfied
+# or fuzzer detects a bug and fails.
+# Fuzzers should be built first. See README.md for instructions.
+if __name__ == "__main__":
+    parser = ArgumentParser(
+        prog="Fuzzer Supervisor",
+        description="Runs a supplied fuzzer target until either stopping criteria\
+ are satisfied or fuzzer detects a bug and fails",
+    )
+    parser.add_argument("fuzzer_target_name")
+    parser.add_argument(
+        "--libfuzzer-log",
+        required=True,
+        type=Path,
+        help="File where libfuzzer log will be written",
+    )
+    parser.add_argument(
+        "--corpus-dir",
+        required=True,
+        type=Path,
+        help="Directory where libuzzer generated corpus will be stored",
+    )
+    parser.add_argument(
+        "--wait-2x-cov",
+        action="store_true",
+        help="Enables 'coverage increased at least twice' criteria",
+    )
+    args = parser.parse_args()
+    Supervisor(
+        args.fuzzer_target_name, args.libfuzzer_log, args.corpus_dir, args.wait_2x_cov
+    ).run()

test/fuzz/httpfast_fuzzer.c

@@ -0,0 +1,79 @@
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include <httpfast.h>
+
+int handle_param(void *uobj, const char *name, size_t name_len, const char *value, size_t value_len)
+{
+    // Simple check to allow branch visiting when calling param callback
+    if (name_len % 2 == 0)
+    {
+        return 0;
+    }
+
+    return 1;
+}
+
+void on_error(void *uobj, int code, const char *fmt, va_list ap)
+{
+}
+
+void on_warn(void *uobj, int code, const char *fmt, va_list ap)
+{
+}
+
+int on_header(void *uobj,
+              const char *name, size_t name_len,
+              const char *value, size_t value_len,
+              int is_continuation)
+{
+    return 0;
+}
+
+int on_body(void *uobj, const char *body, size_t body_len)
+{
+    return 0;
+}
+
+int on_request_line(void *uobj,
+                    const char *method, size_t method_len,
+                    const char *path, size_t path_len,
+                    const char *query, size_t query_len,
+                    int http_major, int http_minor)
+{
+    return 0;
+}
+
+int on_response_line(void *uobj,
+                     unsigned code,
+                     const char *reason, size_t reason_len,
+                     int http_major, int http_minor)
+{
+    return 0;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    const char *str = (const char *)data;
+
+    struct parse_http_events event_callbacks_one = {
+        .on_error = on_error,
+        .on_warn = on_warn,
+        .on_header = on_header,
+        .on_body = on_body,
+        .on_request_line = NULL,
+        .on_response_line = on_response_line};
+    struct parse_http_events event_callbacks_two = {
+        .on_error = on_error,
+        .on_warn = on_warn,
+        .on_header = on_header,
+        .on_body = on_body,
+        .on_request_line = on_request_line,
+        .on_response_line = NULL};
+    httpfast_parse(str, size, &event_callbacks_one, NULL);
+    httpfast_parse(str, size, &event_callbacks_two, NULL);
+    httpfast_parse_params(str, size, handle_param, NULL);
+    return 0;
+}

test/fuzz/tpleval_fuzzer.c

@@ -0,0 +1,15 @@
+#include <stdint.h>
+#include <stddef.h>
+#include <string.h>
+#include <tpleval.h>
+
+void term(int type, const char *str, size_t len, void *data)
+{
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
+{
+    const char *str = (const char *)data;
+    tpe_parse(str, size, term, NULL);
+    return 0;
+}

test/static/corpus/httpfast/043dc8708a5da455879a47a8f3d38d3ead0d800e

@@ -0,0 +1 @@
+name29=value29&name30=

test/static/corpus/httpfast/0a58e9f6076cb8aefb7c9938d31019b04e6ee12a

@@ -0,0 +1 @@
+name18==value18

test/static/corpus/httpfast/16f8c3acf43eac81c31f14bb0a5511eef2e24531

@@ -0,0 +1,14 @@
+HTTP/1.1 200 OK
+Content-Type: text/plain
+Transfer-Encoding: chunked
+
+4\r\n
+Wiki\r\n
+5\r\n
+pedia\r\n
+E\r\n
+in\r\n
+\r\n
+chunks.\r\n
+0\r\n
+\r\n

test/static/corpus/httpfast/1916c57456eef9c7cbee0eba2b5912b31e529081

@@ -0,0 +1,3 @@
+HTTP/1.1 200 OK
+Set-Cookie: SessionToken=abc123; Path=/
+Set-Cookie: Expires=Wed, 09 Jun 2021 10:18:14 GMT

test/static/corpus/httpfast/19902865be8fb87ff9cbeae6eaf4bb7e5831c1c6

@@ -0,0 +1 @@
+name11=value11&