与初赛测试中CGener测试类似，我们使用
python CGener2.py --num-functions 100 --workload-per-func 100 --branch-depth 6 --num-aliases 10 --seed 520 -o 1.c生成测试用例，使用以下脚本
import subprocess
import re
import os
import csv
import concurrent.futures
from typing import List, Tuple, Optional, Any
# --- 配置 ---
# 可执行文件的前缀和路径
EXECUTABLE_PREFIX = "../../miayc_"
# 测试策略列表
STRATEGIES = ["BFS", "DFS", "MPC"]
# 要测试的step范围和步长
START_STEP = 1000
END_STEP = 10000
STEP_INCREMENT = 1000
# 每个测试配置的运行次数
RUNS_PER_TEST = 4
# 输入的LLVM IR文件
INPUT_FILE = "1.ll"
# 输出结果的CSV文件名
OUTPUT_CSV_FILE = "coverage_and_time_stats.csv"
# 并行执行的最大工作线程数。设置为 None 可使用默认值（通常为 os.cpu_count() * 5）。
# 设置为 os.cpu_count() 也是一个不错的选择。
MAX_WORKERS = 4
def parse_output(output: str) -> Tuple[Optional[int], Optional[float]]:
"""从命令的标准输出和标准错误中解析所需数据"""
covered_blocks = None
# 匹配 "Covered Basic block : 4847"
bb_match = re.search(r"Covered Basic block\s*:\s*(\d+)", output)
if bb_match:
covered_blocks = int(bb_match.group(1))
cpu_time_match = re.search(r"User time \(seconds\): ([\d\.]+)", output)
cpu_time = cpu_time_match.group(1) if cpu_time_match else "0.0"
return covered_blocks, float(cpu_time)
def run_single_test(
task: Tuple[str, int],
) -> Tuple[str, int, Optional[float], Optional[float], Optional[str]]:
"""
为单个 (strategy, step) 组合运行测试 RUNS_PER_TEST 次并计算平均值。
此函数将在单独的线程中执行。
返回 (strategy, step, avg_blocks, avg_time_sec, error_message)
"""
strategy, step = task
executable = f"{EXECUTABLE_PREFIX}{strategy}"
command = f"/usr/bin/time -v {executable} --step {step} {INPUT_FILE}"
total_blocks = 0
total_time_sec = 0.0
successful_runs = 0
errors = []
print(f"[STARTING] {strategy} with step {step} (running {RUNS_PER_TEST} times)")
for i in range(RUNS_PER_TEST):
try:
process = subprocess.run(
command, shell=True, capture_output=True, text=True, check=False
)
full_output = process.stdout + process.stderr
blocks, time_sec = parse_output(full_output)
if blocks is None or time_sec is None:
errors.append(
f"Run {i+1} failed to parse output.\nSTDOUT:\n{process.stdout}\nSTDERR:\n{process.stderr}"
)
continue
total_blocks += blocks
total_time_sec += time_sec
successful_runs += 1
except Exception as e:
errors.append(f"Run {i+1} caused an exception: {e}")
if successful_runs == 0:
error_msg = f"All {RUNS_PER_TEST} runs failed. Last error: {errors[-1] if errors else 'N/A'}"
return strategy, step, None, None, error_msg
avg_blocks = total_blocks / successful_runs
avg_time_sec = total_time_sec / successful_runs
# 如果有部分运行失败，也附加一个警告
warning = ""
if successful_runs < RUNS_PER_TEST:
warning = f" ({successful_runs}/{RUNS_PER_TEST} runs succeeded)"
return strategy, step, avg_blocks, avg_time_sec, warning if warning else None
def run_all_tests() -> Tuple[List[str], List[List[Any]]]:
header = [
"Strategy",
"Step",
"Avg Covered Basic Blocks",
"Avg Elapsed Time (s)",
]
results = []
print("Starting performance and coverage test...")
print(f"Each test configuration will be run {RUNS_PER_TEST} times.")
print(f"Using up to {MAX_WORKERS or 'default'} worker threads.")
print("-" * 60)
# 创建所有要运行的任务
tasks = []
steps_to_run = range(START_STEP, END_STEP + 1, STEP_INCREMENT)
for strategy in STRATEGIES:
executable = f"{EXECUTABLE_PREFIX}{strategy}"
if not os.path.exists(executable):
print(
f"Warning: Executable not found at '{executable}', skipping strategy '{strategy}'."
)
continue
for step in steps_to_run:
tasks.append((strategy, step))
# 使用线程池执行任务
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
# 使用 future.result() 来处理每个完成的任务
future_to_task = {
executor.submit(run_single_test, task): task for task in tasks
}
for future in concurrent.futures.as_completed(future_to_task):
strategy, step, avg_blocks, avg_time, error_or_warning = future.result()
if avg_blocks is None: # 表示发生了严重错误
print(
f"[FAILED] {strategy} with step {step}. Reason: {error_or_warning}"
)
else:
status = "[FINISHED]"
if error_or_warning: # 附加警告信息
status = f"[WARNING] "
print(
f"{status} {strategy} with step {step} -> Avg Blocks: {avg_blocks:.2f}, Avg Time: {avg_time:.2f}s {error_or_warning or ''}"
)
results.append([strategy, step, avg_blocks, avg_time])
# 按策略和步数排序结果，以便报告更清晰
results.sort(key=lambda x: (x[0], x[1]))
return header, results
def save_results_to_csv(header: List[str], data: List[List[Any]], filename: str):
try:
with open(filename, "w", newline="") as f:
writer = csv.writer(f)
writer.writerow(header)
# 格式化浮点数以获得更清晰的CSV输出
formatted_data = [
[
row[0],
row[1],
f"{row[2]:.2f}",
f"{row[3]:.2f}",
]
for row in data
]
writer.writerows(formatted_data)
print(f"\nResults successfully saved to {filename}")
except IOError as e:
print(f"\nError saving results to {filename}: {e}")
if __name__ == "__main__":
if not os.path.exists(INPUT_FILE):
print(f"Error: Input file '{INPUT_FILE}' not found in the current directory.")
else:
test_header, test_results = run_all_tests()
if test_results:
print("\n--- Summary ---")
print(
f"{test_header[0]:<10} | {test_header[1]:<10} | {test_header[2]:<25} | {test_header[3]:<22}"
)
print("-" * 80)
for row in test_results:
print(
f"{row[0]:<10} | {row[1]:<10} | {row[2]:<25.2f} | {row[3]:<22.2f}"
)
save_results_to_csv(test_header, test_results, OUTPUT_CSV_FILE)
else:
print("\nNo results were collected. Please check for errors above.")得到结果如下：
Strategy,Step,Avg Covered Basic Blocks,Avg Elapsed Time (s)
BFS,1000,698.00,18.75
BFS,2000,1306.00,36.39
BFS,3000,1879.00,52.62
BFS,4000,2433.00,68.11
BFS,5000,2958.00,81.78
BFS,6000,3464.00,95.98
BFS,7000,3931.00,111.20
BFS,8000,4363.00,123.89
BFS,9000,4786.00,136.41
BFS,10000,5126.00,145.97
DFS,1000,253.00,8.21
DFS,2000,427.00,12.85
DFS,3000,578.00,17.85
DFS,4000,733.00,23.50
DFS,5000,936.00,28.40
DFS,6000,941.00,29.79
DFS,7000,1106.00,34.45
DFS,8000,1272.00,39.77
DFS,9000,1455.00,44.70
DFS,10000,1611.00,49.80
MPC,1000,999.00,26.89
MPC,2000,1999.00,49.63
MPC,3000,2925.00,72.15
MPC,4000,3887.00,98.66
MPC,5000,4847.00,125.22
MPC,6000,5532.00,141.79
MPC,7000,5535.00,142.55
MPC,8000,5536.00,144.72
MPC,9000,5538.00,143.75
MPC,10000,5540.00,143.30结果画出来后如下：
以及函数优化效果测试，请见测试数据中的function_opt_data文件夹
我们使用wllvm工具编译了以下开源项目，并对其进行了静态分析测试：
| 项目名称 | 版本 | 二进制大小 | 基本块数量 | 可疑指令数量 |
|---|---|---|---|---|
| GNU bc | 1.07.1 | 169KB | 2383 | 31 |
| GNU bison | 3.8.2 | 1585KB | 20653 | 39 |
| GNU grep | 3.11 | 448KB | 6085 | 47 |
| GNU make | 4.4.1 | 610KB | 9196 | 38 |
| FLVMeta | 1.2.2 | 376KB | 5065 | 9 |
| libtiff | 4.6.0 | 1221KB | 15587 | 163 |
| JunkMalloc | - | 50KB | 391 | 22 |
FLVMeta的全部输出报告如下：
No specific errors specified to care about. Defaulting to all errors.
--- Running Analysis ---
IR File: flvmeta.ll
Using config file:
No config file path provided. Use defaults config.
Log path: logs/app.log
Start function: main
Function: amf_data_new
Total Errors : 1
----------------------------------------
Instruction: %4 = call noalias ptr @malloc(i64 noundef 48) #12, !dbg !2947 // src/amf.c:207
{
Execution Count: 22
Issues (3 total):
Memory Leak: 3 times (with 3 unique call stacks)
}
----------------------------------------
Function: amf_list_push
Total Errors : 1
----------------------------------------
Instruction: %7 = call noalias ptr @malloc(i64 noundef 24) #12, !dbg !2950 // src/amf.c:36
{
Execution Count: 6
Issues (3 total):
Memory Leak: 3 times (with 3 unique call stacks)
}
----------------------------------------
Function: amf_string_new
Total Errors : 1
----------------------------------------
Instruction: %27 = call noalias ptr @calloc(i64 noundef %26, i64 noundef 1) #12, !dbg !2971 // src/amf.c:912
{
Execution Count: 5
Issues (4 total):
Memory Leak: 4 times (with 4 unique call stacks)
}
----------------------------------------
Function: parse_command_line
Total Errors : 6
----------------------------------------
Instruction: %30 = load ptr, ptr %29, align 8, !dbg !2971 // src/flvmeta.c:180
{
Execution Count: 1
Issues (1 total):
Heap Overflow: 1 times (with 1 unique call stacks)
}
Instruction: %49 = load ptr, ptr %48, align 8, !dbg !2986 // src/flvmeta.c:187
{
Execution Count: 1
Issues (1 total):
Heap Overflow: 1 times (with 1 unique call stacks)
}
Instruction: %68 = load ptr, ptr %67, align 8, !dbg !3001 // src/flvmeta.c:194
{
Execution Count: 1
Issues (1 total):
Heap Overflow: 1 times (with 1 unique call stacks)
}
Instruction: %87 = load ptr, ptr %86, align 8, !dbg !3016 // src/flvmeta.c:201
{
Execution Count: 1
Issues (1 total):
Heap Overflow: 1 times (with 1 unique call stacks)
}
Instruction: %285 = load ptr, ptr %284, align 8, !dbg !3236 // src/flvmeta.c:322
{
Execution Count: 2
Issues (1 total):
Heap Overflow: 1 times (with 1 unique call stacks)
}
Instruction: %307 = load ptr, ptr %306, align 8, !dbg !3254 // src/flvmeta.c:332
{
Execution Count: 2
Issues (1 total):
Heap Overflow: 1 times (with 1 unique call stacks)
}
----------------------------------------
Total Functions with Memory errors: 4
Analysis completed successfully.更多原始数据请见测试数据中OpenSource中的results文件夹。