|
| 1 | +# Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +"""Prepare IHEval splits from the zhihz0535/IHEval HF mirror. |
| 16 | +
|
| 17 | +Writes one test.jsonl per sub-benchmark; variants are discovered from the repo |
| 18 | +file listing. Data is downloaded at prepare time and is not committed. |
| 19 | +""" |
| 20 | + |
| 21 | +import argparse |
| 22 | +import json |
| 23 | +from pathlib import Path |
| 24 | + |
| 25 | +HF_REPO = "zhihz0535/IHEval" |
| 26 | +HF_ROOT = "iheval" # top-level dir inside the HF dataset repo |
| 27 | + |
| 28 | +# split_name -> (category, task, task_id, category_id) |
| 29 | +# `category` and `task` are the path segments in the HF layout |
| 30 | +# iheval/<category>/<task>/<setting>/<variant>/; `category_id` and `task_id` are |
| 31 | +# the normalized labels written to each row's "category" / "task" fields. |
| 32 | +SUB_BENCHMARKS = { |
| 33 | + "rule_following_single": ("rule-following", "single-turn", "single", "rule_following"), |
| 34 | + "rule_following_multi": ("rule-following", "multi-turn", "multi", "rule_following"), |
| 35 | + "task_execution_verb_extract": ("task-execution", "verb-extract", "verb_extract", "task_execution"), |
| 36 | + "task_execution_translation": ("task-execution", "translation", "translation", "task_execution"), |
| 37 | + "task_execution_lang_detect": ("task-execution", "lang-detect", "lang_detect", "task_execution"), |
| 38 | + "safety_hijack": ("safety", "user-prompt-hijack", "hijack", "safety"), |
| 39 | + "safety_extract": ("safety", "system-prompt-extract", "extract", "safety"), |
| 40 | + "tool_use_webpage": ("tool-use", "get-webpage", "webpage", "tool_use"), |
| 41 | + "tool_use_slack_user": ("tool-use", "slack-user", "slack_user", "tool_use"), |
| 42 | +} |
| 43 | + |
| 44 | +SETTINGS = ("aligned", "conflict", "reference") |
| 45 | + |
| 46 | + |
| 47 | +def list_variants(repo_files, category, task, setting): |
| 48 | + """Return variant names that have an ``input_data.json`` for this (category, task, setting).""" |
| 49 | + prefix = f"{HF_ROOT}/{category}/{task}/{setting}/" |
| 50 | + variants = set() |
| 51 | + for f in repo_files: |
| 52 | + if f.startswith(prefix) and f.endswith("/input_data.json"): |
| 53 | + variant = f[len(prefix) : -len("/input_data.json")] |
| 54 | + if "/" not in variant: # exactly one level (the variant dir) |
| 55 | + variants.add(variant) |
| 56 | + return sorted(variants) |
| 57 | + |
| 58 | + |
| 59 | +def load_input_data(category, task, setting, variant): |
| 60 | + """Download + parse one ``input_data.json`` from the HF dataset repo.""" |
| 61 | + from huggingface_hub import hf_hub_download |
| 62 | + |
| 63 | + rel = f"{HF_ROOT}/{category}/{task}/{setting}/{variant}/input_data.json" |
| 64 | + local = hf_hub_download(repo_id=HF_REPO, filename=rel, repo_type="dataset") |
| 65 | + with open(local, "rt", encoding="utf-8") as fin: |
| 66 | + return json.load(fin) |
| 67 | + |
| 68 | + |
| 69 | +def build_messages(row, category_id, task_id, setting): |
| 70 | + """Build an OpenAI-style ``messages`` list from one upstream row.""" |
| 71 | + system = row.get("system") |
| 72 | + instruction = row.get("instruction", "") |
| 73 | + |
| 74 | + # Multi-turn rule-following with a real dialog history. |
| 75 | + conv_hist = row.get("conversation_history") |
| 76 | + if category_id == "rule_following" and task_id == "multi" and isinstance(conv_hist, list) and len(conv_hist) >= 2: |
| 77 | + messages = [] |
| 78 | + if system: |
| 79 | + messages.append({"role": "system", "content": system}) |
| 80 | + # conversation_history is [user_turn_1, assistant_turn_1] (possibly more pairs). |
| 81 | + pairs = list(zip(conv_hist[0::2], conv_hist[1::2], strict=True)) |
| 82 | + for user_msg, assistant_msg in pairs: |
| 83 | + messages.append({"role": "user", "content": user_msg}) |
| 84 | + messages.append({"role": "assistant", "content": assistant_msg}) |
| 85 | + messages.append({"role": "user", "content": instruction}) |
| 86 | + return messages |
| 87 | + |
| 88 | + # Tool-use with an upstream-supplied tool call/return. |
| 89 | + tool = row.get("tool") |
| 90 | + if category_id == "tool_use" and isinstance(tool, dict) and tool: |
| 91 | + messages = [] |
| 92 | + if system: |
| 93 | + messages.append({"role": "system", "content": system}) |
| 94 | + messages.append({"role": "user", "content": instruction}) |
| 95 | + |
| 96 | + call = tool.get("call") or {} |
| 97 | + ret = tool.get("return") or {} |
| 98 | + call_id = call.get("id") or ret.get("id") or "call_0" |
| 99 | + call_name = call.get("name") or (tool.get("definition") or {}).get("name") or "" |
| 100 | + call_args = call.get("arguments", {}) |
| 101 | + |
| 102 | + messages.append( |
| 103 | + { |
| 104 | + "role": "assistant", |
| 105 | + "content": None, |
| 106 | + "tool_calls": [ |
| 107 | + { |
| 108 | + "id": call_id, |
| 109 | + "type": "function", |
| 110 | + "function": {"name": call_name, "arguments": json.dumps(call_args, ensure_ascii=False)}, |
| 111 | + } |
| 112 | + ], |
| 113 | + } |
| 114 | + ) |
| 115 | + messages.append( |
| 116 | + {"role": "tool", "tool_call_id": call_id, "name": call_name, "content": ret.get("content", "")} |
| 117 | + ) |
| 118 | + return messages |
| 119 | + |
| 120 | + # Reference setting with no system prompt — single user message. |
| 121 | + if setting == "reference" or system in (None, ""): |
| 122 | + return [{"role": "user", "content": instruction}] |
| 123 | + |
| 124 | + return [ |
| 125 | + {"role": "system", "content": system}, |
| 126 | + {"role": "user", "content": instruction}, |
| 127 | + ] |
| 128 | + |
| 129 | + |
| 130 | +def _last_user_text(messages): |
| 131 | + for msg in reversed(messages): |
| 132 | + if msg.get("role") == "user": |
| 133 | + content = msg.get("content") |
| 134 | + if isinstance(content, str): |
| 135 | + return content |
| 136 | + return "" |
| 137 | + |
| 138 | + |
| 139 | +def process_sub_benchmark(out_root, repo_files, split_name, spec): |
| 140 | + category, task, task_id, category_id = spec |
| 141 | + out_dir = out_root / split_name |
| 142 | + out_dir.mkdir(parents=True, exist_ok=True) |
| 143 | + output_file = out_dir / "test.jsonl" |
| 144 | + |
| 145 | + rows_written = 0 |
| 146 | + with output_file.open("w", encoding="utf-8") as fout: |
| 147 | + for setting in SETTINGS: |
| 148 | + for variant in list_variants(repo_files, category, task, setting): |
| 149 | + data = load_input_data(category, task, setting, variant) |
| 150 | + for i, row in enumerate(data): |
| 151 | + messages = build_messages(row, category_id, task_id, setting) |
| 152 | + answer = row.get("answer") |
| 153 | + upstream_id = row.get("id", i) |
| 154 | + record = { |
| 155 | + "id": f"iheval-{task_id}-{setting}-{variant}-{upstream_id}", |
| 156 | + "messages": messages, |
| 157 | + "question": _last_user_text(messages), |
| 158 | + "setting": setting, |
| 159 | + "variant": variant, |
| 160 | + "category": category_id, |
| 161 | + "task": task_id, |
| 162 | + "answer": answer, |
| 163 | + "expected_answer": answer, |
| 164 | + } |
| 165 | + fout.write(json.dumps(record, ensure_ascii=False) + "\n") |
| 166 | + rows_written += 1 |
| 167 | + |
| 168 | + return output_file, rows_written |
| 169 | + |
| 170 | + |
| 171 | +def main(args): |
| 172 | + from huggingface_hub import HfApi |
| 173 | + |
| 174 | + out_root = Path(__file__).absolute().parent |
| 175 | + repo_files = HfApi().list_repo_files(HF_REPO, repo_type="dataset") |
| 176 | + |
| 177 | + selected = dict(SUB_BENCHMARKS) |
| 178 | + if args.only: |
| 179 | + wanted = set(args.only) |
| 180 | + missing = wanted.difference(SUB_BENCHMARKS) |
| 181 | + if missing: |
| 182 | + raise SystemExit(f"Unknown sub-benchmarks: {sorted(missing)}") |
| 183 | + selected = {name: SUB_BENCHMARKS[name] for name in wanted} |
| 184 | + |
| 185 | + for split_name, spec in selected.items(): |
| 186 | + output_file, n = process_sub_benchmark(out_root, repo_files, split_name, spec) |
| 187 | + print(f"[{split_name}] wrote {n} rows -> {output_file}") |
| 188 | + |
| 189 | + |
| 190 | +if __name__ == "__main__": |
| 191 | + parser = argparse.ArgumentParser(description="Prepare IHEval test splits from the HuggingFace mirror.") |
| 192 | + parser.add_argument("--split", default="test", choices=("test",), help="Local split name.") |
| 193 | + parser.add_argument( |
| 194 | + "--only", nargs="*", default=None, help="Restrict to these sub-benchmark output dirs (e.g. safety_hijack)." |
| 195 | + ) |
| 196 | + args = parser.parse_args() |
| 197 | + main(args) |
0 commit comments