|
| 1 | +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | +import argparse |
| 15 | +import json |
| 16 | +import re |
| 17 | +from pathlib import Path |
| 18 | +from typing import Callable |
| 19 | + |
| 20 | +import tiktoken |
| 21 | +from datasets import load_dataset |
| 22 | +from tqdm import tqdm |
| 23 | + |
| 24 | +""" |
| 25 | +Usage |
| 26 | +# default setup is "all" (all problem types, no context window filter) |
| 27 | +python prepare.py |
| 28 | +
|
| 29 | +# prepare only "parents" problem type |
| 30 | +python prepare.py --problem_types parents --setup parents |
| 31 | +
|
| 32 | +# prepare with a 128k context window limit |
| 33 | +python prepare.py --max_context_window 131072 --setup 128k |
| 34 | +
|
| 35 | +# prepare BFS problems within 128k context |
| 36 | +python prepare.py --max_context_window 131072 --problem_types bfs --setup bfs_128k |
| 37 | +
|
| 38 | +# use a HuggingFace tokenizer instead of tiktoken |
| 39 | +python prepare.py --tokenizer meta-llama/Llama-3.1-8B-Instruct --max_context_window 131072 --setup 128k |
| 40 | +""" |
| 41 | + |
| 42 | + |
| 43 | +def build_tokenizer(tokenizer_name: str) -> Callable[[str], int]: |
| 44 | + """Return a callable that counts tokens in a string. |
| 45 | +
|
| 46 | + Tries tiktoken first; if the name is not a valid tiktoken encoding, |
| 47 | + falls back to a HuggingFace AutoTokenizer. |
| 48 | + """ |
| 49 | + try: |
| 50 | + enc = tiktoken.get_encoding(tokenizer_name) |
| 51 | + return lambda text: len(enc.encode(text)) |
| 52 | + except ValueError: |
| 53 | + from transformers import AutoTokenizer |
| 54 | + |
| 55 | + hf_tokenizer = AutoTokenizer.from_pretrained(tokenizer_name) |
| 56 | + return lambda text: len(hf_tokenizer.encode(text, add_special_tokens=False)) |
| 57 | + |
| 58 | + |
| 59 | +def write_data_to_file(output_file, data, max_context_window, problem_types, count_tokens: Callable[[str], int]): |
| 60 | + with open(output_file, "wt", encoding="utf-8") as fout: |
| 61 | + for idx, entry in tqdm(enumerate(data), desc=f"Writing {output_file.name}"): |
| 62 | + if problem_types is not None and entry["problem_type"] not in problem_types: |
| 63 | + print(f"Skipping {idx} because problem_type={entry['problem_type']} not in {problem_types}") |
| 64 | + continue |
| 65 | + |
| 66 | + prompt_text = entry["prompt"] |
| 67 | + answer_nodes = entry["answer_nodes"] |
| 68 | + |
| 69 | + pattern = r"Perform a BFS from node (\S+) with depth (\d+)" |
| 70 | + replacement = r"Perform a BFS from node \1 and return only the nodes at exactly depth \2 (not nodes at intermediate depths)" |
| 71 | + prompt_text = re.sub(pattern, replacement, prompt_text) |
| 72 | + |
| 73 | + m = re.search(r"Find the parents of node ([^\s.]+)\.", prompt_text) |
| 74 | + node_id = m.group(1) if m else None |
| 75 | + if node_id is not None and node_id in answer_nodes: |
| 76 | + answer_nodes.remove(node_id) |
| 77 | + print(f"Removing {idx} sample with node {node_id} from answer_nodes because it is in the prompt") |
| 78 | + |
| 79 | + n_tokens = count_tokens(prompt_text) |
| 80 | + |
| 81 | + if max_context_window is not None and n_tokens > max_context_window: |
| 82 | + print(f"Skipping {idx} because it has {n_tokens} tokens (limit={max_context_window})") |
| 83 | + continue |
| 84 | + |
| 85 | + messages = [{"role": "user", "content": prompt_text}] |
| 86 | + |
| 87 | + output_entry = { |
| 88 | + "messages": messages, |
| 89 | + "expected_answer": json.dumps(sorted(answer_nodes)), |
| 90 | + "n_tokens": n_tokens, |
| 91 | + "prompt_chars": entry["prompt_chars"], |
| 92 | + "problem_type": entry["problem_type"], |
| 93 | + } |
| 94 | + json.dump(output_entry, fout) |
| 95 | + fout.write("\n") |
| 96 | + |
| 97 | + |
| 98 | +def get_graphwalks_data(problem_types, setup, max_context_window, tokenizer_name): |
| 99 | + dataset = load_dataset("openai/graphwalks")["train"] |
| 100 | + data_dir = Path(__file__).absolute().parent |
| 101 | + |
| 102 | + count_tokens = build_tokenizer(tokenizer_name) |
| 103 | + output_file = data_dir / f"{setup}.jsonl" |
| 104 | + write_data_to_file(output_file, dataset, max_context_window, problem_types, count_tokens) |
| 105 | + |
| 106 | + |
| 107 | +if __name__ == "__main__": |
| 108 | + parser = argparse.ArgumentParser(description="Prepare GraphWalks dataset.") |
| 109 | + parser.add_argument( |
| 110 | + "--max_context_window", |
| 111 | + type=int, |
| 112 | + default=None, |
| 113 | + help="Maximum context window size in tokens. Samples exceeding this will be skipped.", |
| 114 | + ) |
| 115 | + parser.add_argument( |
| 116 | + "--problem_types", |
| 117 | + nargs="+", |
| 118 | + type=str, |
| 119 | + default=None, |
| 120 | + help="Problem types to include (e.g. parents bfs). Defaults to all types.", |
| 121 | + ) |
| 122 | + parser.add_argument( |
| 123 | + "--setup", |
| 124 | + type=str, |
| 125 | + default="all", |
| 126 | + help="Setup name used as the output filename, e.g. 'all', 'parents', 'bfs_128k'.", |
| 127 | + ) |
| 128 | + parser.add_argument( |
| 129 | + "--tokenizer_name", |
| 130 | + type=str, |
| 131 | + default="cl100k_base", |
| 132 | + help=( |
| 133 | + "Tokenizer to use for counting tokens. Pass a tiktoken encoding name " |
| 134 | + "(e.g. 'cl100k_base', 'o200k_base') or a HuggingFace model id / local path " |
| 135 | + "(e.g. 'meta-llama/Llama-3.1-8B-Instruct')." |
| 136 | + ), |
| 137 | + ) |
| 138 | + |
| 139 | + args = parser.parse_args() |
| 140 | + |
| 141 | + print(f"Preparing GraphWalks dataset with arguments: {args}") |
| 142 | + get_graphwalks_data(args.problem_types, args.setup, args.max_context_window, args.tokenizer_name) |
| 143 | + print(f"GraphWalks dataset preparation with setup '{args.setup}' completed. Use --split={args.setup} to evaluate!") |
0 commit comments