|
21 | 21 | import requests |
22 | 22 | from omegaconf import OmegaConf |
23 | 23 |
|
| 24 | +ANSI_ESCAPE_RE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]") |
| 25 | + |
| 26 | + |
| 27 | +def _extract_metric_value(line, key): |
| 28 | + """Extract a metric value from a log line, tolerating formatting variations.""" |
| 29 | + cleaned_line = ANSI_ESCAPE_RE.sub("", line) |
| 30 | + pattern = re.compile( |
| 31 | + rf"{re.escape(key.rstrip(':'))}\s*:\s*([+-]?\d+\.?\d*(?:[eE][+-]?\d+)?)", |
| 32 | + re.IGNORECASE, |
| 33 | + ) |
| 34 | + match = pattern.search(cleaned_line) |
| 35 | + if not match: |
| 36 | + return None |
| 37 | + |
| 38 | + try: |
| 39 | + return float(match.group(1)) |
| 40 | + except ValueError: |
| 41 | + return None |
| 42 | + |
24 | 43 |
|
25 | 44 | def find_directory(start_path, target_dir_name): |
26 | 45 | """Recursively find directory by name.""" |
@@ -67,25 +86,10 @@ def extract_metrics_from_log(lines, metric_keys=None): |
67 | 86 | results = {key: {"values": []} for key in metric_keys} |
68 | 87 |
|
69 | 88 | for line in lines: |
70 | | - # Skip non-iteration lines |
71 | | - if "iteration" not in line: |
72 | | - continue |
73 | | - |
74 | | - # Split by | and extract key-value pairs |
75 | | - parts = line.split("|") |
76 | | - for part in parts: |
77 | | - part = part.strip() |
78 | | - for key in metric_keys: |
79 | | - # Match "lm loss: 1.161108E+01" format |
80 | | - if part.startswith(key.rstrip(":")): |
81 | | - # Extract the value after the colon |
82 | | - match = re.search(r":\s*([+-]?\d+\.?\d*(?:[eE][+-]?\d+)?)", part) |
83 | | - if match: |
84 | | - try: |
85 | | - value = float(match.group(1)) |
86 | | - results[key]["values"].append(value) |
87 | | - except ValueError: |
88 | | - continue |
| 89 | + for key in metric_keys: |
| 90 | + value = _extract_metric_value(line, key) |
| 91 | + if value is not None: |
| 92 | + results[key]["values"].append(value) |
89 | 93 |
|
90 | 94 | return results |
91 | 95 |
|
@@ -129,7 +133,8 @@ def find_latest_stdout_log(start_path): |
129 | 133 |
|
130 | 134 | # Sort attempt directories numerically (attempt_0, attempt_1, ...) |
131 | 135 | attempt_dirs.sort( |
132 | | - key=lambda x: int(x.split("_")[1]) if x.split("_")[1].isdigit() else -1, reverse=True |
| 136 | + key=lambda x: int(x.split("_")[1]) if x.split("_")[1].isdigit() else -1, |
| 137 | + reverse=True, |
133 | 138 | ) |
134 | 139 | latest_attempt = os.path.join(latest_folder, attempt_dirs[0]) |
135 | 140 |
|
@@ -172,6 +177,38 @@ def test_train_equal(path, task, model, case): |
172 | 177 | with open(result_path, "r", errors="replace") as file: |
173 | 178 | lines = file.readlines() |
174 | 179 |
|
| 180 | + # A case may explicitly opt into smoke validation. This is useful when a |
| 181 | + # new accelerator is first enabled and no trustworthy, platform-specific |
| 182 | + # golden loss curve has been recorded yet. Smoke mode still requires a |
| 183 | + # completed run log and finite loss values; it never invents golden data. |
| 184 | + config_path = os.path.join(path, task, model, "conf", case + ".yaml") |
| 185 | + smoke_config = {} |
| 186 | + if os.path.exists(config_path): |
| 187 | + case_config = OmegaConf.load(config_path) |
| 188 | + training_smoke = case_config.get("test", {}).get("training_smoke", {}) |
| 189 | + smoke_config = ( |
| 190 | + OmegaConf.to_container(training_smoke, resolve=True) |
| 191 | + if OmegaConf.is_config(training_smoke) |
| 192 | + else training_smoke |
| 193 | + ) |
| 194 | + |
| 195 | + if smoke_config and smoke_config.get("enabled", False): |
| 196 | + metric_key = smoke_config.get("metric", "lm loss:") |
| 197 | + min_values = int(smoke_config.get("min_values", 1)) |
| 198 | + result_values = extract_metrics_from_log(lines, [metric_key])[metric_key]["values"] |
| 199 | + |
| 200 | + print("\nTraining smoke validation") |
| 201 | + print(f"Metric: {metric_key}") |
| 202 | + print(f"Values: {result_values}") |
| 203 | + assert len(result_values) >= min_values, ( |
| 204 | + f"Expected at least {min_values} values for '{metric_key}', " |
| 205 | + f"but extracted {len(result_values)}" |
| 206 | + ) |
| 207 | + assert np.all(np.isfinite(result_values)), ( |
| 208 | + f"Metric '{metric_key}' contains NaN or Inf: {result_values}" |
| 209 | + ) |
| 210 | + return |
| 211 | + |
175 | 212 | # Load gold values first to determine which metrics to extract |
176 | 213 | gold_value_path = os.path.join(path, task, model, "gold_values", case + ".json") |
177 | 214 | assert os.path.exists(gold_value_path), f"Failed to find gold result JSON at {gold_value_path}" |
@@ -333,7 +370,11 @@ def test_inference_equal(path, task, model, case): |
333 | 370 | print("\nResult checking") |
334 | 371 | print("Result: ", result_lines) |
335 | 372 | print("Gold Result: ", gold_value_lines) |
336 | | - print("len(result_lines), (gold_value_lines): ", len(result_lines), len(gold_value_lines)) |
| 373 | + print( |
| 374 | + "len(result_lines), (gold_value_lines): ", |
| 375 | + len(result_lines), |
| 376 | + len(gold_value_lines), |
| 377 | + ) |
337 | 378 |
|
338 | 379 | assert len(result_lines) == len(gold_value_lines) |
339 | 380 |
|
|
0 commit comments