Merge pull request #420 from CYHSM/fix_qk_norm

le1nux · web-flow · commit bacc0b96d841 · 2025-11-13T14:41:00.000+01:00
Fix wrong check for qk_norm_config
diff --git a/src/modalities/models/gpt2/gpt2_model.py b/src/modalities/models/gpt2/gpt2_model.py
@@ -475,7 +475,7 @@ def __init__(
         # so if the model wants to increase the distance between logits
         # it needs to scale q or k OR adjust the angle between them
         # qk norm forces the model to mostly adjust the angle between q and k which stabilizes training
-        if attention_config.attention_config is not None:
+        if attention_config.qk_norm_config is not None:
             self.q_norm = attention_config.qk_norm_config.norm_type.value(
                 **dict(attention_config.qk_norm_config.config)
             )

Original file line number	Diff line number	Diff line change
`@@ -475,7 +475,7 @@ def __init__(`
`475`	`475`	`# so if the model wants to increase the distance between logits`
`476`	`476`	`# it needs to scale q or k OR adjust the angle between them`
`477`	`477`	`# qk norm forces the model to mostly adjust the angle between q and k which stabilizes training`
`478`		`- if attention_config.attention_config is not None:`
	`478`	`+ if attention_config.qk_norm_config is not None:`
`479`	`479`	`self.q_norm = attention_config.qk_norm_config.norm_type.value(`
`480`	`480`	`**dict(attention_config.qk_norm_config.config)`
`481`	`481`	`)`