@@ -377,3 +377,56 @@ def test_multimodal_generate(self):
377377 }
378378 output = causal_lm .generate (inputs )
379379 self .assertIsInstance (output , str )
380+
381+ def test_gqa_fit (self ):
382+ # Test fit() with GQA (num_heads != num_kv_heads).
383+ preprocessor = Gemma3nCausalLMPreprocessor (
384+ tokenizer = self .tokenizer ,
385+ image_converter = None ,
386+ audio_converter = None ,
387+ sequence_length = 20 ,
388+ max_images_per_prompt = 0 ,
389+ num_vision_tokens_per_image = 0 ,
390+ max_audios_per_prompt = 0 ,
391+ num_audio_tokens_per_audio = 0 ,
392+ )
393+ backbone = Gemma3nBackbone (
394+ text_vocab_size = preprocessor .tokenizer .vocabulary_size (),
395+ text_hidden_size = 8 ,
396+ num_hidden_layers = 1 ,
397+ pad_token_id = 0 ,
398+ num_attention_heads = 4 ,
399+ num_key_value_heads = 2 ,
400+ head_dim = 2 ,
401+ intermediate_size = [16 ],
402+ hidden_activation = "gelu_approximate" ,
403+ layer_types = ["full_attention" ],
404+ sliding_window = 4 ,
405+ rope_theta = 10000.0 ,
406+ max_position_embeddings = 20 ,
407+ vocab_size_per_layer_input = 10 ,
408+ hidden_size_per_layer_input = 2 ,
409+ altup_num_inputs = 2 ,
410+ laurel_rank = 1 ,
411+ )
412+ self .run_task_test (
413+ cls = Gemma3nCausalLM ,
414+ init_kwargs = {
415+ "preprocessor" : preprocessor ,
416+ "backbone" : backbone ,
417+ },
418+ train_data = (
419+ {
420+ "prompts" : ["the quick brown fox" , "the quick brown fox" ],
421+ "responses" : [
422+ "the earth is round" ,
423+ "the earth is round" ,
424+ ],
425+ },
426+ ),
427+ expected_output_shape = (
428+ 2 ,
429+ 20 ,
430+ preprocessor .tokenizer .vocabulary_size (),
431+ ),
432+ )
0 commit comments