Add MOSS-TTS-v1.5 (delay) and MOSS-Audio-Tokenizer

docs/source/en/_toctree.yml

@@ -1105,6 +1105,10 @@
         title: Moonshine Streaming
       - local: model_doc/moshi
         title: Moshi
+      - local: model_doc/moss-audio-tokenizer
+        title: MOSS Audio Tokenizer
+      - local: model_doc/moss_tts_delay
+        title: MOSS-TTS Delay
       - local: model_doc/musicgen
         title: MusicGen
       - local: model_doc/musicgen_melody

docs/source/en/model_doc/moss-audio-tokenizer.md

@@ -0,0 +1,133 @@
+<!--Copyright 2026 OpenMOSS and The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+*This model was contributed to Hugging Face Transformers on 2026-07-05.*
+
+# MOSS Audio Tokenizer
+
+[MOSS-Audio-Tokenizer](https://huggingface.co/OpenMOSS-Team/MOSS-Audio-Tokenizer) is the neural audio codec used by
+MOSS-TTS. It encodes waveforms into discrete audio codebook tokens and decodes those tokens back into waveform audio.
+
+## Checkpoint conversion
+
+The original OpenMOSS checkpoint uses its native config and weight names. Convert it to the Transformers format before
+loading it with [`AutoModelForAudioTokenization`].
+
+```bash
+export PYTHONPATH="$PWD/src"
+
+python src/transformers/models/moss_audio_tokenizer/convert_moss_audio_tokenizer_to_hf.py \
+  --input_path_or_repo OpenMOSS-Team/MOSS-Audio-Tokenizer \
+  --output_dir /path/to/moss-audio-tokenizer-hf
+```
+
+The examples below use the converted local path. Replace it with a Hub repo id if you have pushed the converted
+checkpoint.
+
+## Single audio
+
+```python
+import torch
+from datasets import Audio, load_dataset
+from scipy.io.wavfile import write
+from transformers import AutoFeatureExtractor, AutoModelForAudioTokenization
+
+
+model_id = "/path/to/moss-audio-tokenizer-hf"
+feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)
+model = AutoModelForAudioTokenization.from_pretrained(model_id, dtype="auto", device_map="auto")
+
+dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
+dataset = dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate))
+audio = dataset[0]["audio"]["array"]
+inputs = feature_extractor(audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt").to(model.device)
+
+encoded = model.encode(**inputs, return_dict=True)
+code_positions = torch.arange(encoded.audio_codes.shape[-1], device=encoded.audio_codes.device)
+codes_mask = code_positions[None, :] < encoded.audio_codes_lengths[:, None]
+decoded = model.decode(encoded.audio_codes, padding_mask=codes_mask, return_dict=True)
+
+audio_length = int(decoded.audio_lengths[0])
+audio_values = decoded.audio[0, 0, :audio_length].float().cpu().numpy()
+write("moss_audio_tokenizer_reconstruction.wav", feature_extractor.sampling_rate, audio_values)
+```
+
+## Batch audio
+
+```python
+audios = [dataset[i]["audio"]["array"] for i in range(2)]
+inputs = feature_extractor(audios, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt").to(model.device)
+
+encoded = model.encode(**inputs, return_dict=True)
+code_positions = torch.arange(encoded.audio_codes.shape[-1], device=encoded.audio_codes.device)
+codes_mask = code_positions[None, :] < encoded.audio_codes_lengths[:, None]
+decoded = model.decode(encoded.audio_codes, padding_mask=codes_mask, return_dict=True)
+
+first_length = int(decoded.audio_lengths[0])
+second_length = int(decoded.audio_lengths[1])
+first_reconstruction = decoded.audio[0, 0, :first_length]
+second_reconstruction = decoded.audio[1, 0, :second_length]
+```
+
+## Fewer Quantizers
+
+Use fewer residual quantizers to trade reconstruction quality for a lower bitrate. The full codec depth is stored in
+`config.quantizer_config.n_codebooks`; `num_quantizers` selects how many quantizers to use for a specific encode or
+decode call.
+
+```python
+encoded = model.encode(**inputs, num_quantizers=8, return_dict=True)
+code_positions = torch.arange(encoded.audio_codes.shape[-1], device=encoded.audio_codes.device)
+codes_mask = code_positions[None, :] < encoded.audio_codes_lengths[:, None]
+decoded = model.decode(encoded.audio_codes, padding_mask=codes_mask, num_quantizers=8, return_dict=True)
+```
+
+## Streaming Chunks
+
+`chunk_duration` is expressed in seconds. It must be no longer than
+`config.sliding_window_duration`, and `chunk_duration * config.sampling_rate` must be divisible by
+`config.hop_length`.
+
+```python
+single_inputs = feature_extractor(
+    audio,
+    sampling_rate=feature_extractor.sampling_rate,
+    return_tensors="pt",
+).to(model.device)
+
+encoded = model.encode(**single_inputs, chunk_duration=0.08, return_dict=True)
+code_positions = torch.arange(encoded.audio_codes.shape[-1], device=encoded.audio_codes.device)
+codes_mask = code_positions[None, :] < encoded.audio_codes_lengths[:, None]
+decoded = model.decode(encoded.audio_codes, padding_mask=codes_mask, chunk_duration=0.08, return_dict=True)
+```
+
+## MossAudioTokenizerConfig
+
+[[autodoc]] MossAudioTokenizerConfig
+
+## MossAudioTokenizerQuantizerConfig
+
+[[autodoc]] MossAudioTokenizerQuantizerConfig
+
+## MossAudioTokenizerFeatureExtractor
+
+[[autodoc]] MossAudioTokenizerFeatureExtractor
+
+## MossAudioTokenizerModel
+
+[[autodoc]] MossAudioTokenizerModel
+    - encode
+    - decode
+    - forward

docs/source/en/model_doc/moss_tts_delay.md

@@ -0,0 +1,185 @@
+<!--Copyright 2026 OpenMOSS and The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+
+⚠️ Note that this file is in Markdown but contain specific syntax for our doc-builder (similar to MDX) that may not be
+rendered properly in your Markdown viewer.
+
+-->
+*This model was contributed to Hugging Face Transformers on 2026-07-05.*
+
+# MOSS-TTS Delay
+
+[MOSS-TTS-v1.5](https://huggingface.co/OpenMOSS-Team/MOSS-TTS-v1.5) is a multilingual text-to-speech model from
+OpenMOSS. The model uses a Qwen3 language backbone and predicts delayed audio codebook tokens for speech generation.
+
+## Prepare the TTS Checkpoint
+
+`OpenMOSS-Team/MOSS-TTS-v1.5` can be used directly when its `config.json` is already in the Transformers format. If you
+are starting from an original OpenMOSS config, convert it before loading the model with Transformers. The converter
+writes the converted `config.json` only, so run it into the same local checkpoint directory as the model weights and
+tokenizer files.
+
+```bash
+export PYTHONPATH="$PWD/src"
+
+python src/transformers/models/moss_tts_delay/convert_moss_tts_delay_to_hf.py \
+  --input_path_or_repo /path/to/moss-tts-delay-checkpoint/config.json \
+  --output_dir /path/to/moss-tts-delay-checkpoint
+```
+
+## Prepare the Audio Tokenizer
+
+The processor needs a Transformers-format MOSS Audio Tokenizer checkpoint for reference-audio encoding and waveform
+decoding. If the model repo's processor config does not already point to one, convert the original codec checkpoint
+before running MOSS-TTS Delay.
+
+```bash
+export PYTHONPATH="$PWD/src"
+
+python src/transformers/models/moss_audio_tokenizer/convert_moss_audio_tokenizer_to_hf.py \
+  --input_path_or_repo OpenMOSS-Team/MOSS-Audio-Tokenizer \
+  --output_dir /path/to/moss-audio-tokenizer-hf
+```
+
+Pass the converted path or Hub repo id through `codec_path`.
+
+## Text-to-Speech
+
+```python
+from scipy.io.wavfile import write
+
+from transformers import AutoProcessor, AutoModelForTextToWaveform
+
+
+model_id = "OpenMOSS-Team/MOSS-TTS-v1.5"  # or "/path/to/moss-tts-delay-checkpoint"
+codec_path = "/path/to/moss-audio-tokenizer-hf"
+
+processor = AutoProcessor.from_pretrained(model_id, codec_path=codec_path)
+model = AutoModelForTextToWaveform.from_pretrained(model_id, dtype="auto", device_map="auto")
+
+message = processor.build_user_message(text="Hello from MOSS-TTS.", language="English")
+inputs = processor([message], mode="generation").to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=1024)
+
+messages = processor.decode(outputs)
+audio_values = messages[0].audio_codes_list[0]
+write("moss_tts.wav", processor.model_config.sampling_rate, audio_values.cpu().numpy())
+```
+
+## Language and Pause Control
+
+Set `language` when the input language is known. Inline pause markers such as `[pause 1.5s]` can be used to request
+an explicit pause in the generated speech.
+
+```python
+message = processor.build_user_message(
+    text="Bonjour, je voudrais essayer une voix francaise naturelle. [pause 1.5s] Merci.",
+    language="French",
+)
+inputs = processor([message], mode="generation").to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=1024)
+messages = processor.decode(outputs)
+```
+
+## Duration Control
+
+Use `tokens` to request a shorter or longer generated audio segment.
+
+```python
+short_message = processor.build_user_message(
+    text="Transformers makes text-to-speech easy.",
+    language="English",
+    tokens=200,
+)
+long_message = processor.build_user_message(
+    text="Transformers makes text-to-speech easy.",
+    language="English",
+    tokens=450,
+)
+
+inputs = processor([short_message, long_message], mode="generation").to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=1024)
+messages = processor.decode(outputs)
+```
+
+## Voice Cloning
+
+Pass one or more reference audio paths to `reference`. The processor loads the files, encodes them with the MOSS audio
+tokenizer, and inserts the resulting audio codes into the prompt.
+
+```python
+message = processor.build_user_message(
+    text="Please read this sentence with the reference speaker voice.",
+    reference=["reference_speaker.wav"],
+    language="English",
+)
+inputs = processor([message], mode="generation").to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=1024)
+messages = processor.decode(outputs)
+```
+
+## Batch generation
+
+```python
+messages = [
+    processor.build_user_message(text="Hello from MOSS-TTS.", language="English"),
+    processor.build_user_message(text="Transformers supports batched speech generation.", language="English"),
+]
+inputs = processor(messages, mode="generation").to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=1024)
+messages = processor.decode(outputs)
+```
+
+## Audio Codebooks
+
+MOSS-TTS Delay uses `config.n_codebooks` audio codebook channels. Reference audio codes passed to the processor should
+have shape `(sequence_length, config.n_codebooks)`, and any `n_codebooks` processor override should match the model
+configuration used for generation.
+
+```python
+message = processor.build_user_message(
+    text="Please match the reference voice.",
+    reference=["reference_speaker.wav"],
+    language="English",
+)
+inputs = processor([message], mode="generation", n_codebooks=model.config.n_codebooks).to(model.device)
+outputs = model.generate(**inputs, max_new_tokens=1024)
+messages = processor.decode(outputs)
+```
+
+## MossTTSDelayConfig
+
+[[autodoc]] MossTTSDelayConfig
+
+## MossTTSDelayProcessor
+
+[[autodoc]] MossTTSDelayProcessor
+
+## Message
+
+[[autodoc]] Message
+
+## UserMessage
+
+[[autodoc]] UserMessage
+
+## AssistantMessage
+
+[[autodoc]] AssistantMessage
+
+## MossTTSDelayModel
+
+[[autodoc]] MossTTSDelayModel
+    - forward
+
+## MossTTSDelayOutputWithPast
+
+[[autodoc]] MossTTSDelayOutputWithPast

src/transformers/models/__init__.py

@@ -286,6 +286,8 @@
     from .moonshine import *
     from .moonshine_streaming import *
     from .moshi import *
+    from .moss_audio_tokenizer import *
+    from .moss_tts_delay import *
     from .mpnet import *
     from .mpt import *
     from .mra import *

src/transformers/models/auto/auto_mappings.py

@@ -382,6 +382,9 @@
         ("moonshine_streaming_encoder", "MoonshineStreamingEncoderConfig"),
         ("moshi", "MoshiConfig"),
         ("moshi_depth", "MoshiDepthConfig"),
+        ("moss-audio-tokenizer", "MossAudioTokenizerConfig"),
+        ("moss-audio-tokenizer-quantizer", "MossAudioTokenizerQuantizerConfig"),
+        ("moss_tts_delay", "MossTTSDelayConfig"),
         ("mpnet", "MPNetConfig"),
         ("mpt", "MptConfig"),
         ("mra", "MraConfig"),
@@ -793,6 +796,8 @@
         ("modernbert-decoder", "modernbert_decoder"),
         ("moonshine_streaming_encoder", "moonshine_streaming"),
         ("moshi_depth", "moshi"),
+        ("moss-audio-tokenizer", "moss_audio_tokenizer"),
+        ("moss-audio-tokenizer-quantizer", "moss_audio_tokenizer"),
         ("musicgen_decoder", "musicgen"),
         ("musicgen_melody_decoder", "musicgen_melody"),
         ("nllb-moe", "nllb_moe"),
@@ -935,6 +940,7 @@
         ("gemma4_unified", "Gemma4UnifiedAudioFeatureExtractor"),
         ("granite_speech", "GraniteSpeechFeatureExtractor"),
         ("kyutai_speech_to_text", "KyutaiSpeechToTextFeatureExtractor"),
+        ("moss-audio-tokenizer", "MossAudioTokenizerFeatureExtractor"),
         ("musicgen_melody", "MusicgenMelodyFeatureExtractor"),
         ("pe_audio", "PeAudioFeatureExtractor"),
         ("phi4_multimodal", "Phi4MultimodalFeatureExtractor"),
@@ -1022,6 +1028,7 @@
         ("minicpmv4_6", "MiniCPMV4_6Processor"),
         ("mllama", "MllamaProcessor"),
         ("moonshine_streaming", "MoonshineStreamingProcessor"),
+        ("moss_tts_delay", "MossTTSDelayProcessor"),
         ("musicflamingo", "MusicFlamingoProcessor"),
         ("musicgen", "MusicgenProcessor"),
         ("musicgen_melody", "MusicgenMelodyProcessor"),

src/transformers/models/auto/modeling_auto.py

@@ -322,6 +322,8 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("moonshine", "MoonshineModel"),
         ("moonshine_streaming", "MoonshineStreamingModel"),
         ("moshi", "MoshiModel"),
+        ("moss-audio-tokenizer", "MossAudioTokenizerModel"),
+        ("moss_tts_delay", "MossTTSDelayModel"),
         ("mpnet", "MPNetModel"),
         ("mpt", "MptModel"),
         ("mra", "MraModel"),
@@ -1746,6 +1748,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
         ("csm", "CsmForConditionalGeneration"),
         ("fastspeech2_conformer_with_hifigan", "FastSpeech2ConformerWithHifiGan"),
         ("higgs_audio_v2", "HiggsAudioV2ForConditionalGeneration"),
+        ("moss_tts_delay", "MossTTSDelayModel"),
         ("musicgen", "MusicgenForConditionalGeneration"),
         ("musicgen_melody", "MusicgenMelodyForConditionalGeneration"),
         ("qwen2_5_omni", "Qwen2_5OmniForConditionalGeneration"),
@@ -1903,6 +1906,7 @@ class _BaseModelWithGenerate(PreTrainedModel, GenerationMixin):
     [
         ("dac", "DacModel"),
         ("higgs_audio_v2_tokenizer", "HiggsAudioV2TokenizerModel"),
+        ("moss-audio-tokenizer", "MossAudioTokenizerModel"),
         ("vibevoice_acoustic_tokenizer", "VibeVoiceAcousticTokenizerModel"),
     ]
 )