File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -288,14 +288,14 @@ pub fn load_tokenizer(model_root: &Path) -> Result<Tokenizer> {
288288 m. set_prepend_scheme ( PrependScheme :: First ) ;
289289 tokenizer. with_pre_tokenizer ( Some ( PreTokenizerWrapper :: Metaspace ( m) ) ) ;
290290 } else if let PreTokenizerWrapper :: Sequence ( s) = pre_tokenizer {
291- let pre_tokenizers = s. get_pre_tokenizers ( ) ;
291+ let pre_tokenizers: Vec < _ > = s. clone ( ) . into_iter ( ) . collect ( ) ;
292292 // Check if we have a Metaspace pre tokenizer in the sequence
293293 let has_metaspace = pre_tokenizers
294294 . iter ( )
295295 . any ( |t| matches ! ( t, PreTokenizerWrapper :: Metaspace ( _) ) ) ;
296296
297297 if has_metaspace {
298- let mut new_pre_tokenizers = Vec :: with_capacity ( s . get_pre_tokenizers ( ) . len ( ) ) ;
298+ let mut new_pre_tokenizers = Vec :: with_capacity ( pre_tokenizers . len ( ) ) ;
299299
300300 for pre_tokenizer in pre_tokenizers {
301301 if let PreTokenizerWrapper :: WhitespaceSplit ( _) = pre_tokenizer {
You can’t perform that action at this time.
0 commit comments