Skip to content

Commit 66b6a7b

Browse files
committed
Use into_iter for Sequence as get_pre_tokenizers unavailable
This is due to the recent version bump on `tokenizers` from 0.21.1 to 0.21.4
1 parent 43a8e99 commit 66b6a7b

1 file changed

Lines changed: 2 additions & 2 deletions

File tree

backends/candle/tests/common.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,14 +288,14 @@ pub fn load_tokenizer(model_root: &Path) -> Result<Tokenizer> {
288288
m.set_prepend_scheme(PrependScheme::First);
289289
tokenizer.with_pre_tokenizer(Some(PreTokenizerWrapper::Metaspace(m)));
290290
} else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer {
291-
let pre_tokenizers = s.get_pre_tokenizers();
291+
let pre_tokenizers: Vec<_> = s.clone().into_iter().collect();
292292
// Check if we have a Metaspace pre tokenizer in the sequence
293293
let has_metaspace = pre_tokenizers
294294
.iter()
295295
.any(|t| matches!(t, PreTokenizerWrapper::Metaspace(_)));
296296

297297
if has_metaspace {
298-
let mut new_pre_tokenizers = Vec::with_capacity(s.get_pre_tokenizers().len());
298+
let mut new_pre_tokenizers = Vec::with_capacity(pre_tokenizers.len());
299299

300300
for pre_tokenizer in pre_tokenizers {
301301
if let PreTokenizerWrapper::WhitespaceSplit(_) = pre_tokenizer {

0 commit comments

Comments
 (0)