Merge pull request #113 from IAPark/dependabot/cargo/cargo-21fde7aa03

Bump the cargo group across 1 directory with 2 updates

Author: gjtorikian

Cargo.lock

@@ -24,9 +24,9 @@ GEM
     rake (13.3.1)
     rake-compiler (1.3.1)
       rake
-    rake-compiler-dock (1.11.0)
-    rb_sys (0.9.124)
-      rake-compiler-dock (= 1.11.0)
+    rake-compiler-dock (1.12.0)
+    rb_sys (0.9.127)
+      rake-compiler-dock (= 1.12.0)
     regexp_parser (2.11.3)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)

Gemfile.lock

@@ -11,5 +11,5 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8.2" }
-rb-sys = { version = "0.9.124", features = ["stable-api-compiled-fallback"] }
-tiktoken-rs = { version = "0.9.0" }
+rb-sys = { version = "0.9.127", features = ["stable-api-compiled-fallback"] }
+tiktoken-rs = { version = "0.11.0" }

ext/tiktoken_ruby/Cargo.toml

@@ -60,7 +60,7 @@ unsafe extern "C" fn encode_special_without_gvl(data: *mut c_void) -> *mut c_voi
 unsafe extern "C" fn decode_without_gvl(data: *mut c_void) -> *mut c_void {
     let data = &mut *(data as *mut DecodeData);
     let core_bpe = &*data.core_bpe;
-    data.result = core_bpe.decode(std::mem::take(&mut data.ids)).map_err(|e| e.to_string());
+    data.result = core_bpe.decode(&data.ids).map_err(|e| e.to_string());
     std::ptr::null_mut()
 }
 

ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -79,6 +79,7 @@ def list_model_names
       o3: "o200k_base",
       "o4-mini": "o200k_base",
       # chat
+      "gpt-5": "o200k_base",
       "gpt-4.1": "o200k_base",
       "chatgpt-4o": "o200k_base",
       "gpt-4o": "o200k_base",
@@ -136,7 +137,9 @@ def list_model_names
       "o3-": "o200k_base",
       "o4-": "o200k_base",
       # chat
+      "gpt-5.": "o200k_base", # e.g., gpt-5.4, gpt-5.4-mini, gpt-5.3-codex, gpt-5.2-pro
       "gpt-5-": "o200k_base",
+      "codex-mini": "o200k_base", # e.g., codex-mini, codex-mini-latest
       "gpt-4.5-": "o200k_base",
       "gpt-4.1-": "o200k_base",
       "chatgpt-4o-": "o200k_base",

lib/tiktoken_ruby.rb

@@ -21,6 +21,21 @@
     expect(Tiktoken.encoding_for_model("o3")).to be_a(Tiktoken::Encoding)
   end
 
+  {
+    "gpt-5" => :o200k_base,
+    "gpt-5-mini" => :o200k_base,
+    "gpt-5.4" => :o200k_base,
+    "gpt-5.4-mini" => :o200k_base,
+    "gpt-5.3-codex" => :o200k_base,
+    "gpt-5.2-pro" => :o200k_base,
+    "codex-mini" => :o200k_base,
+    "codex-mini-latest" => :o200k_base
+  }.each do |model, expected|
+    it "maps #{model} to #{expected}" do
+      expect(Tiktoken.encoding_for_model(model)).to eq(Tiktoken.get_encoding(expected))
+    end
+  end
+
   it "fails gracefully when getting an encoding for an unknown model" do
     expect(Tiktoken.encoding_for_model("bad-model-name")).to be_nil
   end