[Feature request] Java binding for tokenizers

[kirklandsign]

Add Java API to invoke C++ implementation.

This helps us expose tokenizers only, without using LLM generate.

Ideally, we use Java built-in JNI, to minimize dependencies.

[kirklandsign]

Basically have a class like

class Tokenizer {
public int load(String path, int type);
public long[] encode(String input, int bos, int eos);
public String decode(long prev_token, long token);

public int vocabSize();
public long bosTok();
public long eosTok();
}

jni side

#include <jni.h>
#include "Tokenizer.h" // Your Java class header file

// Load method
JNIEXPORT jint JNICALL Java_Tokenizer_load(JNIEnv* env, jobject obj, jstring path);

// Encode method
JNIEXPORT jlongArray JNICALL Java_Tokenizer_encode(JNIEnv* env, jobject obj, jstring input, jint bos, jint eos);

// Decode method
JNIEXPORT jstring JNICALL Java_Tokenizer_decode(JNIEnv* env, jobject obj, jlong prevToken, jlong token);

// Load method
JNIEXPORT jint JNICALL Java_Tokenizer_load(JNIEnv* env, jobject obj, jstring path) {
    // Call the load method on the Java object
    jclass clazz = env->GetObjectClass(obj);
    jmethodID methodId = env->GetMethodID(clazz, "load", "(Ljava/lang/String;)I");
    return env->CallIntMethod(obj, methodId, path);
}

// Encode method
JNIEXPORT jlongArray JNICALL Java_Tokenizer_encode(JNIEnv* env, jobject obj, jstring input, jint bos, jint eos) {
    // Call the encode method on the Java object
    jclass clazz = env->GetObjectClass(obj);
    jmethodID methodId = env->GetMethodID(clazz, "encode", "(Ljava/lang/String;II)[J");
    return env->CallLongArrayMethod(obj, methodId, input, bos, eos);
}

// Decode method
JNIEXPORT jstring JNICALL Java_Tokenizer_decode(JNIEnv* env, jobject obj, jlong prevToken, jlong token) {
    // Call the decode method on the Java object
    jclass clazz = env->GetObjectClass(obj);
    jmethodID methodId = env->GetMethodID(clazz, "decode", "(JJ)Ljava/lang/String;");
    return env->CallStringMethod(obj, methodId, prevToken, token);
}

JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) {
    JNIEnv* env;
    if (vm->GetEnv((void**)&env, JNI_VERSION_1_6) != JNI_OK) {
        return -1;
    }

    JNINativeMethod methods[] = {
        {"load", "(Ljava/lang/String;)I", (void*)Java_Tokenizer_load},
        {"encode", "(Ljava/lang/String;II)[J", (void*)Java_Tokenizer_encode},
        {"decode", "(JJ)Ljava/lang/String;", (void*)Java_Tokenizer_decode}
    };

    jclass clazz = env->FindClass("Tokenizer");
    if (clazz == NULL) {
        return -1;
    }

    if (env->RegisterNatives(clazz, methods, sizeof(methods) / sizeof(methods[0])) < 0) {
        return -1;
    }

    return JNI_VERSION_1_6;
}