feat(tokenizers): add experimental Korean tokenizer (#1018)

Co-authored-by: thatjuan <juan@cxdmail.com>

Author: dayongkr

packages/tokenizers/README.md

@@ -3,26 +3,28 @@
 This package provides support for additional tokenizers for the Orama Search Engine.
 
 Available tokenizers:
+
 - Chinese (Mandarin, experimental)
 - Japanese (experimental)
-- Korean (work in progress)
+- Korean (experimental)
 
 Usage:
 
 ```js
-import { create } from '@orama/orama'
-import { createTokenizer } from '@orama/tokenizers/mandarin'
+import { create } from "@orama/orama";
+import { createTokenizer } from "@orama/tokenizers/mandarin";
 
 const db = await create({
   schema: {
-    myProperty: 'string',
-    anotherProperty: 'number'
+    myProperty: "string",
+    anotherProperty: "number",
   },
   components: {
-    tokenizer: await createTokenizer()
-  }
-})
+    tokenizer: await createTokenizer(),
+  },
+});
 ```
 
 # License
-[Apache 2.0](/LICENSE.md)
+
+[Apache 2.0](/LICENSE.md)

packages/tokenizers/package.json

@@ -25,6 +25,16 @@
         "default": "./dist/commonjs/mandarin.js"
       }
     },
+    "./korean": {
+      "import": {
+        "types": "./dist/esm/korean.d.ts",
+        "default": "./dist/esm/korean.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/korean.d.ts",
+        "default": "./dist/commonjs/korean.js"
+      }
+    },
     "./package.json": "./package.json"
   },
   "dependencies": {
@@ -42,7 +52,7 @@
   },
   "scripts": {
     "build": "tshy",
-    "test": "tsx ./tests/japanese.test.ts && tsx ./tests/mandarin.test.ts"
+    "test": "tsx ./tests/japanese.test.ts && tsx ./tests/mandarin.test.ts && tsx ./tests/korean.test.ts"
   },
   "keywords": [
     "full-text search",
@@ -54,7 +64,8 @@
     "stemming",
     "tokenizers",
     "mandarin",
-    "chinese"
+    "chinese",
+    "korean"
   ],
   "author": {
     "name": "Michele Riva",
@@ -74,6 +85,7 @@
     "exports": {
       "./japanese": "./src/japanese.ts",
       "./mandarin": "./src/mandarin.ts",
+      "./korean": "./src/korean.ts",
       "./package.json": "./package.json"
     }
   },
@@ -82,4 +94,4 @@
     "tshy": "^3.0.2",
     "tsx": "^4.19.2"
   }
-}
+}

packages/tokenizers/src/index.ts

@@ -1,7 +1,9 @@
-import { createTokenizer as createJapaneseTokenizer } from "./japanese.js";
-import { createTokenizer as createMandarinTokenizer } from "./mandarin.js";
+import { createTokenizer as createJapaneseTokenizer } from './japanese.js'
+import { createTokenizer as createKoreanTokenizer } from './korean.js'
+import { createTokenizer as createMandarinTokenizer } from './mandarin.js'
 
 export default {
-    japanese: createJapaneseTokenizer,
-    mandarin: createMandarinTokenizer,
-}
+  japanese: createJapaneseTokenizer,
+  korean: createKoreanTokenizer,
+  mandarin: createMandarinTokenizer
+}

packages/tokenizers/src/korean.ts

@@ -0,0 +1,81 @@
+import type { DefaultTokenizer, DefaultTokenizerConfig } from '@orama/orama'
+import { normalizeToken } from '@orama/orama/internals'
+
+const tokenizerLanguage = 'korean'
+
+type TLanguage = typeof tokenizerLanguage
+
+type KoreanTokenizerConfig = DefaultTokenizerConfig & {
+  language: TLanguage
+}
+
+const defaultConfig: KoreanTokenizerConfig = {
+  language: tokenizerLanguage
+}
+
+const segmenter = new Intl.Segmenter('ko', { granularity: 'word' })
+
+/* c8 ignore next 10 */
+function trim(text: string[]): string[] {
+  while (text[text.length - 1] === '') {
+    text.pop()
+  }
+  while (text[0] === '') {
+    text.shift()
+  }
+  return text
+}
+
+function tokenize(text: string): string[] {
+  const segments = segmenter.segment(text)
+
+  const tokens: string[] = []
+  for (const segment of segments) {
+    if (segment.isWordLike) {
+      tokens.push(segment.segment)
+    }
+  }
+
+  return tokens
+}
+
+function tokenizeInternal(this: DefaultTokenizer, input: string, language?: TLanguage, prop?: string): string[] {
+  /* c8 ignore next 3 */
+  if (typeof input !== 'string') {
+    return [input]
+  }
+
+  let tokens: string[]
+  if (prop && this?.tokenizeSkipProperties?.has(prop)) {
+    // @ts-ignore
+    tokens = [this?.normalizeToken?.bind(this, prop ?? '')(input)]
+  } else {
+    tokens = tokenize(input)
+  }
+
+  const trimTokens = trim(tokens)
+
+  if (!this.allowDuplicates) {
+    return Array.from(new Set(trimTokens))
+  }
+
+  return trimTokens
+}
+
+export function createTokenizer(config: KoreanTokenizerConfig = defaultConfig): DefaultTokenizer {
+  const tokenizerConfig = {
+    tokenize: tokenizeInternal,
+    language: config.language,
+    stemmerSkipProperties: new Set(config.stemmerSkipProperties ? [config.stemmerSkipProperties].flat() : []),
+    tokenizeSkipProperties: new Set(config.tokenizeSkipProperties ? [config.tokenizeSkipProperties].flat() : []),
+    stopWords: config.stopWords as string[] | undefined,
+    allowDuplicates: Boolean(config.allowDuplicates),
+    normalizeToken,
+    normalizationCache: new Map()
+  }
+
+  // @ts-ignore
+  tokenizerConfig.tokenize = tokenizeInternal.bind(tokenizeInternal)
+
+  return tokenizerConfig
+}

packages/tokenizers/tests/korean.test.ts

@@ -0,0 +1,76 @@
+import t from 'tap'
+import { create, insert, search } from '@orama/orama'
+import { createTokenizer } from '../src/korean.js'
+
+const db = create({
+  schema: {
+    name: 'string'
+  },
+  components: {
+    tokenizer: createTokenizer()
+  }
+})
+
+function getHitsNames(hits) {
+  return hits.map((hit) => hit.document.name)
+}
+
+t.test('Korean tokenizer', async (t) => {
+  await insert(db, { name: '서울' }) // Seoul
+  await insert(db, { name: '부산' }) // Busan
+  await insert(db, { name: '대구' }) // Daegu
+  await insert(db, { name: '인천' }) // Incheon
+  await insert(db, { name: '광주' }) // Gwangju
+  await insert(db, { name: '수원' }) // Suwon
+  await insert(db, { name: '포항' }) // Pohang
+  await insert(db, { name: '서울대학교' }) // Seoul National University
+  await insert(db, { name: '부산대학교' }) // Pusan National University
+  await insert(db, { name: '포항공과대학교' }) // Pohang University of Science and Technology
+  await insert(db, { name: '대전 울산 세종' }) // space-separated multi-word string (Daejeon Ulsan Sejong)
+
+  // Intl.Segmenter('ko', { granularity: 'word' }) segments Korean on whitespace (UAX#29), so
+  // space-less compounds such as '서울대학교' stay a single token and are NOT split into '서울' + '대학교'
+  // (unlike the Japanese/Mandarin tokenizers, where dictionary segmentation splits e.g. '東京大学' into
+  // '東京' + '大学'). This is an experimental limitation of the Korean tokenizer. The '서울'/'부산'/'포항'
+  // searches below still return the compound because Orama matches the radix index by prefix
+  // (default exact: false): '서울' is a prefix of the '서울대학교' token.
+  const resultsSeoul = await search(db, { term: '서울', threshold: 0 })
+
+  t.equal(resultsSeoul.count, 2)
+  t.equal(getHitsNames(resultsSeoul.hits).join(', '), '서울, 서울대학교')
+
+  const resultsBusan = await search(db, { term: '부산', threshold: 0 })
+
+  t.equal(resultsBusan.count, 2)
+  t.equal(getHitsNames(resultsBusan.hits).join(', '), '부산, 부산대학교')
+
+  const resultsGwangju = await search(db, { term: '광주', threshold: 0 })
+
+  t.equal(resultsGwangju.count, 1)
+  t.equal(getHitsNames(resultsGwangju.hits).join(', '), '광주')
+
+  const resultsIncheon = await search(db, { term: '인천', threshold: 0 })
+
+  t.equal(resultsIncheon.count, 1)
+  t.equal(getHitsNames(resultsIncheon.hits).join(', '), '인천')
+
+  const resultsPohang = await search(db, { term: '포항', threshold: 0 })
+
+  t.equal(resultsPohang.count, 2)
+  t.equal(getHitsNames(resultsPohang.hits).join(', '), '포항, 포항공과대학교')
+
+  // Searching the shared suffix '대학교' (university) returns nothing: the compounds are indexed as
+  // whole tokens ('서울대학교', '부산대학교', '포항공과대학교'), so the suffix is not a prefix of any token.
+  // Contrast japanese.test.ts, where '大学' matches all three universities because the Japanese
+  // tokenizer dictionary-splits the compounds. This assertion locks in the no-segmentation behavior.
+  const resultsUniversity = await search(db, { term: '대학교', threshold: 0 })
+
+  t.equal(resultsUniversity.count, 0)
+
+  // Whitespace word segmentation works: the space-separated '대전 울산 세종' is tokenized into
+  // '대전' / '울산' / '세종', so searching the middle word finds that document.
+  const resultsUlsan = await search(db, { term: '울산', threshold: 0 })
+
+  t.equal(resultsUlsan.count, 1)
+  t.equal(getHitsNames(resultsUlsan.hits).join(', '), '대전 울산 세종')
+})