Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/tools/package.ts
Original file line number Diff line number Diff line change
Expand Up @@ -224,13 +224,18 @@ const QUERY_STOPWORDS = new Set([
"those"
]);

/** Solr boolean keywords: all-caps by convention, never a term to score on. */
const SOLR_OPERATORS = new Set(["AND", "OR", "NOT"]);

export const extractQueryTerms = (query: string): string[] => {
const raw = query.normalize("NFC").match(/[\p{L}\p{N}]+/gu) ?? [];
// An all-caps token is an acronym, not an article: the stopword list is there for
// `defibrillatori Comune di Lecce`, and must not swallow the `UN` of `UN population`
// on a catalog in another language.
// on a catalog in another language. Solr's own operators are the exception to the
// exception — `aria OR acqua` is a query, not a mention of an organisation called OR.
const terms = raw
.filter((token) => {
if (SOLR_OPERATORS.has(token)) return false;
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Outdated
const term = token.toLowerCase();
if (term.length <= 1) return false;
if (!QUERY_STOPWORDS.has(term)) return true;
Expand Down
9 changes: 9 additions & 0 deletions tests/unit/package-scoring.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,15 @@ describe('multilingual term extraction', () => {
expect(extractQueryTerms('un comune di lecce')).toEqual(['comune', 'lecce']);
});

it('never scores on a Solr operator, even though it is all-caps', () => {
// The acronym rule let `OR` through: `aria OR acqua` scored on three terms and
// a title carrying one of them got 4 × 1/3 instead of 4 × 1/2.
expect(extractQueryTerms('aria OR acqua')).toEqual(['aria', 'acqua']);
expect(extractQueryTerms('aria AND NOT rifiuti')).toEqual(['aria', 'rifiuti']);
// lowercase `or` was already a stopword; an English `Or` mid-sentence stays one too
expect(extractQueryTerms('water or sewage')).toEqual(['water', 'sewage']);
});

it('matches across Unicode normal forms', () => {
Comment thread
greptile-apps[bot] marked this conversation as resolved.
const nfd = 'mobilita\u0300 urbana'; // decomposed
expect(countMatchingTerms(nfd, ['mobilità'])).toBe(1);
Expand Down