Skip to content

Commit 96e6980

Browse files
crimxcrimx
committed
fix: implement network compatibility for Collins dictionary and enhance error handling
1 parent 6c06238 commit 96e6980

9 files changed

Lines changed: 755 additions & 305 deletions

File tree

‎src/components/dictionaries/cambridge/network.ts‎

Lines changed: 12 additions & 261 deletions
Original file line numberDiff line numberDiff line change
@@ -1,262 +1,13 @@
1-
const CAMBRIDGE_ORIGIN = 'https://dictionary.cambridge.org'
2-
const CAMBRIDGE_PARTITION_KEY_SITE = 'https://cambridge.org'
3-
const CAMBRIDGE_URLS = ['https://dictionary.cambridge.org/*']
4-
const CAMBRIDGE_RULE_ID = 32002
5-
const CAMBRIDGE_CLEARANCE_COOKIE = 'cf_clearance'
6-
const CAMBRIDGE_CLEARANCE_COOKIE_FILTERS = [
7-
{
8-
url: CAMBRIDGE_ORIGIN,
9-
name: CAMBRIDGE_CLEARANCE_COOKIE
10-
},
11-
{
12-
url: CAMBRIDGE_ORIGIN,
13-
name: CAMBRIDGE_CLEARANCE_COOKIE,
14-
partitionKey: {
15-
topLevelSite: CAMBRIDGE_PARTITION_KEY_SITE
16-
}
17-
}
18-
]
19-
20-
type ChromeDeclarativeNetRequest = {
21-
updateSessionRules: (options: {
22-
addRules?: Array<{
23-
id: number
24-
priority: number
25-
action: {
26-
type: 'modifyHeaders'
27-
requestHeaders: Array<ChromeModifyHeaderInfo>
28-
}
29-
condition: {
30-
regexFilter: string
31-
resourceTypes: string[]
32-
}
33-
}>
34-
removeRuleIds?: number[]
35-
}) => Promise<void>
36-
}
37-
38-
type BrowserWebRequest = typeof browser.webRequest
39-
40-
type ChromeModifyHeaderInfo = {
41-
header: string
42-
operation: 'append' | 'set'
43-
value: string
44-
}
45-
46-
let ensureNetworkCompatibilityPromise: Promise<void> | null = null
47-
48-
export function ensureNetworkCompatibility() {
49-
if (!ensureNetworkCompatibilityPromise) {
50-
ensureNetworkCompatibilityPromise = doEnsureCambridgeNetworkCompatibility().catch(
51-
error => {
52-
ensureNetworkCompatibilityPromise = null
53-
throw error
54-
}
55-
)
56-
}
57-
58-
return ensureNetworkCompatibilityPromise
59-
}
60-
61-
async function doEnsureCambridgeNetworkCompatibility() {
62-
const dnr = getChromeDeclarativeNetRequest()
63-
if (dnr) {
64-
await installMv3HeaderRule(dnr, await getCambridgeClearanceCookie())
65-
return
66-
}
67-
68-
const webRequest = getBrowserWebRequest()
69-
if (webRequest && webRequest.onBeforeSendHeaders) {
70-
installMv2HeaderListener(webRequest, await getCambridgeClearanceCookie())
71-
return
72-
}
73-
74-
if (isManifestV3()) {
75-
throw new Error(
76-
'declarativeNetRequest is unavailable in the current MV3 context.'
77-
)
78-
}
79-
80-
throw new Error('webRequest.onBeforeSendHeaders is unavailable.')
81-
}
82-
83-
function isManifestV3() {
84-
const manifest = browser.runtime.getManifest && browser.runtime.getManifest()
85-
return !!(manifest && manifest.manifest_version === 3)
86-
}
87-
88-
function installMv2HeaderListener(
89-
webRequest: BrowserWebRequest,
90-
clearanceCookie: string
91-
) {
92-
const extraInfoSpec = ['blocking', 'requestHeaders']
93-
const onBeforeSendHeadersOptions = (webRequest as any)
94-
.OnBeforeSendHeadersOptions
95-
96-
if (
97-
onBeforeSendHeadersOptions &&
98-
Object.prototype.hasOwnProperty.call(
99-
onBeforeSendHeadersOptions,
100-
'EXTRA_HEADERS'
101-
)
102-
) {
103-
extraInfoSpec.push('extraHeaders')
104-
}
105-
106-
webRequest.onBeforeSendHeaders.addListener(
107-
details => {
108-
if (details && details.requestHeaders) {
109-
setRequestHeader(details.requestHeaders, 'Referer', CAMBRIDGE_ORIGIN)
110-
if (clearanceCookie) {
111-
appendCookieHeader(details.requestHeaders, clearanceCookie)
112-
}
113-
}
114-
return { requestHeaders: details.requestHeaders }
115-
},
116-
{ urls: CAMBRIDGE_URLS },
117-
/** WebExt type is missing Chrome support */
118-
extraInfoSpec as any
119-
)
120-
}
1+
import { createCookieHeaderNetworkCompatibility } from '../network-compat'
1212

122-
async function installMv3HeaderRule(
123-
dnr: ChromeDeclarativeNetRequest,
124-
clearanceCookie: string
125-
) {
126-
const requestHeaders: ChromeModifyHeaderInfo[] = [
127-
{
128-
header: 'referer',
129-
operation: 'set',
130-
value: CAMBRIDGE_ORIGIN
131-
}
132-
]
133-
134-
if (clearanceCookie) {
135-
requestHeaders.push({
136-
header: 'cookie',
137-
operation: 'append',
138-
value: clearanceCookie
139-
})
140-
}
141-
142-
await dnr.updateSessionRules({
143-
removeRuleIds: [CAMBRIDGE_RULE_ID],
144-
addRules: [
145-
{
146-
id: CAMBRIDGE_RULE_ID,
147-
priority: 1,
148-
action: {
149-
type: 'modifyHeaders',
150-
requestHeaders
151-
},
152-
condition: {
153-
regexFilter: '^https://dictionary\\.cambridge\\.org/.*',
154-
resourceTypes: ['xmlhttprequest', 'media']
155-
}
156-
}
157-
]
158-
})
159-
}
160-
161-
async function getCambridgeClearanceCookie() {
162-
const cookiesApi = getBrowserCookies()
163-
if (!cookiesApi || !cookiesApi.get) {
164-
return ''
165-
}
166-
167-
for (const filter of CAMBRIDGE_CLEARANCE_COOKIE_FILTERS) {
168-
const cookie = await cookiesApi.get(filter as any)
169-
if (cookie && cookie.value) {
170-
return `${CAMBRIDGE_CLEARANCE_COOKIE}=${cookie.value}`
171-
}
172-
}
173-
174-
if (cookiesApi.getAll) {
175-
for (const partitionKey of [
176-
undefined,
177-
{
178-
topLevelSite: CAMBRIDGE_PARTITION_KEY_SITE
179-
}
180-
]) {
181-
const cookies = await cookiesApi.getAll({
182-
domain: 'dictionary.cambridge.org',
183-
name: CAMBRIDGE_CLEARANCE_COOKIE,
184-
partitionKey
185-
} as any)
186-
const clearanceCookie = cookies && cookies.find(cookie => cookie.value)
187-
if (clearanceCookie) {
188-
return `${CAMBRIDGE_CLEARANCE_COOKIE}=${clearanceCookie.value}`
189-
}
190-
}
191-
}
192-
193-
return ''
194-
}
195-
196-
function setRequestHeader(
197-
requestHeaders: Array<{ name: string; value?: string }>,
198-
name: string,
199-
value: string
200-
) {
201-
const target = name.toLowerCase()
202-
for (var i = 0; i < requestHeaders.length; ++i) {
203-
if (requestHeaders[i].name.toLowerCase() === target) {
204-
requestHeaders[i].value = value
205-
return
206-
}
207-
}
208-
209-
requestHeaders.push({ name, value })
210-
}
211-
212-
function appendCookieHeader(
213-
requestHeaders: Array<{ name: string; value?: string }>,
214-
cookie: string
215-
) {
216-
const cookieHeader = getRequestHeader(requestHeaders, 'Cookie')
217-
if (cookieHeader) {
218-
if (
219-
!new RegExp(`(?:^|;\\s*)${CAMBRIDGE_CLEARANCE_COOKIE}=`).test(
220-
cookieHeader.value || ''
221-
)
222-
) {
223-
cookieHeader.value = cookieHeader.value
224-
? `${cookieHeader.value}; ${cookie}`
225-
: cookie
226-
}
227-
return
228-
}
229-
230-
requestHeaders.push({ name: 'Cookie', value: cookie })
231-
}
232-
233-
function getRequestHeader(
234-
requestHeaders: Array<{ name: string; value?: string }>,
235-
name: string
236-
) {
237-
const target = name.toLowerCase()
238-
return requestHeaders.find(header => header.name.toLowerCase() === target)
239-
}
240-
241-
function getChromeDeclarativeNetRequest():
242-
| ChromeDeclarativeNetRequest
243-
| undefined {
244-
const chromeApi = (self as any).chrome
245-
return chromeApi && chromeApi.declarativeNetRequest
246-
? chromeApi.declarativeNetRequest
247-
: undefined
248-
}
249-
250-
function getBrowserWebRequest(): BrowserWebRequest | undefined {
251-
const browserApi = browser as
252-
| typeof browser
253-
| { webRequest?: BrowserWebRequest }
254-
return browserApi && browserApi.webRequest ? browserApi.webRequest : undefined
255-
}
256-
257-
function getBrowserCookies(): typeof browser.cookies | undefined {
258-
const browserApi = browser as
259-
| typeof browser
260-
| { cookies?: typeof browser.cookies }
261-
return browserApi && browserApi.cookies ? browserApi.cookies : undefined
262-
}
3+
export const ensureNetworkCompatibility = createCookieHeaderNetworkCompatibility(
4+
{
5+
origin: 'https://dictionary.cambridge.org',
6+
cookieDomain: 'dictionary.cambridge.org',
7+
topLevelSite: 'https://cambridge.org',
8+
urls: ['https://dictionary.cambridge.org/*'],
9+
ruleId: 32002,
10+
ruleRegexFilter: '^https://dictionary\\.cambridge\\.org/.*',
11+
fallbackCookieNames: ['cf_clearance']
12+
}
13+
)

‎src/components/dictionaries/cobuild/engine.ts‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import {
55
getInnerHTML,
66
handleNoResult,
77
handleNetWorkError,
8+
handleManualVerification,
89
SearchFunction,
910
GetSrcPageFunction,
1011
externalLink,
@@ -52,7 +53,8 @@ export const search: SearchFunction<COBUILDResult> = async (
5253
profile,
5354
payload
5455
) => {
55-
text = encodeURIComponent(text.replace(/\s+/g, '-'))
56+
const searchText = text
57+
const encodedText = encodeURIComponent(text.replace(/\s+/g, '-'))
5658
const { options } = profile.dicts.all.cobuild
5759
const sources: string[] = [
5860
'https://www.collinsdictionary.com/dictionary/english/',
@@ -63,19 +65,43 @@ export const search: SearchFunction<COBUILDResult> = async (
6365
sources.reverse()
6466
}
6567

68+
const primaryUrl = sources[0] + encodedText
69+
const secondaryUrl = sources[1] + encodedText
70+
6671
try {
67-
return handleDOM(await fetchDirtyDOM(sources[0] + text), config)
68-
} catch (e) {
72+
return handleDOM(
73+
await fetchDirtyDOM(primaryUrl, { withCredentials: true }),
74+
config
75+
)
76+
} catch (firstError) {
6977
let doc: Document
7078
try {
71-
doc = await fetchDirtyDOM(sources[1] + text)
72-
} catch (e) {
79+
doc = await fetchDirtyDOM(secondaryUrl, {
80+
withCredentials: true
81+
})
82+
} catch (secondError) {
83+
const forbiddenUrl = isForbidden(firstError)
84+
? primaryUrl
85+
: isForbidden(secondError)
86+
? secondaryUrl
87+
: ''
88+
89+
if (forbiddenUrl) {
90+
return handleManualVerification({
91+
text: searchText,
92+
url: forbiddenUrl
93+
})
94+
}
7395
return handleNetWorkError()
7496
}
7597
return handleDOM(doc, config)
7698
}
7799
}
78100

101+
function isForbidden(e: any): boolean {
102+
return e && e.response && e.response.status === 403
103+
}
104+
79105
async function handleDOM(
80106
doc: Document,
81107
config: AppConfig

‎src/components/dictionaries/cobuild/network.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { createCookieHeaderNetworkCompatibility } from '../network-compat'
2+
3+
export const ensureNetworkCompatibility = createCookieHeaderNetworkCompatibility(
4+
{
5+
origin: 'https://www.collinsdictionary.com',
6+
cookieDomain: 'www.collinsdictionary.com',
7+
topLevelSite: 'https://collinsdictionary.com',
8+
urls: ['https://www.collinsdictionary.com/*'],
9+
ruleId: 32003,
10+
ruleRegexFilter: '^https://www\\.collinsdictionary\\.com/.*',
11+
fallbackCookieNames: ['cf_clearance']
12+
}
13+
)

0 commit comments

Comments
 (0)