Skip to content

Commit e6d6594

Browse files
wallentxwallentx
committed
refactor(fetch): Implement context-scoped LLMService interface, named cache structs, and optimized salary filter
1 parent c3b8094 commit e6d6594

9 files changed

Lines changed: 187 additions & 51 deletions

File tree

‎internal/fetcher/job_fetcher.go‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -93,13 +93,14 @@ func fetchAllJobsWithCandidateCache(ctx context.Context, appCfg *AppConfig, crit
9393

9494
// 1. Fetch via LLM job search when enabled.
9595
if appCfg.LLM.Enabled && appCfg.LLM.JobSearch {
96-
if fetchAllJobsInitConfiguredLLM == nil || fetchAllJobsExecuteLLMSearch == nil {
96+
llmSvc := getLLMService(ctx)
97+
if !llmSvc.IsAvailable(ctx, "llm_job_search") {
9798
mu.Lock()
9899
setFetchSearchStatus(&summary, fetchSearchLLM, "disabled: LLM job search runner unavailable")
99100
mu.Unlock()
100101
} else {
101102
reportFetchProgress(progress, "Preparing LLM job search...")
102-
llm, restoreAuth, initErr := fetchAllJobsInitConfiguredLLM(ctx, appCfg, llmTaskJobSearch)
103+
llm, restoreAuth, initErr := llmSvc.InitConfiguredLLM(ctx, appCfg, llmTaskJobSearch)
103104
if initErr == nil {
104105
defer restoreAuth()
105106
promptBytes, err := fetchAllJobsReadFile(runtimeSearchPromptPath)
@@ -110,7 +111,7 @@ func fetchAllJobsWithCandidateCache(ctx context.Context, appCfg *AppConfig, crit
110111
mu.Unlock()
111112
} else {
112113
reportFetchProgress(progress, "Running LLM job search...")
113-
jobs, err := fetchAllJobsExecuteLLMSearch(ctx, llm, string(promptBytes))
114+
jobs, err := llmSvc.ExecuteSearch(ctx, llm, string(promptBytes))
114115
if err != nil {
115116
mu.Lock()
116117
summary.Notices = append(summary.Notices, fmt.Sprintf("LLM job search failed: %v", err))
@@ -155,7 +156,7 @@ func fetchAllJobsWithCandidateCache(ctx context.Context, appCfg *AppConfig, crit
155156
switch {
156157
case !appCfg.LLM.Enabled:
157158
setFetchSearchStatus(&summary, fetchSearchLLMWeb, "disabled: LLM is disabled in config")
158-
case fetchAllJobsExecuteLLMWebSearch == nil && (fetchAllJobsInitConfiguredLLM == nil || fetchAllJobsExecuteLLMSearch == nil):
159+
case !getLLMService(ctx).IsAvailable(ctx, "llm_web_search"):
159160
setFetchSearchStatus(&summary, fetchSearchLLMWeb, "disabled: LLM search runner unavailable")
160161
case len(effectiveSources.LLMWebTargets) == 0:
161162
setFetchSearchStatus(&summary, fetchSearchLLMWeb, "enabled, but no llm_web targets were configured or resolved")
@@ -508,15 +509,8 @@ func fetchAllJobsWithCandidateCache(ctx context.Context, appCfg *AppConfig, crit
508509
}
509510

510511
func executeLLMWebSearchForFetch(ctx context.Context, appCfg *AppConfig, prompt string) ([]Job, error) {
511-
if fetchAllJobsExecuteLLMWebSearch != nil {
512-
return fetchAllJobsExecuteLLMWebSearch(ctx, appCfg, prompt)
513-
}
514-
llm, restoreAuth, err := fetchAllJobsInitConfiguredLLM(ctx, appCfg, llmTaskJobSearch)
515-
if err != nil {
516-
return nil, fmt.Errorf("failed to initialize: %w", err)
517-
}
518-
defer restoreAuth()
519-
return fetchAllJobsExecuteLLMSearch(ctx, llm, prompt)
512+
llmSvc := getLLMService(ctx)
513+
return llmSvc.ExecuteWebSearch(ctx, appCfg, prompt)
520514
}
521515

522516
func markLLMWebJobsForValidation(jobs []Job) {

‎internal/fetcher/job_filter.go‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,14 +106,18 @@ func extractAndCheckSalary(text string, minBase int) bool {
106106
// Remove commas for easier parsing: "$180,000" -> "$180000"
107107
cleanText := strings.ReplaceAll(text, ",", "")
108108

109+
foundAnyNumber := false
110+
109111
// Look for explicitly high numbers (e.g., 180000)
110112
reFull := regexp.MustCompile(`\b[1-9]\d{4,5}\b`)
111113
matchesFull := reFull.FindAllString(cleanText, -1)
112114
for _, m := range matchesFull {
113115
var val int
114-
_, _ = fmt.Sscanf(m, "%d", &val)
115-
if val >= minBase {
116-
return true
116+
if _, err := fmt.Sscanf(m, "%d", &val); err == nil {
117+
foundAnyNumber = true
118+
if val >= minBase {
119+
return true
120+
}
117121
}
118122
}
119123

@@ -122,14 +126,22 @@ func extractAndCheckSalary(text string, minBase int) bool {
122126
matchesK := reK.FindAllStringSubmatch(cleanText, -1)
123127
for _, m := range matchesK {
124128
var val int
125-
_, _ = fmt.Sscanf(m[1], "%d", &val)
126-
if val*1000 >= minBase {
127-
return true
129+
if _, err := fmt.Sscanf(m[1], "%d", &val); err == nil {
130+
foundAnyNumber = true
131+
if val*1000 >= minBase {
132+
return true
133+
}
128134
}
129135
}
130136

131-
// We couldn't definitively prove it meets the criteria.
132-
// We return false, meaning we aggressively filter out jobs missing compensation details.
137+
// If we did not discover any numerical salary figures (e.g., "Competitive", "Depends on Experience"),
138+
// we keep the job to avoid aggressively discarding unlisted opportunities.
139+
if !foundAnyNumber {
140+
return true
141+
}
142+
143+
// If numerical figures were found, but ALL of them are strictly below the threshold,
144+
// then we definitively prove it fails the criteria and filter it out.
133145
return false
134146
}
135147

‎internal/fetcher/job_identity_enricher.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,19 @@ import (
77
)
88

99
func newJobIdentityLLMEnricher(ctx context.Context, appCfg *AppConfig) (jobIdentityPageEnrichFunc, func(), string) {
10-
if appCfg == nil || !appCfg.LLM.Enabled || fetchAllJobsInitConfiguredLLM == nil || fetchAllJobsEnrichJobIdentity == nil {
10+
llmSvc := getLLMService(ctx)
11+
if appCfg == nil || !appCfg.LLM.Enabled || !llmSvc.IsAvailable(ctx, "llm_job_enrichment") {
1112
return nil, nil, ""
1213
}
13-
llm, restoreAuth, err := fetchAllJobsInitConfiguredLLM(ctx, appCfg, llmTaskJobIdentity)
14+
llm, restoreAuth, err := llmSvc.InitConfiguredLLM(ctx, appCfg, llmTaskJobIdentity)
1415
if err != nil {
1516
return nil, nil, fmt.Sprintf("LLM job identity enrichment skipped: %v", err)
1617
}
1718
var mu sync.Mutex
1819
enrich := func(ctx context.Context, job Job, page JobIdentityPage) (*JobIdentityEnrichment, LLMTokenUsage, error) {
1920
mu.Lock()
2021
defer mu.Unlock()
21-
return fetchAllJobsEnrichJobIdentity(ctx, llm, job, page)
22+
return llmSvc.EnrichJobIdentity(ctx, llm, job, page)
2223
}
2324
return enrich, restoreAuth, ""
2425
}

‎internal/fetcher/linkedin_typeahead.go‎

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,31 @@ type linkedInTypeaheadCacheEntry struct {
3434

3535
type linkedInTypeaheadCache map[string]linkedInTypeaheadCacheEntry
3636

37-
var linkedInTypeaheadMemoryCache = struct {
38-
sync.Mutex
37+
type linkedinTypeaheadMemoryCache struct {
38+
mu sync.Mutex
3939
path string
4040
loaded bool
4141
cache linkedInTypeaheadCache
42-
}{}
42+
}
43+
44+
var linkedInTypeaheadMemoryCache = &linkedinTypeaheadMemoryCache{}
45+
46+
func (c *linkedinTypeaheadMemoryCache) Get(cachePath string) (linkedInTypeaheadCache, bool) {
47+
c.mu.Lock()
48+
defer c.mu.Unlock()
49+
if c.loaded && c.path == cachePath {
50+
return cloneLinkedInTypeaheadCache(c.cache), true
51+
}
52+
return nil, false
53+
}
54+
55+
func (c *linkedinTypeaheadMemoryCache) Set(cachePath string, cache linkedInTypeaheadCache) {
56+
c.mu.Lock()
57+
defer c.mu.Unlock()
58+
c.path = cachePath
59+
c.loaded = true
60+
c.cache = cloneLinkedInTypeaheadCache(cache)
61+
}
4362

4463
func refreshLinkedInCriteriaHints(ctx context.Context, criteria *CriteriaConfig) error {
4564
if criteria == nil {
@@ -140,19 +159,15 @@ func loadLinkedInTypeaheadCache() (linkedInTypeaheadCache, error) {
140159
return linkedInTypeaheadCache{}, nil
141160
}
142161

143-
linkedInTypeaheadMemoryCache.Lock()
144-
if linkedInTypeaheadMemoryCache.loaded && linkedInTypeaheadMemoryCache.path == cachePath {
145-
cache := cloneLinkedInTypeaheadCache(linkedInTypeaheadMemoryCache.cache)
146-
linkedInTypeaheadMemoryCache.Unlock()
162+
if cache, ok := linkedInTypeaheadMemoryCache.Get(cachePath); ok {
147163
return cache, nil
148164
}
149-
linkedInTypeaheadMemoryCache.Unlock()
150165

151166
data, err := os.ReadFile(cachePath)
152167
if err != nil {
153168
if os.IsNotExist(err) {
154169
cache := linkedInTypeaheadCache{}
155-
storeLinkedInTypeaheadMemoryCache(cachePath, cache)
170+
linkedInTypeaheadMemoryCache.Set(cachePath, cache)
156171
return cache, nil
157172
}
158173
return nil, err
@@ -164,7 +179,7 @@ func loadLinkedInTypeaheadCache() (linkedInTypeaheadCache, error) {
164179
if cache == nil {
165180
cache = linkedInTypeaheadCache{}
166181
}
167-
storeLinkedInTypeaheadMemoryCache(cachePath, cache)
182+
linkedInTypeaheadMemoryCache.Set(cachePath, cache)
168183
return cache, nil
169184
}
170185

@@ -183,18 +198,10 @@ func saveLinkedInTypeaheadCache(cache linkedInTypeaheadCache) error {
183198
if err := os.WriteFile(cachePath, data, 0600); err != nil {
184199
return err
185200
}
186-
storeLinkedInTypeaheadMemoryCache(cachePath, cache)
201+
linkedInTypeaheadMemoryCache.Set(cachePath, cache)
187202
return nil
188203
}
189204

190-
func storeLinkedInTypeaheadMemoryCache(cachePath string, cache linkedInTypeaheadCache) {
191-
linkedInTypeaheadMemoryCache.Lock()
192-
linkedInTypeaheadMemoryCache.path = cachePath
193-
linkedInTypeaheadMemoryCache.loaded = true
194-
linkedInTypeaheadMemoryCache.cache = cloneLinkedInTypeaheadCache(cache)
195-
linkedInTypeaheadMemoryCache.Unlock()
196-
}
197-
198205
func cloneLinkedInTypeaheadCache(cache linkedInTypeaheadCache) linkedInTypeaheadCache {
199206
if cache == nil {
200207
return linkedInTypeaheadCache{}

‎internal/fetcher/llm_hooks.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package fetcher
22

33
import (
44
"context"
5+
"fmt"
6+
"sync"
57

68
"github.com/tmc/langchaingo/llms"
79
)
@@ -14,6 +16,96 @@ type ExecuteLLMSearchFunc func(ctx context.Context, llm llms.Model, prompt strin
1416
type ExecuteLLMWebSearchFunc func(ctx context.Context, appCfg *AppConfig, prompt string) ([]Job, error)
1517
type EnrichJobIdentityFunc func(ctx context.Context, llm llms.Model, job Job, page JobIdentityPage) (*JobIdentityEnrichment, LLMTokenUsage, error)
1618

19+
type LLMService interface {
20+
InitConfiguredLLM(ctx context.Context, appCfg *AppConfig, task string) (llms.Model, func(), error)
21+
ExecuteSearch(ctx context.Context, llm llms.Model, prompt string) ([]Job, error)
22+
ExecuteWebSearch(ctx context.Context, appCfg *AppConfig, prompt string) ([]Job, error)
23+
EnrichJobIdentity(ctx context.Context, llm llms.Model, job Job, page JobIdentityPage) (*JobIdentityEnrichment, LLMTokenUsage, error)
24+
IsAvailable(ctx context.Context, task string) bool
25+
}
26+
27+
type llmServiceContextKey struct{}
28+
29+
var contextKey = llmServiceContextKey{}
30+
31+
func WithLLMService(ctx context.Context, service LLMService) context.Context {
32+
return context.WithValue(ctx, contextKey, service)
33+
}
34+
35+
var (
36+
globalServiceMutex sync.RWMutex
37+
globalLLMService LLMService
38+
)
39+
40+
func RegisterLLMService(service LLMService) {
41+
globalServiceMutex.Lock()
42+
defer globalServiceMutex.Unlock()
43+
globalLLMService = service
44+
}
45+
46+
func getLLMService(ctx context.Context) LLMService {
47+
if svc, ok := ctx.Value(contextKey).(LLMService); ok {
48+
return svc
49+
}
50+
globalServiceMutex.RLock()
51+
defer globalServiceMutex.RUnlock()
52+
if globalLLMService != nil {
53+
return globalLLMService
54+
}
55+
return fallbackLLMService{}
56+
}
57+
58+
type fallbackLLMService struct{}
59+
60+
func (fallbackLLMService) InitConfiguredLLM(ctx context.Context, appCfg *AppConfig, task string) (llms.Model, func(), error) {
61+
if fetchAllJobsInitConfiguredLLM != nil {
62+
return fetchAllJobsInitConfiguredLLM(ctx, appCfg, task)
63+
}
64+
return nil, nil, fmt.Errorf("LLM init hook not configured")
65+
}
66+
67+
func (fallbackLLMService) ExecuteSearch(ctx context.Context, llm llms.Model, prompt string) ([]Job, error) {
68+
if fetchAllJobsExecuteLLMSearch != nil {
69+
return fetchAllJobsExecuteLLMSearch(ctx, llm, prompt)
70+
}
71+
return nil, fmt.Errorf("LLM search hook not configured")
72+
}
73+
74+
func (fallbackLLMService) ExecuteWebSearch(ctx context.Context, appCfg *AppConfig, prompt string) ([]Job, error) {
75+
if fetchAllJobsExecuteLLMWebSearch != nil {
76+
return fetchAllJobsExecuteLLMWebSearch(ctx, appCfg, prompt)
77+
}
78+
if fetchAllJobsInitConfiguredLLM == nil || fetchAllJobsExecuteLLMSearch == nil {
79+
return nil, fmt.Errorf("LLM web search hook not configured")
80+
}
81+
llm, restoreAuth, err := fetchAllJobsInitConfiguredLLM(ctx, appCfg, llmTaskJobSearch)
82+
if err != nil {
83+
return nil, fmt.Errorf("failed to initialize: %w", err)
84+
}
85+
defer restoreAuth()
86+
return fetchAllJobsExecuteLLMSearch(ctx, llm, prompt)
87+
}
88+
89+
func (fallbackLLMService) EnrichJobIdentity(ctx context.Context, llm llms.Model, job Job, page JobIdentityPage) (*JobIdentityEnrichment, LLMTokenUsage, error) {
90+
if fetchAllJobsEnrichJobIdentity != nil {
91+
return fetchAllJobsEnrichJobIdentity(ctx, llm, job, page)
92+
}
93+
return nil, LLMTokenUsage{}, fmt.Errorf("LLM identity enrichment hook not configured")
94+
}
95+
96+
func (fallbackLLMService) IsAvailable(ctx context.Context, task string) bool {
97+
if task == "llm_web_search" {
98+
return fetchAllJobsExecuteLLMWebSearch != nil || (fetchAllJobsInitConfiguredLLM != nil && fetchAllJobsExecuteLLMSearch != nil)
99+
}
100+
if task == "llm_job_search" {
101+
return fetchAllJobsInitConfiguredLLM != nil && fetchAllJobsExecuteLLMSearch != nil
102+
}
103+
if task == "llm_job_enrichment" {
104+
return fetchAllJobsInitConfiguredLLM != nil && fetchAllJobsEnrichJobIdentity != nil
105+
}
106+
return false
107+
}
108+
17109
var (
18110
fetchAllJobsInitConfiguredLLM InitLLMFunc
19111
fetchAllJobsExecuteLLMSearch ExecuteLLMSearchFunc

‎internal/jobscout/cli.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,7 @@ func runFetchDryRun(options appruntime.Options, stores appruntime.Stores, jsonOu
125125
existing = []domain.Job{}
126126
}
127127

128-
fetcher.ConfigureLLM(llmpkg.InitConfiguredLLMForTask, llmpkg.ExecuteLLMSearch, llmpkg.EnrichJobIdentityWithLLMUsage)
129-
fetcher.ConfigureLLMWebSearch(llmpkg.ExecuteLLMWebSearch)
128+
fetcher.RegisterLLMService(llmpkg.NewLLMService())
130129
newJobs, summary, err := fetcher.FetchAllJobsSkippingExisting(ctx, appCfg, criteriaCfg, existing, nil)
131130
if err != nil {
132131
if !jsonOutput {
@@ -224,8 +223,7 @@ func runRepairJobIdentityCLI(options appruntime.Options, stores appruntime.Store
224223
repairJobs, repairIndexes := domain.IdentityRepairTargets(jobs)
225224
fmt.Printf("Repairing identity data for %d of %d jobs.\n", len(repairJobs), len(jobs))
226225
if len(repairJobs) > 0 {
227-
fetcher.ConfigureLLM(llmpkg.InitConfiguredLLMForTask, llmpkg.ExecuteLLMSearch, llmpkg.EnrichJobIdentityWithLLMUsage)
228-
fetcher.ConfigureLLMWebSearch(llmpkg.ExecuteLLMWebSearch)
226+
fetcher.RegisterLLMService(llmpkg.NewLLMService())
229227
repairJobs = fetcher.EnrichJobsFromApplyPagesWithConfigStoreAndProgress(ctx, repairJobs, appCfg, stores.CompanyIdentity, func(message string) {
230228
fmt.Println(message)
231229
}, nil)

‎internal/llm/llm_service.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package llm
2+
3+
import (
4+
"context"
5+
6+
"github.com/tmc/langchaingo/llms"
7+
"github.com/wallentx/jobscout/internal/fetcher"
8+
)
9+
10+
type defaultLLMService struct{}
11+
12+
func NewLLMService() fetcher.LLMService {
13+
return defaultLLMService{}
14+
}
15+
16+
func (defaultLLMService) InitConfiguredLLM(ctx context.Context, appCfg *fetcher.AppConfig, task string) (llms.Model, func(), error) {
17+
return InitConfiguredLLMForTask(ctx, appCfg, task)
18+
}
19+
20+
func (defaultLLMService) ExecuteSearch(ctx context.Context, llm llms.Model, prompt string) ([]fetcher.Job, error) {
21+
return ExecuteLLMSearch(ctx, llm, prompt)
22+
}
23+
24+
func (defaultLLMService) ExecuteWebSearch(ctx context.Context, appCfg *fetcher.AppConfig, prompt string) ([]fetcher.Job, error) {
25+
return ExecuteLLMWebSearch(ctx, appCfg, prompt)
26+
}
27+
28+
func (defaultLLMService) EnrichJobIdentity(ctx context.Context, llm llms.Model, job fetcher.Job, page fetcher.JobIdentityPage) (*fetcher.JobIdentityEnrichment, fetcher.LLMTokenUsage, error) {
29+
return EnrichJobIdentityWithLLMUsage(ctx, llm, job, page)
30+
}
31+
32+
func (defaultLLMService) IsAvailable(ctx context.Context, task string) bool {
33+
return true
34+
}

‎internal/tuiapp/background_task.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,8 +319,7 @@ func enrichAcceptedFetchCmd(taskID int, disableLLM bool, jobs []Job, progressCh
319319
appCfg = sessionFetchConfig(disableLLM, appCfg)
320320
}
321321

322-
fetcher.ConfigureLLM(llmpkg.InitConfiguredLLMForTask, llmpkg.ExecuteLLMSearch, llmpkg.EnrichJobIdentityWithLLMUsage)
323-
fetcher.ConfigureLLMWebSearch(llmpkg.ExecuteLLMWebSearch)
322+
fetcher.RegisterLLMService(llmpkg.NewLLMService())
324323
reportAcceptedFetchProgress(progressCh, "Enriching accepted jobs in the background...")
325324
enriched := fetcher.EnrichJobsFromApplyPagesWithConfigStoreAndProgress(ctx, append([]Job(nil), jobs...), appCfg, runtimeCompanyIdentityStore, func(message string) {
326325
reportAcceptedFetchProgress(progressCh, message)

‎internal/tuiapp/fetch_flow.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,7 @@ func applyOptionalLLMJobFilteringWithFreshTimeout(appCfg *AppConfig, criteriaCfg
219219
}
220220

221221
func fetchAllJobs(ctx context.Context, appCfg *AppConfig, criteriaCfg *CriteriaConfig, existingJobs []Job, progress func(string)) ([]Job, FetchSummary, error) {
222-
fetcher.ConfigureLLM(llmpkg.InitConfiguredLLMForTask, llmpkg.ExecuteLLMSearch, llmpkg.EnrichJobIdentityWithLLMUsage)
223-
fetcher.ConfigureLLMWebSearch(llmpkg.ExecuteLLMWebSearch)
222+
fetcher.RegisterLLMService(llmpkg.NewLLMService())
224223
return fetcher.FetchAllJobsSkippingExistingWithCandidateCache(ctx, appCfg, criteriaCfg, existingJobs, progress, runtimeCandidateStore)
225224
}
226225

0 commit comments

Comments
 (0)