Skip to content

Commit 3daa48e

Browse files
karustkarust
committed
Proxy policy wiring and fail-closed execution
- replace proxy wiring with global and per-engine tag policies - split stats endpoints and lock fail-closed proxy behavior with tests
1 parent 8bec557 commit 3daa48e

15 files changed

Lines changed: 1899 additions & 577 deletions

‎cmd/proxy_policy.go‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package cmd
2+
3+
import (
4+
"strings"
5+
6+
"github.com/karust/openserp/core"
7+
)
8+
9+
func buildEngineProxyPolicyMap() map[string]string {
10+
return map[string]string{
11+
"google": config.GoogleConfig.Proxy,
12+
"yandex": config.YandexConfig.Proxy,
13+
"baidu": config.BaiduConfig.Proxy,
14+
"bing": config.BingConfig.Proxy,
15+
"duckduckgo": config.DuckDuckGoConfig.Proxy,
16+
}
17+
}
18+
19+
func buildNormalizedProxyConfig(runtime string) (core.ProxyConfig, error) {
20+
return core.NormalizeProxyConfig(core.ProxyConfig{
21+
Runtime: runtime,
22+
Proxies: config.Proxies,
23+
EnginePolicies: buildEngineProxyPolicyMap(),
24+
})
25+
}
26+
27+
func resolveEngineProxyPolicy(proxyCfg core.ProxyConfig, engineName string) core.ProxyPolicy {
28+
engineKey := strings.ToLower(strings.TrimSpace(engineName))
29+
return core.ResolveEffectiveProxyPolicy(proxyCfg.Proxies.Global, proxyCfg.EnginePolicies[engineKey])
30+
}

‎cmd/root.go‎

Lines changed: 130 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -19,44 +19,47 @@ const (
1919
)
2020

2121
type Config struct {
22-
App AppConfig `mapstructure:"app"`
23-
ProxyPool ProxyPoolConfig `mapstructure:"proxy_pool"`
24-
Cache CacheConfig `mapstructure:"cache"`
25-
Resilience ResilienceConfig `mapstructure:"resilience"`
26-
CircuitBreaker CircuitBreakerConfig `mapstructure:"circuit_breaker"`
27-
CORS CORSConfig `mapstructure:"cors"`
28-
Config2Capcha Config2Captcha `mapstructure:"2captcha"`
29-
GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
30-
YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
31-
BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"`
32-
BingConfig core.SearchEngineOptions `mapstructure:"bing"`
33-
DuckDuckGoConfig core.SearchEngineOptions `mapstructure:"duckduckgo"`
22+
Server ServerConfig `mapstructure:"server"`
23+
App AppConfig `mapstructure:"app"`
24+
Proxies core.ProxiesConfig `mapstructure:"proxies"`
25+
Cache CacheConfig `mapstructure:"cache"`
26+
Resilience ResilienceConfig `mapstructure:"resilience"`
27+
CircuitBreaker CircuitBreakerConfig `mapstructure:"circuit_breaker"`
28+
CORS CORSConfig `mapstructure:"cors"`
29+
Config2Capcha Config2Captcha `mapstructure:"2captcha"`
30+
GoogleConfig EngineConfig `mapstructure:"google"`
31+
YandexConfig EngineConfig `mapstructure:"yandex"`
32+
BaiduConfig EngineConfig `mapstructure:"baidu"`
33+
BingConfig EngineConfig `mapstructure:"bing"`
34+
DuckDuckGoConfig EngineConfig `mapstructure:"duckduckgo"`
3435
}
3536

3637
type Config2Captcha struct {
3738
ApiKey string `mapstructure:"apikey"`
3839
}
3940

40-
type AppConfig struct {
41+
type ServerConfig struct {
4142
Host string `mapstructure:"host"`
4243
Port int `mapstructure:"port"`
43-
Timeout int `mapstructure:"timeout"`
4444
ConfigPath string `mapstructure:"config_path"`
45-
BrowserPath string `mapstructure:"browser_path"`
46-
IsBrowserHead bool `mapstructure:"head"`
47-
IsLeaveHead bool `mapstructure:"leave_head"`
48-
IsLeakless bool `mapstructure:"leakless"`
4945
IsDebug bool `mapstructure:"debug"`
5046
IsVerbose bool `mapstructure:"verbose"`
5147
IsRawRequests bool `mapstructure:"raw_requests"`
52-
ProxyURL string `mapstructure:"proxy"`
5348
Insecure bool `mapstructure:"insecure"`
49+
}
50+
51+
type AppConfig struct {
52+
Timeout int `mapstructure:"timeout"`
53+
BrowserPath string `mapstructure:"browser_path"`
54+
IsBrowserHead bool `mapstructure:"head"`
55+
IsLeaveHead bool `mapstructure:"leave_head"`
56+
IsLeakless bool `mapstructure:"leakless"`
5457
IsStealth bool `mapstructure:"stealth"`
5558
}
5659

57-
type ProxyPoolConfig struct {
58-
URLs []string `mapstructure:"urls"`
59-
FailureThreshold int `mapstructure:"failure_threshold"`
60+
type EngineConfig struct {
61+
core.SearchEngineOptions `mapstructure:",squash"`
62+
Proxy string `mapstructure:"proxy"`
6063
}
6164

6265
type CacheConfig struct {
@@ -86,11 +89,21 @@ type CORSConfig struct {
8689
var config = Config{}
8790

8891
var flagToConfigKey = map[string]string{
89-
"config": "app.config_path",
92+
"host": "server.host",
93+
"port": "server.port",
94+
"timeout": "app.timeout",
95+
"config": "server.config_path",
9096
"browser-path": "app.browser_path",
97+
"verbose": "server.verbose",
98+
"debug": "server.debug",
99+
"head": "app.head",
100+
"leakless": "app.leakless",
101+
"raw": "server.raw_requests",
91102
"leave": "app.leave_head",
92-
"raw": "app.raw_requests",
93103
"2captcha_key": "2captcha.apikey",
104+
"proxy": "proxies.global",
105+
"stealth": "app.stealth",
106+
"insecure": "server.insecure",
94107
"cache_ttl": "cache.ttl_seconds",
95108
"cache_max_size": "cache.max_size",
96109
"max_retries": "resilience.max_retries",
@@ -107,22 +120,15 @@ var RootCmd = &cobra.Command{
107120
Version: version,
108121
SilenceUsage: true,
109122
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
110-
core.InitLogger(config.App.IsVerbose, config.App.IsDebug)
111-
112123
err := initializeConfig(cmd)
113124
if err != nil {
114125
return err
115126
}
116127

128+
core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug)
117129
logrus.Debugf("Final config: %+v", config)
118130
return nil
119131
},
120-
121-
// Run: func(cmd *cobra.Command, args []string) {
122-
// // Working with OutOrStdout/OutOrStderr allows us to unit test our command easier
123-
// //out := cmd.OutOrStdout()
124-
// logrus.Trace("Config:", config)
125-
// },
126132
}
127133

128134
// Bind each cobra flag to its associated viper configuration (config file and environment variable)
@@ -189,35 +195,107 @@ func initializeConfig(cmd *cobra.Command) error {
189195
// 3. Command flags (highest priority). Bind the current command's flags to viper
190196
bindFlags(cmd, v)
191197

198+
if err := validateRemovedConfigPaths(v); err != nil {
199+
return err
200+
}
201+
192202
// Dump Viper values to config struct
203+
if err := validateEngineProxyTags(v); err != nil {
204+
return err
205+
}
206+
193207
err = v.Unmarshal(&config)
194208
if err != nil {
195209
return fmt.Errorf("cannot unmarshall config: %v", err)
196210
}
197211

198-
config.App.ProxyURL, err = core.NormalizeProxyURL(config.App.ProxyURL)
212+
config.Proxies, err = core.NormalizeProxiesConfig(config.Proxies)
199213
if err != nil {
200-
return fmt.Errorf("invalid app.proxy: %w", err)
214+
return fmt.Errorf("invalid proxies config: %w", err)
201215
}
202216

203-
config.ProxyPool.URLs, err = core.NormalizeProxyURLs(config.ProxyPool.URLs)
204-
if err != nil {
205-
return fmt.Errorf("invalid proxy_pool.urls: %w", err)
217+
if config.Server.IsDebug {
218+
logrus.Debug("Viper config:")
219+
v.Debug()
220+
}
221+
return nil
222+
}
223+
224+
func validateEngineProxyTags(v *viper.Viper) error {
225+
for _, engineName := range []string{"google", "yandex", "baidu", "bing", "duckduckgo"} {
226+
key := engineName + ".proxy"
227+
if !v.IsSet(key) {
228+
continue
229+
}
230+
231+
raw := v.Get(key)
232+
tag, ok := raw.(string)
233+
if !ok {
234+
return fmt.Errorf("invalid %s.proxy config: proxy must be a string tag", engineName)
235+
}
236+
237+
if _, err := core.NormalizeProxyTag(tag); err != nil {
238+
return fmt.Errorf("invalid %s.proxy config: %w", engineName, err)
239+
}
206240
}
207-
if config.ProxyPool.FailureThreshold <= 0 {
208-
config.ProxyPool.FailureThreshold = core.DefaultProxyPoolFailureThreshold
241+
242+
return nil
243+
}
244+
245+
func validateRemovedConfigPaths(v *viper.Viper) error {
246+
legacyKeys := map[string]string{
247+
"app.proxy": "use proxies.global or proxies.entries with per-engine proxy tags instead",
248+
"proxy_pool": "use proxies.entries and proxies.health.failure_threshold instead",
249+
"proxy_pool.urls": "use proxies.entries instead",
250+
"proxy_pool.failure_threshold": "use proxies.health.failure_threshold instead",
251+
"app.host": "move to server.host",
252+
"app.port": "move to server.port",
253+
"app.debug": "move to server.debug",
254+
"app.verbose": "move to server.verbose",
255+
"app.raw_requests": "move to server.raw_requests",
256+
"app.insecure": "move to server.insecure",
257+
"proxies.defaults": "use proxies.global or per-engine proxy tags instead",
258+
"proxies.defaults.mode": "use proxies.global or per-engine proxy tags instead",
259+
"proxies.defaults.tag": "use per-engine proxy tags on each engine instead",
260+
"google.proxy.mode": "use google.proxy: <tag> or omit it for direct mode",
261+
"google.proxy.tag": "use google.proxy: <tag>",
262+
"yandex.proxy.mode": "use yandex.proxy: <tag> or omit it for direct mode",
263+
"yandex.proxy.tag": "use yandex.proxy: <tag>",
264+
"baidu.proxy.mode": "use baidu.proxy: <tag> or omit it for direct mode",
265+
"baidu.proxy.tag": "use baidu.proxy: <tag>",
266+
"bing.proxy.mode": "use bing.proxy: <tag> or omit it for direct mode",
267+
"bing.proxy.tag": "use bing.proxy: <tag>",
268+
"duckduckgo.proxy.mode": "use duckduckgo.proxy: <tag> or omit it for direct mode",
269+
"duckduckgo.proxy.tag": "use duckduckgo.proxy: <tag>",
209270
}
210271

211-
if config.App.IsDebug {
212-
logrus.Debug("Viper config:")
213-
v.Debug()
272+
for key, hint := range legacyKeys {
273+
if v.IsSet(key) {
274+
return fmt.Errorf("config key %q is removed in proxy v2: %s", key, hint)
275+
}
214276
}
215277
return nil
216278
}
217279

218280
func setConfigDefaults(v *viper.Viper) {
219-
v.SetDefault("proxy_pool.urls", []string{})
220-
v.SetDefault("proxy_pool.failure_threshold", core.DefaultProxyPoolFailureThreshold)
281+
v.SetDefault("server.host", "127.0.0.1")
282+
v.SetDefault("server.port", 7070)
283+
v.SetDefault("server.debug", false)
284+
v.SetDefault("server.verbose", false)
285+
v.SetDefault("server.raw_requests", false)
286+
v.SetDefault("server.insecure", false)
287+
288+
v.SetDefault("app.timeout", 30)
289+
v.SetDefault("app.browser_path", "")
290+
v.SetDefault("app.head", false)
291+
v.SetDefault("app.leave_head", false)
292+
v.SetDefault("app.leakless", false)
293+
v.SetDefault("app.stealth", false)
294+
295+
v.SetDefault("proxies.entries", []interface{}{})
296+
v.SetDefault("proxies.global", "")
297+
v.SetDefault("proxies.health.failure_threshold", core.DefaultProxyFailureThreshold)
298+
221299
v.SetDefault("cache.ttl_seconds", 300)
222300
v.SetDefault("cache.max_size", 1000)
223301
// Keep stage2 defaults stable even when config file is absent.
@@ -234,21 +312,21 @@ func setConfigDefaults(v *viper.Viper) {
234312
}
235313

236314
func init() {
237-
RootCmd.PersistentFlags().IntVarP(&config.App.Port, "port", "p", 7070, "Port number to run server")
238-
RootCmd.PersistentFlags().StringVarP(&config.App.Host, "host", "a", "127.0.0.1", "Host address to run server")
315+
RootCmd.PersistentFlags().IntVarP(&config.Server.Port, "port", "p", 7070, "Port number to run server")
316+
RootCmd.PersistentFlags().StringVarP(&config.Server.Host, "host", "a", "127.0.0.1", "Host address to run server")
239317
RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
240-
RootCmd.PersistentFlags().StringVarP(&config.App.ConfigPath, "config", "c", "", "Configuration file path")
318+
RootCmd.PersistentFlags().StringVarP(&config.Server.ConfigPath, "config", "c", "", "Configuration file path")
241319
RootCmd.PersistentFlags().StringVarP(&config.App.BrowserPath, "browser-path", "", "", "Custom browser binary path (Chrome/Chromium/Edge/Brave..)")
242-
RootCmd.PersistentFlags().BoolVarP(&config.App.IsVerbose, "verbose", "v", false, "Use verbose output")
243-
RootCmd.PersistentFlags().BoolVarP(&config.App.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
320+
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
321+
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
244322
RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
245323
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
246-
RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
324+
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
247325
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
248326
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
249-
RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP/HTTPS/SOCKS5/SOCKS5H proxy URL (e.g. socks5h://127.0.0.1:1080)")
327+
RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
250328
RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
251-
RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
329+
RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
252330
RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")
253331
RootCmd.PersistentFlags().IntVar(&config.Cache.MaxSize, "cache_max_size", 1000, "Maximum number of cached responses")
254332
RootCmd.PersistentFlags().IntVar(&config.Resilience.MaxRetries, "max_retries", 3, "Max retry attempts per search engine (0 to disable)")

0 commit comments

Comments
 (0)