149 lines
3.2 KiB
Go
149 lines
3.2 KiB
Go
package util
|
|
|
|
import (
|
|
"govd/config"
|
|
"govd/models"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
var (
|
|
defaultClient *http.Client
|
|
defaultClientOnce sync.Once
|
|
extractorClients = make(map[string]models.HTTPClient)
|
|
)
|
|
|
|
func GetDefaultHTTPClient() *http.Client {
|
|
defaultClientOnce.Do(func() {
|
|
defaultClient = &http.Client{
|
|
Transport: GetBaseTransport(),
|
|
Timeout: 60 * time.Second,
|
|
}
|
|
})
|
|
return defaultClient
|
|
}
|
|
|
|
func GetBaseTransport() *http.Transport {
|
|
return &http.Transport{
|
|
Proxy: http.ProxyFromEnvironment,
|
|
DialContext: (&net.Dialer{
|
|
Timeout: 30 * time.Second,
|
|
KeepAlive: 30 * time.Second,
|
|
}).DialContext,
|
|
ForceAttemptHTTP2: true,
|
|
MaxIdleConns: 100,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 5 * time.Second,
|
|
ExpectContinueTimeout: 1 * time.Second,
|
|
MaxIdleConnsPerHost: 100,
|
|
MaxConnsPerHost: 100,
|
|
ResponseHeaderTimeout: 10 * time.Second,
|
|
DisableCompression: false,
|
|
}
|
|
}
|
|
|
|
func GetHTTPClient(extractor string) models.HTTPClient {
|
|
if client, exists := extractorClients[extractor]; exists {
|
|
return client
|
|
}
|
|
|
|
cfg := config.GetExtractorConfig(extractor)
|
|
if cfg == nil {
|
|
return GetDefaultHTTPClient()
|
|
}
|
|
|
|
var client models.HTTPClient
|
|
|
|
if cfg.EdgeProxyURL != "" {
|
|
client = NewEdgeProxyFromConfig(cfg)
|
|
} else {
|
|
client = NewClientFromConfig(cfg)
|
|
}
|
|
extractorClients[extractor] = client
|
|
return client
|
|
}
|
|
|
|
func NewClientFromConfig(cfg *models.ExtractorConfig) *http.Client {
|
|
var baseClient *http.Client
|
|
if cfg.Impersonate {
|
|
baseClient = NewChromeClient()
|
|
} else {
|
|
baseClient = GetDefaultHTTPClient()
|
|
}
|
|
transport := GetBaseTransport()
|
|
if cfg.HTTPProxy != "" || cfg.HTTPSProxy != "" {
|
|
configureProxyTransport(transport, cfg)
|
|
}
|
|
baseClient.Transport = transport
|
|
return baseClient
|
|
}
|
|
|
|
func configureProxyTransport(
|
|
transport *http.Transport,
|
|
cfg *models.ExtractorConfig,
|
|
) {
|
|
var httpProxyURL, httpsProxyURL *url.URL
|
|
var err error
|
|
|
|
if cfg.HTTPProxy != "" {
|
|
httpProxyURL, err = url.Parse(cfg.HTTPProxy)
|
|
if err != nil {
|
|
log.Printf("warning: invalid HTTP proxy URL '%s': %v\n", cfg.HTTPProxy, err)
|
|
}
|
|
}
|
|
if cfg.HTTPSProxy != "" {
|
|
httpsProxyURL, err = url.Parse(cfg.HTTPSProxy)
|
|
if err != nil {
|
|
log.Printf("warning: invalid HTTPS proxy URL '%s': %v\n", cfg.HTTPSProxy, err)
|
|
}
|
|
}
|
|
if httpProxyURL == nil && httpsProxyURL == nil {
|
|
return
|
|
}
|
|
noProxyList := parseNoProxyList(cfg.NoProxy)
|
|
transport.Proxy = func(req *http.Request) (*url.URL, error) {
|
|
if shouldBypassProxy(req.URL.Hostname(), noProxyList) {
|
|
return nil, nil
|
|
}
|
|
|
|
if req.URL.Scheme == "https" && httpsProxyURL != nil {
|
|
return httpsProxyURL, nil
|
|
}
|
|
if req.URL.Scheme == "http" && httpProxyURL != nil {
|
|
return httpProxyURL, nil
|
|
}
|
|
if httpsProxyURL != nil {
|
|
return httpsProxyURL, nil
|
|
}
|
|
return httpProxyURL, nil
|
|
}
|
|
}
|
|
|
|
func parseNoProxyList(noProxy string) []string {
|
|
if noProxy == "" {
|
|
return nil
|
|
}
|
|
|
|
list := strings.Split(noProxy, ",")
|
|
for i := range list {
|
|
list[i] = strings.TrimSpace(list[i])
|
|
}
|
|
return list
|
|
}
|
|
|
|
func shouldBypassProxy(host string, noProxyList []string) bool {
|
|
for _, p := range noProxyList {
|
|
if p == "" {
|
|
continue
|
|
}
|
|
if p == host || (strings.HasPrefix(p, ".") && strings.HasSuffix(host, p)) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|