From 10c113f400151086d3648ea94a7bf1dec3194081 Mon Sep 17 00:00:00 2001 From: stefanodvx <69367859+stefanodvx@users.noreply.github.com> Date: Mon, 14 Apr 2025 21:50:49 +0200 Subject: [PATCH] env: supports for http and socks5 proxies --- .env.example | 7 ++++++- bot/middleware.go | 10 +++++++++- ext/instagram/main.go | 20 +++----------------- ext/instagram/util.go | 2 +- ext/pinterest/main.go | 5 +++-- ext/reddit/main.go | 13 +++---------- ext/tiktok/main.go | 18 ++---------------- ext/twitter/main.go | 6 +++--- util/download.go | 18 +++++++++++------- util/http.go | 12 ++++++++++-- 10 files changed, 51 insertions(+), 60 deletions(-) diff --git a/.env.example b/.env.example index 706da5e..81ebafc 100644 --- a/.env.example +++ b/.env.example @@ -7,4 +7,9 @@ DB_NAME=govd DB_USER=govd DB_PASSWORD=password -REPO_URL=https://github.com/govdbot/govd \ No newline at end of file +REPO_URL=https://github.com/govdbot/govd + +# proxy +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY= \ No newline at end of file diff --git a/bot/middleware.go b/bot/middleware.go index ce4ea5d..5bfb91e 100644 --- a/bot/middleware.go +++ b/bot/middleware.go @@ -5,6 +5,7 @@ import ( "encoding/json" "log" "net/http" + "net/url" "os" "strings" "time" @@ -45,7 +46,14 @@ func NewBotClient() BotClient { } return BotClient{ BotClient: &gotgbot.BaseBotClient{ - Client: http.Client{}, + Client: http.Client{ + Transport: &http.Transport{ + // avoid using proxy for telegram + Proxy: func(r *http.Request) (*url.URL, error) { + return nil, nil + }, + }, + }, UseTestEnvironment: false, DefaultRequestOpts: &gotgbot.RequestOpts{ Timeout: 10 * time.Minute, diff --git a/ext/instagram/main.go b/ext/instagram/main.go index f85f516..187d848 100644 --- a/ext/instagram/main.go +++ b/ext/instagram/main.go @@ -1,7 +1,6 @@ package instagram import ( - "crypto/tls" "fmt" "govd/enums" "govd/models" @@ -9,9 +8,6 @@ import ( "io" "net/http" "regexp" - - "github.com/quic-go/quic-go" - "github.com/quic-go/quic-go/http3" ) // as a public service, we can't use the official API @@ -46,17 +42,7 @@ var igHeaders = map[string]string{ "User-Agent": util.ChromeUA, } -var HTTPClient = &http.Client{ - Transport: &http3.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - QUICConfig: &quic.Config{ - MaxIncomingStreams: -1, - EnableDatagrams: true, - }, - }, -} +var HTTPSession = util.NewHTTPSession() var Extractor = &models.Extractor{ Name: "Instagram", @@ -110,7 +96,7 @@ var ShareURLExtractor = &models.Extractor{ for k, v := range igHeaders { req.Header.Set(k, v) } - resp, err := HTTPClient.Do(req) + resp, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -197,7 +183,7 @@ func GetVideoAPI(contentURL string) (*IGramResponse, error) { req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", util.ChromeUA) - resp, err := HTTPClient.Do(req) + resp, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/instagram/util.go b/ext/instagram/util.go index 21cd9a9..4e9d4eb 100644 --- a/ext/instagram/util.go +++ b/ext/instagram/util.go @@ -115,7 +115,7 @@ func GetPostCaption( req.Header.Set("Cache-Control", "no-cache") req.Header.Set("TE", "trailers") - resp, err := HTTPClient.Do(req) + resp, err := HTTPSession.Do(req) if err != nil { return "", fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/pinterest/main.go b/ext/pinterest/main.go index 687f398..f639836 100644 --- a/ext/pinterest/main.go +++ b/ext/pinterest/main.go @@ -17,6 +17,8 @@ const ( shortenerAPIFormat = "https://api.pinterest.com/url_shortener/%s/redirect/" ) +var HTTPSession = util.NewHTTPSession() + var ShortExtractor = &models.Extractor{ Name: "Pinterest (Short)", CodeName: "pinterest:short", @@ -146,8 +148,7 @@ func GetPinData(pinID string) (*PinData, error) { // fix 403 error req.Header.Set("X-Pinterest-PWS-Handler", "www/[username].js") - client := &http.Client{} - resp, err := client.Do(req) + resp, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/reddit/main.go b/ext/reddit/main.go index 0500e97..8335641 100644 --- a/ext/reddit/main.go +++ b/ext/reddit/main.go @@ -12,14 +12,7 @@ import ( "govd/util" ) -var HTTPClient = &http.Client{ - CheckRedirect: func(req *http.Request, via []*http.Request) error { - if len(via) >= 10 { - return fmt.Errorf("stopped after 10 redirects") - } - return nil - }, -} +var HTTPSession = util.NewHTTPSession() var ShortExtractor = &models.Extractor{ Name: "Reddit (Short)", @@ -44,7 +37,7 @@ var ShortExtractor = &models.Extractor{ req.AddCookie(cookie) } - res, err := HTTPClient.Do(req) + res, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -236,7 +229,7 @@ func GetRedditData(host string, slug string) (RedditResponse, error) { req.AddCookie(cookie) } - res, err := HTTPClient.Do(req) + res, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/tiktok/main.go b/ext/tiktok/main.go index 5045f5b..fe58b89 100644 --- a/ext/tiktok/main.go +++ b/ext/tiktok/main.go @@ -1,16 +1,12 @@ package tiktok import ( - "crypto/tls" "encoding/json" "fmt" "io" "net/http" "regexp" - "github.com/quic-go/quic-go" - "github.com/quic-go/quic-go/http3" - "govd/enums" "govd/models" "govd/util" @@ -27,17 +23,7 @@ const ( appUserAgent = packageID + " (Linux; U; Android 13; en_US; Pixel 7; Build/TD1A.220804.031; Cronet/58.0.2991.0)" ) -var HTTPClient = &http.Client{ - Transport: &http3.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - QUICConfig: &quic.Config{ - MaxIncomingStreams: -1, - EnableDatagrams: true, - }, - }, -} +var HTTPSession = util.NewHTTPSession() var VMExtractor = &models.Extractor{ Name: "TikTok VM", @@ -161,7 +147,7 @@ func GetVideoAPI(awemeID string) (*AwemeDetails, error) { req.Header.Set("Accept", "application/json") req.Header.Set("X-Argus", "") - resp, err := HTTPClient.Do(req) + resp, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/twitter/main.go b/ext/twitter/main.go index fca8923..a79877e 100644 --- a/ext/twitter/main.go +++ b/ext/twitter/main.go @@ -17,7 +17,7 @@ const ( apiEndpoint = "https://x.com/i/api/graphql/zZXycP0V6H7m-2r0mOnFcA/TweetDetail" ) -var HTTPClient = &http.Client{} +var HTTPSession = util.NewHTTPSession() var ShortExtractor = &models.Extractor{ Name: "Twitter (Short)", @@ -33,7 +33,7 @@ var ShortExtractor = &models.Extractor{ return nil, fmt.Errorf("failed to create req: %w", err) } req.Header.Set("User-Agent", util.ChromeUA) - res, err := HTTPClient.Do(req) + res, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -151,7 +151,7 @@ func GetTweetAPI(tweetID string) (*Tweet, error) { } req.URL.RawQuery = q.Encode() - resp, err := HTTPClient.Do(req) + resp, err := HTTPSession.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/util/download.go b/util/download.go index 80a6c02..0e9135f 100644 --- a/util/download.go +++ b/util/download.go @@ -45,7 +45,7 @@ func DownloadFile( return "", ctx.Err() default: // create the download directory if it doesn't exist - if err := ensureDownloadDir(config.DownloadDir); err != nil { + if err := EnsureDownloadDir(config.DownloadDir); err != nil { return "", err } @@ -78,7 +78,7 @@ func DownloadFileWithSegments( if config == nil { config = DefaultConfig() } - if err := ensureDownloadDir(config.DownloadDir); err != nil { + if err := EnsureDownloadDir(config.DownloadDir); err != nil { return "", err } tempDir := filepath.Join(config.DownloadDir, "segments_"+time.Now().Format("20060102_150405")) @@ -151,10 +151,14 @@ func downloadInMemory(ctx context.Context, fileURL string, timeout time.Duration return io.ReadAll(resp.Body) } -func ensureDownloadDir(dir string) error { - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err := os.MkdirAll(dir, 0755); err != nil { - return fmt.Errorf("failed to create downloads directory: %w", err) +func EnsureDownloadDir(dir string) error { + if _, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create downloads directory: %w", err) + } + } else { + return fmt.Errorf("error accessing directory: %w", err) } } return nil @@ -453,7 +457,7 @@ func MergeSegmentFiles( config = DefaultConfig() } - if err := ensureDownloadDir(config.DownloadDir); err != nil { + if err := EnsureDownloadDir(config.DownloadDir); err != nil { return "", err } diff --git a/util/http.go b/util/http.go index fbfee9b..7891587 100644 --- a/util/http.go +++ b/util/http.go @@ -5,8 +5,16 @@ import ( "time" ) -var httpSession = &http.Client{ - Timeout: 20 * time.Second, +var httpSession = NewHTTPSession() + +func NewHTTPSession() *http.Client { + session := &http.Client{ + Timeout: 20 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + }, + } + return session } func GetHTTPSession() *http.Client {