env: supports for http and socks5 proxies

.env.example

@@ -8,3 +8,8 @@ DB_USER=govd
 DB_PASSWORD=password
 
 REPO_URL=https://github.com/govdbot/govd
+
+# proxy
+HTTP_PROXY=
+HTTPS_PROXY=
+NO_PROXY=

bot/middleware.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"time"
@@ -45,7 +46,14 @@ func NewBotClient() BotClient {
 	}
 	return BotClient{
 		BotClient: &gotgbot.BaseBotClient{
-			Client:             http.Client{},
+			Client: http.Client{
+				Transport: &http.Transport{
+					// avoid using proxy for telegram
+					Proxy: func(r *http.Request) (*url.URL, error) {
+						return nil, nil
+					},
+				},
+			},
 			UseTestEnvironment: false,
 			DefaultRequestOpts: &gotgbot.RequestOpts{
 				Timeout: 10 * time.Minute,

ext/instagram/main.go

@@ -1,7 +1,6 @@
 package instagram
 
 import (
-	"crypto/tls"
 	"fmt"
 	"govd/enums"
 	"govd/models"
@@ -9,9 +8,6 @@ import (
 	"io"
 	"net/http"
 	"regexp"
-
-	"github.com/quic-go/quic-go"
-	"github.com/quic-go/quic-go/http3"
 )
 
 // as a public service, we can't use the official API
@@ -46,17 +42,7 @@ var igHeaders = map[string]string{
 	"User-Agent":                util.ChromeUA,
 }
 
-var HTTPClient = &http.Client{
+var HTTPSession = util.NewHTTPSession()
-	Transport: &http3.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-		QUICConfig: &quic.Config{
-			MaxIncomingStreams: -1,
-			EnableDatagrams:    true,
-		},
-	},
-}
 
 var Extractor = &models.Extractor{
 	Name:       "Instagram",
@@ -110,7 +96,7 @@ var ShareURLExtractor = &models.Extractor{
 		for k, v := range igHeaders {
 			req.Header.Set(k, v)
 		}
-		resp, err := HTTPClient.Do(req)
+		resp, err := HTTPSession.Do(req)
 		if err != nil {
 			return nil, fmt.Errorf("failed to send request: %w", err)
 		}
@@ -197,7 +183,7 @@ func GetVideoAPI(contentURL string) (*IGramResponse, error) {
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("User-Agent", util.ChromeUA)
 
-	resp, err := HTTPClient.Do(req)
+	resp, err := HTTPSession.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

ext/instagram/util.go

@@ -115,7 +115,7 @@ func GetPostCaption(
 	req.Header.Set("Cache-Control", "no-cache")
 	req.Header.Set("TE", "trailers")
 
-	resp, err := HTTPClient.Do(req)
+	resp, err := HTTPSession.Do(req)
 	if err != nil {
 		return "", fmt.Errorf("failed to send request: %w", err)
 	}

ext/pinterest/main.go

@@ -17,6 +17,8 @@ const (
 	shortenerAPIFormat  = "https://api.pinterest.com/url_shortener/%s/redirect/"
 )
 
+var HTTPSession = util.NewHTTPSession()
+
 var ShortExtractor = &models.Extractor{
 	Name:       "Pinterest (Short)",
 	CodeName:   "pinterest:short",
@@ -146,8 +148,7 @@ func GetPinData(pinID string) (*PinData, error) {
 	// fix 403 error
 	req.Header.Set("X-Pinterest-PWS-Handler", "www/[username].js")
 
-	client := &http.Client{}
+	resp, err := HTTPSession.Do(req)
-	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

ext/reddit/main.go

@@ -12,14 +12,7 @@ import (
 	"govd/util"
 )
 
-var HTTPClient = &http.Client{
+var HTTPSession = util.NewHTTPSession()
-	CheckRedirect: func(req *http.Request, via []*http.Request) error {
-		if len(via) >= 10 {
-			return fmt.Errorf("stopped after 10 redirects")
-		}
-		return nil
-	},
-}
 
 var ShortExtractor = &models.Extractor{
 	Name:       "Reddit (Short)",
@@ -44,7 +37,7 @@ var ShortExtractor = &models.Extractor{
 			req.AddCookie(cookie)
 		}
 
-		res, err := HTTPClient.Do(req)
+		res, err := HTTPSession.Do(req)
 		if err != nil {
 			return nil, fmt.Errorf("failed to send request: %w", err)
 		}
@@ -236,7 +229,7 @@ func GetRedditData(host string, slug string) (RedditResponse, error) {
 		req.AddCookie(cookie)
 	}
 
-	res, err := HTTPClient.Do(req)
+	res, err := HTTPSession.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

ext/tiktok/main.go

@@ -1,16 +1,12 @@
 package tiktok
 
 import (
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"regexp"
 
-	"github.com/quic-go/quic-go"
-	"github.com/quic-go/quic-go/http3"
-
 	"govd/enums"
 	"govd/models"
 	"govd/util"
@@ -27,17 +23,7 @@ const (
 	appUserAgent       = packageID + " (Linux; U; Android 13; en_US; Pixel 7; Build/TD1A.220804.031; Cronet/58.0.2991.0)"
 )
 
-var HTTPClient = &http.Client{
+var HTTPSession = util.NewHTTPSession()
-	Transport: &http3.Transport{
-		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true,
-		},
-		QUICConfig: &quic.Config{
-			MaxIncomingStreams: -1,
-			EnableDatagrams:    true,
-		},
-	},
-}
 
 var VMExtractor = &models.Extractor{
 	Name:       "TikTok VM",
@@ -161,7 +147,7 @@ func GetVideoAPI(awemeID string) (*AwemeDetails, error) {
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("X-Argus", "")
 
-	resp, err := HTTPClient.Do(req)
+	resp, err := HTTPSession.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

ext/twitter/main.go

@@ -17,7 +17,7 @@ const (
 	apiEndpoint = "https://x.com/i/api/graphql/zZXycP0V6H7m-2r0mOnFcA/TweetDetail"
 )
 
-var HTTPClient = &http.Client{}
+var HTTPSession = util.NewHTTPSession()
 
 var ShortExtractor = &models.Extractor{
 	Name:       "Twitter (Short)",
@@ -33,7 +33,7 @@ var ShortExtractor = &models.Extractor{
 			return nil, fmt.Errorf("failed to create req: %w", err)
 		}
 		req.Header.Set("User-Agent", util.ChromeUA)
-		res, err := HTTPClient.Do(req)
+		res, err := HTTPSession.Do(req)
 		if err != nil {
 			return nil, fmt.Errorf("failed to send request: %w", err)
 		}
@@ -151,7 +151,7 @@ func GetTweetAPI(tweetID string) (*Tweet, error) {
 	}
 	req.URL.RawQuery = q.Encode()
 
-	resp, err := HTTPClient.Do(req)
+	resp, err := HTTPSession.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

util/download.go

@@ -45,7 +45,7 @@ func DownloadFile(
 			return "", ctx.Err()
 		default:
 			// create the download directory if it doesn't exist
-			if err := ensureDownloadDir(config.DownloadDir); err != nil {
+			if err := EnsureDownloadDir(config.DownloadDir); err != nil {
 				return "", err
 			}
 
@@ -78,7 +78,7 @@ func DownloadFileWithSegments(
 	if config == nil {
 		config = DefaultConfig()
 	}
-	if err := ensureDownloadDir(config.DownloadDir); err != nil {
+	if err := EnsureDownloadDir(config.DownloadDir); err != nil {
 		return "", err
 	}
 	tempDir := filepath.Join(config.DownloadDir, "segments_"+time.Now().Format("20060102_150405"))
@@ -151,10 +151,14 @@ func downloadInMemory(ctx context.Context, fileURL string, timeout time.Duration
 	return io.ReadAll(resp.Body)
 }
 
-func ensureDownloadDir(dir string) error {
+func EnsureDownloadDir(dir string) error {
-	if _, err := os.Stat(dir); os.IsNotExist(err) {
+	if _, err := os.Stat(dir); err != nil {
-		if err := os.MkdirAll(dir, 0755); err != nil {
+		if os.IsNotExist(err) {
-			return fmt.Errorf("failed to create downloads directory: %w", err)
+			if err := os.MkdirAll(dir, 0755); err != nil {
+				return fmt.Errorf("failed to create downloads directory: %w", err)
+			}
+		} else {
+			return fmt.Errorf("error accessing directory: %w", err)
 		}
 	}
 	return nil
@@ -453,7 +457,7 @@ func MergeSegmentFiles(
 		config = DefaultConfig()
 	}
 
-	if err := ensureDownloadDir(config.DownloadDir); err != nil {
+	if err := EnsureDownloadDir(config.DownloadDir); err != nil {
 		return "", err
 	}
 

util/http.go

@@ -5,8 +5,16 @@ import (
 	"time"
 )
 
-var httpSession = &http.Client{
+var httpSession = NewHTTPSession()
-	Timeout: 20 * time.Second,
+
+func NewHTTPSession() *http.Client {
+	session := &http.Client{
+		Timeout: 20 * time.Second,
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+		},
+	}
+	return session
 }
 
 func GetHTTPSession() *http.Client {