packages/govd
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

env: supports for http and socks5 proxies

Browse source
This commit is contained in:
stefanodvx 2025-04-14 21:50:49 +02:00
parent 386c3991cd
commit 10c113f400
10 changed files with 51 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

5
.env.example
View file

@ -8,3 +8,8 @@ DB_USER=govd
DB_PASSWORD=password
REPO_URL=https://github.com/govdbot/govd
# proxy
HTTP_PROXY=
HTTPS_PROXY=
NO_PROXY=

10
bot/middleware.go
View file

@ -5,6 +5,7 @@ import (
"encoding/json"
"log"
"net/http"
"net/url"
"os"
"strings"
"time"
@ -45,7 +46,14 @@ func NewBotClient() BotClient {
}
return BotClient{
BotClient: &gotgbot.BaseBotClient{
Client: http.Client{},
Client: http.Client{
Transport: &http.Transport{
// avoid using proxy for telegram
Proxy: func(r *http.Request) (*url.URL, error) {
return nil, nil
},
},
},
UseTestEnvironment: false,
DefaultRequestOpts: &gotgbot.RequestOpts{
Timeout: 10 * time.Minute,

20
ext/instagram/main.go
View file

@ -1,7 +1,6 @@
package instagram
import (
"crypto/tls"
"fmt"
"govd/enums"
"govd/models"
@ -9,9 +8,6 @@ import (
"io"
"net/http"
"regexp"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
)
// as a public service, we can't use the official API
@ -46,17 +42,7 @@ var igHeaders = map[string]string{
"User-Agent": util.ChromeUA,
}
var HTTPClient = &http.Client{
Transport: &http3.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
QUICConfig: &quic.Config{
MaxIncomingStreams: -1,
EnableDatagrams: true,
},
},
}
var HTTPSession = util.NewHTTPSession()
var Extractor = &models.Extractor{
Name: "Instagram",
@ -110,7 +96,7 @@ var ShareURLExtractor = &models.Extractor{
for k, v := range igHeaders {
req.Header.Set(k, v)
}
resp, err := HTTPClient.Do(req)
resp, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
@ -197,7 +183,7 @@ func GetVideoAPI(contentURL string) (*IGramResponse, error) {
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", util.ChromeUA)
resp, err := HTTPClient.Do(req)
resp, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}

2
ext/instagram/util.go
View file

@ -115,7 +115,7 @@ func GetPostCaption(
req.Header.Set("Cache-Control", "no-cache")
req.Header.Set("TE", "trailers")
resp, err := HTTPClient.Do(req)
resp, err := HTTPSession.Do(req)
if err != nil {
return "", fmt.Errorf("failed to send request: %w", err)
}

5
ext/pinterest/main.go
View file

@ -17,6 +17,8 @@ const (
shortenerAPIFormat = "https://api.pinterest.com/url_shortener/%s/redirect/"
)
var HTTPSession = util.NewHTTPSession()
var ShortExtractor = &models.Extractor{
Name: "Pinterest (Short)",
CodeName: "pinterest:short",
@ -146,8 +148,7 @@ func GetPinData(pinID string) (*PinData, error) {
// fix 403 error
req.Header.Set("X-Pinterest-PWS-Handler", "www/[username].js")
client := &http.Client{}
resp, err := client.Do(req)
resp, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}

13
ext/reddit/main.go
View file

@ -12,14 +12,7 @@ import (
"govd/util"
)
var HTTPClient = &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
return fmt.Errorf("stopped after 10 redirects")
}
return nil
},
}
var HTTPSession = util.NewHTTPSession()
var ShortExtractor = &models.Extractor{
Name: "Reddit (Short)",
@ -44,7 +37,7 @@ var ShortExtractor = &models.Extractor{
req.AddCookie(cookie)
}
res, err := HTTPClient.Do(req)
res, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
@ -236,7 +229,7 @@ func GetRedditData(host string, slug string) (RedditResponse, error) {
req.AddCookie(cookie)
}
res, err := HTTPClient.Do(req)
res, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}

18
ext/tiktok/main.go
View file

@ -1,16 +1,12 @@
package tiktok
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net/http"
"regexp"
"github.com/quic-go/quic-go"
"github.com/quic-go/quic-go/http3"
"govd/enums"
"govd/models"
"govd/util"
@ -27,17 +23,7 @@ const (
appUserAgent = packageID + " (Linux; U; Android 13; en_US; Pixel 7; Build/TD1A.220804.031; Cronet/58.0.2991.0)"
)
var HTTPClient = &http.Client{
Transport: &http3.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
QUICConfig: &quic.Config{
MaxIncomingStreams: -1,
EnableDatagrams: true,
},
},
}
var HTTPSession = util.NewHTTPSession()
var VMExtractor = &models.Extractor{
Name: "TikTok VM",
@ -161,7 +147,7 @@ func GetVideoAPI(awemeID string) (*AwemeDetails, error) {
req.Header.Set("Accept", "application/json")
req.Header.Set("X-Argus", "")
resp, err := HTTPClient.Do(req)
resp, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}

6
ext/twitter/main.go
View file

@ -17,7 +17,7 @@ const (
apiEndpoint = "https://x.com/i/api/graphql/zZXycP0V6H7m-2r0mOnFcA/TweetDetail"
)
var HTTPClient = &http.Client{}
var HTTPSession = util.NewHTTPSession()
var ShortExtractor = &models.Extractor{
Name: "Twitter (Short)",
@ -33,7 +33,7 @@ var ShortExtractor = &models.Extractor{
return nil, fmt.Errorf("failed to create req: %w", err)
}
req.Header.Set("User-Agent", util.ChromeUA)
res, err := HTTPClient.Do(req)
res, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
@ -151,7 +151,7 @@ func GetTweetAPI(tweetID string) (*Tweet, error) {
}
req.URL.RawQuery = q.Encode()
resp, err := HTTPClient.Do(req)
resp, err := HTTPSession.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}

18
util/download.go
View file

@ -45,7 +45,7 @@ func DownloadFile(
return "", ctx.Err()
default:
// create the download directory if it doesn't exist
if err := ensureDownloadDir(config.DownloadDir); err != nil {
if err := EnsureDownloadDir(config.DownloadDir); err != nil {
return "", err
}
@ -78,7 +78,7 @@ func DownloadFileWithSegments(
if config == nil {
config = DefaultConfig()
}
if err := ensureDownloadDir(config.DownloadDir); err != nil {
if err := EnsureDownloadDir(config.DownloadDir); err != nil {
return "", err
}
tempDir := filepath.Join(config.DownloadDir, "segments_"+time.Now().Format("20060102_150405"))
@ -151,10 +151,14 @@ func downloadInMemory(ctx context.Context, fileURL string, timeout time.Duration
return io.ReadAll(resp.Body)
}
func ensureDownloadDir(dir string) error {
if _, err := os.Stat(dir); os.IsNotExist(err) {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create downloads directory: %w", err)
func EnsureDownloadDir(dir string) error {
if _, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("failed to create downloads directory: %w", err)
}
} else {
return fmt.Errorf("error accessing directory: %w", err)
}
}
return nil
@ -453,7 +457,7 @@ func MergeSegmentFiles(
config = DefaultConfig()
}
if err := ensureDownloadDir(config.DownloadDir); err != nil {
if err := EnsureDownloadDir(config.DownloadDir); err != nil {
return "", err
}

12
util/http.go
View file

@ -5,8 +5,16 @@ import (
"time"
)
var httpSession = &http.Client{
Timeout: 20 * time.Second,
var httpSession = NewHTTPSession()
func NewHTTPSession() *http.Client {
session := &http.Client{
Timeout: 20 * time.Second,
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
},
}
return session
}
func GetHTTPSession() *http.Client {