diff --git a/README.md b/README.md index 23f2ad3..c22a759 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ you can configure specific extractors options with `ext-cfg.yaml` file. document # proxying there are two types of proxying available: http and edge. - **http proxy**: this is a standard http proxy that can be used to route requests through a proxy server. you can set the `HTTP_PROXY` and `HTTPS_PROXY` environment variables to use this feature. (SOCKS5 is supported too) -- **edge proxy**: this is a custom proxy that is used to route requests through a specific url. you can set the `EDGE_PROXY_URL` environment variable to use this feature. this is useful for routing requests through a specific server or service. however, this feature is not totally implemented yet. +- **edge proxy**: this is a custom proxy that is used to route requests through a specific url. currenrly, you can only set this proxy with `ext-cfg.yaml` file. this is useful for routing requests through a specific server or service. however, this feature is not totally implemented yet. **note:** by settings `NO_PROXY` environment variable, you can specify domains that should not be proxied. diff --git a/ext/instagram/main.go b/ext/instagram/main.go index 3c2a7fd..21755bb 100644 --- a/ext/instagram/main.go +++ b/ext/instagram/main.go @@ -74,22 +74,12 @@ var ShareURLExtractor = &models.Extractor{ Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { client := util.GetHTTPClient(ctx.Extractor.CodeName) - req, err := http.NewRequest( - http.MethodGet, - ctx.MatchedContentURL, - nil, - ) + redirectURL, err := util.GetLocationURL(client, ctx.MatchedContentURL, "") if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) + return nil, fmt.Errorf("failed to get url location: %w", err) } - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - return &models.ExtractorResponse{ - URL: resp.Request.URL.String(), + URL: redirectURL, }, nil }, } diff --git a/ext/pinterest/main.go b/ext/pinterest/main.go index e496632..6d0b55b 100644 --- a/ext/pinterest/main.go +++ b/ext/pinterest/main.go @@ -46,8 +46,9 @@ var ShortExtractor = &models.Extractor{ IsRedirect: true, Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { + client := util.GetHTTPClient(ctx.Extractor.CodeName) shortURL := fmt.Sprintf(shortenerAPIFormat, ctx.MatchedContentID) - location, err := util.GetLocationURL(shortURL, "") + location, err := util.GetLocationURL(client, shortURL, "") if err != nil { return nil, fmt.Errorf("failed to get real url: %w", err) } diff --git a/ext/tiktok/main.go b/ext/tiktok/main.go index b934f56..682f42b 100644 --- a/ext/tiktok/main.go +++ b/ext/tiktok/main.go @@ -46,12 +46,13 @@ var VMExtractor = &models.Extractor{ IsRedirect: true, Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { - location, err := util.GetLocationURL(ctx.MatchedContentURL, "") + client := util.GetHTTPClient(ctx.Extractor.CodeName) + redirectURL, err := util.GetLocationURL(client, ctx.MatchedContentURL, "") if err != nil { return nil, fmt.Errorf("failed to get url location: %w", err) } return &models.ExtractorResponse{ - URL: location, + URL: redirectURL, }, nil }, } diff --git a/util/misc.go b/util/misc.go index d6b2110..caa67ab 100644 --- a/util/misc.go +++ b/util/misc.go @@ -2,6 +2,7 @@ package util import ( "fmt" + "govd/models" "net/http" "os" "os/exec" @@ -18,9 +19,13 @@ import ( var cookiesCache = make(map[string][]*http.Cookie) func GetLocationURL( + client models.HTTPClient, url string, userAgent string, ) (string, error) { + if client == nil { + client = GetDefaultHTTPClient() + } req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return "", fmt.Errorf("failed to create request: %w", err) @@ -29,8 +34,7 @@ func GetLocationURL( userAgent = ChromeUA } req.Header.Set("User-Agent", userAgent) - session := GetDefaultHTTPClient() - resp, err := session.Do(req) + resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("failed to send request: %w", err) }