diff --git a/config/main.go b/config/main.go index fe3a129..651d2a2 100644 --- a/config/main.go +++ b/config/main.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "maps" "os" "govd/models" @@ -21,17 +22,15 @@ func LoadExtractorConfigs() error { } data, err := os.ReadFile(configPath) if err != nil { - return fmt.Errorf("errore nella lettura del file di configurazione: %w", err) + return fmt.Errorf("failed reading config file: %w", err) } var rawConfig map[string]*models.ExtractorConfig if err := yaml.Unmarshal(data, &rawConfig); err != nil { - return fmt.Errorf("errore nella decodifica del file YAML: %w", err) - } - for codeName, config := range rawConfig { - extractorConfigs[codeName] = config + return fmt.Errorf("failed parsing config file: %w", err) } + maps.Copy(extractorConfigs, rawConfig) return nil } diff --git a/ext/instagram/main.go b/ext/instagram/main.go index cecfbbc..e911fe3 100644 --- a/ext/instagram/main.go +++ b/ext/instagram/main.go @@ -37,7 +37,6 @@ var Extractor = &models.Extractor{ URLPattern: regexp.MustCompile(`https:\/\/(www\.)?instagram\.com\/(reel|p|tv)\/(?P[a-zA-Z0-9_-]+)`), Host: instagramHost, IsRedirect: false, - Client: util.GetHTTPSession("instagram"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { mediaList, err := MediaListFromAPI(ctx, false) @@ -55,7 +54,6 @@ var StoriesExtractor = &models.Extractor{ URLPattern: regexp.MustCompile(`https:\/\/(www\.)?instagram\.com\/stories\/[a-zA-Z0-9._]+\/(?P\d+)`), Host: instagramHost, IsRedirect: false, - Client: util.GetHTTPSession("instagram_stories"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { mediaList, err := MediaListFromAPI(ctx, true) @@ -73,9 +71,9 @@ var ShareURLExtractor = &models.Extractor{ URLPattern: regexp.MustCompile(`https?:\/\/(www\.)?instagram\.com\/share\/((reels?|video|s|p)\/)?(?P[^\/\?]+)`), Host: instagramHost, IsRedirect: true, - Client: util.GetHTTPSession("instagram_share"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { + client := util.GetHTTPSession(ctx.Extractor.CodeName) req, err := http.NewRequest( http.MethodGet, ctx.MatchedContentURL, @@ -84,7 +82,7 @@ var ShareURLExtractor = &models.Extractor{ if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } - resp, err := ctx.Extractor.Client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -100,15 +98,17 @@ func MediaListFromAPI( ctx *models.DownloadContext, stories bool, ) ([]*models.Media, error) { + client := util.GetHTTPSession(ctx.Extractor.CodeName) + var mediaList []*models.Media postURL := ctx.MatchedContentURL - details, err := GetVideoAPI(ctx, postURL) + details, err := GetVideoAPI(client, postURL) if err != nil { return nil, fmt.Errorf("failed to get post: %w", err) } var caption string if !stories { - caption, err = GetPostCaption(ctx, postURL) + caption, err = GetPostCaption(client, postURL) if err != nil { return nil, fmt.Errorf("failed to get caption: %w", err) } @@ -157,7 +157,7 @@ func MediaListFromAPI( } func GetVideoAPI( - ctx *models.DownloadContext, + client models.HTTPClient, contentURL string, ) (*IGramResponse, error) { apiURL := fmt.Sprintf( @@ -175,7 +175,7 @@ func GetVideoAPI( req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", util.ChromeUA) - resp, err := ctx.Extractor.Client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/instagram/util.go b/ext/instagram/util.go index fce58dd..16ad34f 100644 --- a/ext/instagram/util.go +++ b/ext/instagram/util.go @@ -96,7 +96,7 @@ func GetCDNURL(contentURL string) (string, error) { } func GetPostCaption( - ctx *models.DownloadContext, + client models.HTTPClient, postURL string, ) (string, error) { req, err := http.NewRequest( @@ -122,7 +122,7 @@ func GetPostCaption( req.Header.Set("Cache-Control", "no-cache") req.Header.Set("TE", "trailers") - resp, err := ctx.Extractor.Client.Do(req) + resp, err := client.Do(req) if err != nil { return "", fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/pinterest/main.go b/ext/pinterest/main.go index c68158d..b7d6612 100644 --- a/ext/pinterest/main.go +++ b/ext/pinterest/main.go @@ -44,7 +44,6 @@ var ShortExtractor = &models.Extractor{ return domains }(), IsRedirect: true, - Client: util.GetHTTPSession("pinterest_short"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { shortURL := fmt.Sprintf(shortenerAPIFormat, ctx.MatchedContentID) @@ -72,7 +71,6 @@ var Extractor = &models.Extractor{ } return domains }(), - Client: util.GetHTTPSession("pinterest"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { media, err := ExtractPinMedia(ctx) @@ -88,8 +86,9 @@ var Extractor = &models.Extractor{ func ExtractPinMedia(ctx *models.DownloadContext) ([]*models.Media, error) { pinID := ctx.MatchedContentID contentURL := ctx.MatchedContentURL + client := util.GetHTTPSession(ctx.Extractor.CodeName) - pinData, err := GetPinData(ctx, pinID) + pinData, err := GetPinData(client, pinID) if err != nil { return nil, err } @@ -160,7 +159,7 @@ func ExtractPinMedia(ctx *models.DownloadContext) ([]*models.Media, error) { } func GetPinData( - ctx *models.DownloadContext, + client models.HTTPClient, pinID string, ) (*PinData, error) { params := BuildPinRequestParams(pinID) @@ -179,7 +178,7 @@ func GetPinData( // fix 403 error req.Header.Set("X-Pinterest-PWS-Handler", "www/[username].js") - resp, err := ctx.Extractor.Client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/reddit/main.go b/ext/reddit/main.go index 6c305f5..938de6f 100644 --- a/ext/reddit/main.go +++ b/ext/reddit/main.go @@ -29,9 +29,9 @@ var ShortExtractor = &models.Extractor{ URLPattern: regexp.MustCompile(`https?://(?P(?:\w+\.)?reddit(?:media)?\.com)/(?P(?:(?:r|user)/[^/]+/)?s/(?P[^/?#&]+))`), Host: baseHost, IsRedirect: true, - Client: util.GetHTTPSession("reddit_short"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { + client := util.GetHTTPSession(ctx.Extractor.CodeName) req, err := http.NewRequest(http.MethodGet, ctx.MatchedContentURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) @@ -46,7 +46,7 @@ var ShortExtractor = &models.Extractor{ req.AddCookie(cookie) } - res, err := ctx.Extractor.Client.Do(req) + res, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -67,7 +67,6 @@ var Extractor = &models.Extractor{ Category: enums.ExtractorCategorySocial, URLPattern: regexp.MustCompile(`https?://(?P(?:\w+\.)?reddit(?:media)?\.com)/(?P(?:(?:r|user)/[^/]+/)?comments/(?P[^/?#&]+))`), Host: baseHost, - Client: util.GetHTTPSession("reddit"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { mediaList, err := MediaListFromAPI(ctx) @@ -81,13 +80,15 @@ var Extractor = &models.Extractor{ } func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { + client := util.GetHTTPSession(ctx.Extractor.CodeName) + host := ctx.MatchedGroups["host"] slug := ctx.MatchedGroups["slug"] contentID := ctx.MatchedContentID contentURL := ctx.MatchedContentURL - manifest, err := GetRedditData(ctx, host, slug) + manifest, err := GetRedditData(client, host, slug) if err != nil { return nil, err } @@ -224,7 +225,7 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { } func GetRedditData( - ctx *models.DownloadContext, + client models.HTTPClient, host string, slug string, ) (RedditResponse, error) { @@ -244,7 +245,7 @@ func GetRedditData( req.AddCookie(cookie) } - res, err := ctx.Extractor.Client.Do(req) + res, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -257,7 +258,7 @@ func GetRedditData( altHost = "www.reddit.com" } - return GetRedditData(ctx, altHost, slug) + return GetRedditData(client, altHost, slug) } var response RedditResponse diff --git a/ext/redgifs/main.go b/ext/redgifs/main.go index 780ed39..30b3e17 100644 --- a/ext/redgifs/main.go +++ b/ext/redgifs/main.go @@ -35,7 +35,6 @@ var Extractor = &models.Extractor{ "redgifs.com", "thumbs2.redgifs.com", }, - Client: util.GetHTTPSession("redgifs"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { mediaList, err := MediaListFromAPI(ctx) @@ -50,8 +49,10 @@ var Extractor = &models.Extractor{ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { var mediaList []*models.Media + client := util.GetHTTPSession(ctx.Extractor.CodeName) - response, err := GetVideo(ctx) + response, err := GetVideo( + client, ctx.MatchedContentID) if err != nil { return nil, fmt.Errorf("failed to get from api: %w", err) } @@ -115,14 +116,16 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { return mediaList, nil } -func GetVideo(ctx *models.DownloadContext) (*Response, error) { - videoID := ctx.MatchedContentID +func GetVideo( + client models.HTTPClient, + videoID string, +) (*Response, error) { url := videoEndpoint + videoID + "?views=true" req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } - token, err := GetAccessToken(ctx) + token, err := GetAccessToken(client) if err != nil { return nil, fmt.Errorf("failed to get access token: %w", err) } @@ -132,7 +135,7 @@ func GetVideo(ctx *models.DownloadContext) (*Response, error) { for k, v := range baseApiHeaders { req.Header.Set(k, v) } - res, err := ctx.Extractor.Client.Do(req) + res, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/redgifs/util.go b/ext/redgifs/util.go index 7765e3d..2bf186b 100644 --- a/ext/redgifs/util.go +++ b/ext/redgifs/util.go @@ -12,22 +12,22 @@ import ( var accessToken *Token -func GetAccessToken(ctx *models.DownloadContext) (*Token, error) { +func GetAccessToken(client models.HTTPClient) (*Token, error) { if accessToken == nil || time.Now().Unix() >= accessToken.ExpiresIn { - if err := RefreshAccessToken(ctx); err != nil { + if err := RefreshAccessToken(client); err != nil { return nil, err } } return accessToken, nil } -func RefreshAccessToken(ctx *models.DownloadContext) error { +func RefreshAccessToken(client models.HTTPClient) error { req, err := http.NewRequest(http.MethodGet, tokenEndpoint, nil) if err != nil { return fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", util.ChromeUA) - res, err := ctx.Extractor.Client.Do(req) + res, err := client.Do(req) if err != nil { return fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/tiktok/main.go b/ext/tiktok/main.go index 29b0d9d..0d802d4 100644 --- a/ext/tiktok/main.go +++ b/ext/tiktok/main.go @@ -44,7 +44,6 @@ var VMExtractor = &models.Extractor{ URLPattern: regexp.MustCompile(`https:\/\/((?:vm|vt|www)\.)?(vx)?tiktok\.com\/(?:t\/)?(?P[a-zA-Z0-9]+)`), Host: baseHost, IsRedirect: true, - Client: util.GetHTTPSession("tiktokvm"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { location, err := util.GetLocationURL(ctx.MatchedContentURL, "") @@ -64,7 +63,6 @@ var Extractor = &models.Extractor{ Category: enums.ExtractorCategorySocial, URLPattern: regexp.MustCompile(`https?:\/\/((www|m)\.)?(vx)?tiktok\.com\/((?:embed|@[\w\.-]+)\/)?(v(ideo)?|p(hoto)?)\/(?P[0-9]+)`), Host: baseHost, - Client: util.GetHTTPSession("tiktok"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { mediaList, err := MediaListFromAPI(ctx) @@ -78,9 +76,11 @@ var Extractor = &models.Extractor{ } func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { + client := util.GetHTTPSession(ctx.Extractor.CodeName) var mediaList []*models.Media - details, err := GetVideoAPI(ctx) + details, err := GetVideoAPI( + client, ctx.MatchedContentID) if err != nil { return nil, fmt.Errorf("failed to get from api: %w", err) } @@ -138,8 +138,10 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { return mediaList, nil } -func GetVideoAPI(ctx *models.DownloadContext) (*AwemeDetails, error) { - awemeID := ctx.MatchedContentID +func GetVideoAPI( + client models.HTTPClient, + awemeID string, +) (*AwemeDetails, error) { apiURL := fmt.Sprintf( "https://%s/aweme/v1/multi/aweme/detail/", apiHostname, @@ -163,7 +165,7 @@ func GetVideoAPI(ctx *models.DownloadContext) (*AwemeDetails, error) { req.Header.Set("Accept", "application/json") req.Header.Set("X-Argus", "") - resp, err := ctx.Extractor.Client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/ext/twitter/main.go b/ext/twitter/main.go index b794cc7..2dceefd 100644 --- a/ext/twitter/main.go +++ b/ext/twitter/main.go @@ -26,15 +26,15 @@ var ShortExtractor = &models.Extractor{ URLPattern: regexp.MustCompile(`https?://t\.co/(?P\w+)`), Host: []string{"t.co"}, IsRedirect: true, - Client: util.GetHTTPSession("twitter_short"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { + client := util.GetHTTPSession(ctx.Extractor.CodeName) req, err := http.NewRequest(http.MethodGet, ctx.MatchedContentURL, nil) if err != nil { return nil, fmt.Errorf("failed to create req: %w", err) } req.Header.Set("User-Agent", util.ChromeUA) - res, err := ctx.Extractor.Client.Do(req) + res, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } @@ -65,7 +65,6 @@ var Extractor = &models.Extractor{ "vxx.com", "vxtwitter.com", }, - Client: util.GetHTTPSession("twitter"), Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) { mediaList, err := MediaListFromAPI(ctx) @@ -80,8 +79,10 @@ var Extractor = &models.Extractor{ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { var mediaList []*models.Media + client := util.GetHTTPSession(ctx.Extractor.CodeName) - tweetData, err := GetTweetAPI(ctx) + tweetData, err := GetTweetAPI( + client, ctx.MatchedContentID) if err != nil { return nil, fmt.Errorf("failed to get tweet data: %w", err) } @@ -129,8 +130,10 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) { return mediaList, nil } -func GetTweetAPI(ctx *models.DownloadContext) (*Tweet, error) { - tweetID := ctx.MatchedContentID +func GetTweetAPI( + client models.HTTPClient, + tweetID string, +) (*Tweet, error) { cookies, err := util.ParseCookieFile("twitter.txt") if err != nil { return nil, fmt.Errorf("failed to get cookies: %w", err) @@ -160,7 +163,7 @@ func GetTweetAPI(ctx *models.DownloadContext) (*Tweet, error) { } req.URL.RawQuery = q.Encode() - resp, err := ctx.Extractor.Client.Do(req) + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to send request: %w", err) } diff --git a/models/ext.go b/models/ext.go index 81984f5..44bfe55 100644 --- a/models/ext.go +++ b/models/ext.go @@ -14,7 +14,6 @@ type Extractor struct { Host []string IsDRM bool IsRedirect bool - Client HTTPClient Run func(*DownloadContext) (*ExtractorResponse, error) }