packages/govd
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

fix: better slices allocation

Browse source 
pre-allocate slices' size when possible when extracting media
This commit is contained in:
stefanodvx 2025-04-28 15:57:30 +02:00
parent 0d986d4573
commit b5c5801ec2
8 changed files with 65 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

18
ext/instagram/util.go
View file

@ -58,8 +58,6 @@ func ParseGQLMedia(
ctx *models.DownloadContext,
data *Media,
) ([]*models.Media, error) {
var mediaList []*models.Media
var caption string
if data.EdgeMediaToCaption != nil && len(data.EdgeMediaToCaption.Edges) > 0 {
caption = data.EdgeMediaToCaption.Edges[0].Node.Text
@ -84,8 +82,7 @@ func ParseGQLMedia(
Height: int64(data.Dimensions.Height),
})
mediaList = append(mediaList, media)
return []*models.Media{media}, nil
case "GraphImage", "XDTGraphImage":
media := ctx.Extractor.NewMedia(contentID, contentURL)
media.SetCaption(caption)
@ -96,12 +93,14 @@ func ParseGQLMedia(
URL: []string{data.DisplayURL},
})
mediaList = append(mediaList, media)
return []*models.Media{media}, nil
case "GraphSidecar", "XDTGraphSidecar":
if data.EdgeSidecarToChildren != nil && len(data.EdgeSidecarToChildren.Edges) > 0 {
for _, edge := range data.EdgeSidecarToChildren.Edges {
node := edge.Node
edges := data.EdgeSidecarToChildren.Edges
mediaList := make([]*models.Media, 0, len(edges))
for i := range edges {
node := edges[i].Node
media := ctx.Extractor.NewMedia(contentID, contentURL)
media.SetCaption(caption)
@ -128,10 +127,11 @@ func ParseGQLMedia(
mediaList = append(mediaList, media)
}
return mediaList, nil
}
}
return mediaList, nil
return nil, fmt.Errorf("unknown media type: %s", data.Typename)
}
func ParseEmbedGQL(

8
ext/ninegag/main.go
View file

@ -42,8 +42,6 @@ var Extractor = &models.Extractor{
}
func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
var mediaList []*models.Media
contentID := ctx.MatchedContentID
contentURL := ctx.MatchedContentURL
@ -84,12 +82,12 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
media.AddFormat(format)
}
}
if len(media.Formats) > 0 {
mediaList = append(mediaList, media)
return []*models.Media{media}, nil
}
return mediaList, nil
// no media found
return nil, nil
}
func GetPostData(postID string) (*Post, error) {

8
ext/pinterest/main.go
View file

@ -87,9 +87,9 @@ var Extractor = &models.Extractor{
func ExtractPinMedia(ctx *models.DownloadContext) ([]*models.Media, error) {
pinID := ctx.MatchedContentID
contentURL := ctx.MatchedContentURL
client := util.GetHTTPClient(ctx.Extractor.CodeName)
session := util.GetHTTPClient(ctx.Extractor.CodeName)
pinData, err := GetPinData(client, pinID)
pinData, err := GetPinData(session, pinID)
if err != nil {
return nil, err
}
@ -160,7 +160,7 @@ func ExtractPinMedia(ctx *models.DownloadContext) ([]*models.Media, error) {
}
func GetPinData(
client models.HTTPClient,
session models.HTTPClient,
pinID string,
) (*PinData, error) {
params := BuildPinRequestParams(pinID)
@ -179,7 +179,7 @@ func GetPinData(
// fix 403 error
req.Header.Set("X-Pinterest-PWS-Handler", "www/[username].js")
resp, err := client.Do(req)
resp, err := session.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}

66
ext/reddit/main.go
View file

@ -81,7 +81,7 @@ var Extractor = &models.Extractor{
}
func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
client := util.GetHTTPClient(ctx.Extractor.CodeName)
session := util.GetHTTPClient(ctx.Extractor.CodeName)
host := ctx.MatchedGroups["host"]
slug := ctx.MatchedGroups["slug"]
@ -89,7 +89,7 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
contentID := ctx.MatchedContentID
contentURL := ctx.MatchedContentURL
manifest, err := GetRedditData(client, host, slug, false)
manifest, err := GetRedditData(session, host, slug, false)
if err != nil {
return nil, err
}
@ -102,16 +102,12 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
title := data.Title
isNsfw := data.Over18
var mediaList []*models.Media
if !data.IsVideo {
// check for single photo
if data.Preview != nil && len(data.Preview.Images) > 0 {
media := ctx.Extractor.NewMedia(contentID, contentURL)
media.SetCaption(title)
if isNsfw {
media.NSFW = true
}
media.NSFW = isNsfw
image := data.Preview.Images[0]
@ -130,8 +126,7 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
media.AddFormat(format)
}
mediaList = append(mediaList, media)
return mediaList, nil
return []*models.Media{media}, nil
}
// check for MP4 variant (animated GIF)
@ -145,8 +140,7 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
Thumbnail: []string{util.FixURL(image.Source.URL)},
})
mediaList = append(mediaList, media)
return mediaList, nil
return []*models.Media{media}, nil
}
// regular photo
@ -156,39 +150,45 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
URL: []string{util.FixURL(image.Source.URL)},
})
mediaList = append(mediaList, media)
return mediaList, nil
return []*models.Media{media}, nil
}
// check for gallery/collection
if len(data.MediaMetadata) > 0 {
for key, obj := range data.MediaMetadata {
if obj.E == "Image" {
media := ctx.Extractor.NewMedia(key, contentURL)
media.SetCaption(title)
if isNsfw {
media.NSFW = true
}
// known issue: collection is unordered
collection := data.MediaMetadata
mediaList := make([]*models.Media, 0, len(collection))
for _, obj := range collection {
media := ctx.Extractor.NewMedia(contentID, contentURL)
media.SetCaption(title)
media.NSFW = isNsfw
switch obj.Type {
case "Image":
media.AddFormat(&models.MediaFormat{
FormatID: "photo",
Type: enums.MediaTypePhoto,
URL: []string{util.FixURL(obj.S.U)},
URL: []string{util.FixURL(obj.Media.URL)},
})
case "AnimatedImage":
media.AddFormat(&models.MediaFormat{
FormatID: "video",
Type: enums.MediaTypeVideo,
VideoCodec: enums.MediaCodecAVC,
AudioCodec: enums.MediaCodecAAC,
URL: []string{util.FixURL(obj.Media.MP4)},
})
}
mediaList = append(mediaList, media)
}
}
return mediaList, nil
}
} else {
// video
media := ctx.Extractor.NewMedia(contentID, contentURL)
media.SetCaption(title)
if isNsfw {
media.NSFW = true
}
media.NSFW = isNsfw
var redditVideo *Video
@ -218,16 +218,16 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
media.AddFormat(format)
}
mediaList = append(mediaList, media)
return mediaList, nil
return []*models.Media{media}, nil
}
}
return mediaList, nil
// no media found
return nil, nil
}
func GetRedditData(
client models.HTTPClient,
session models.HTTPClient,
host string,
slug string,
raise bool,
@ -248,7 +248,7 @@ func GetRedditData(
req.AddCookie(cookie)
}
res, err := client.Do(req)
res, err := session.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to send request: %w", err)
}
@ -264,7 +264,7 @@ func GetRedditData(
altHost = "www.reddit.com"
}
return GetRedditData(client, altHost, slug, true)
return GetRedditData(session, altHost, slug, true)
}
var response Response

11
ext/reddit/models.go
View file

@ -71,10 +71,11 @@ type VideoPreview struct {
type MediaMetadata struct {
Status string `json:"status"`
E string `json:"e"`
S struct {
U string `json:"u"`
X int64 `json:"x"`
Y int64 `json:"y"`
Type string `json:"e"`
Media struct {
MP4 string `json:"mp4"`
URL string `json:"u"`
Width int64 `json:"x"`
Height int64 `json:"y"`
} `json:"s"`
}

7
ext/redgifs/main.go
View file

@ -48,7 +48,6 @@ var Extractor = &models.Extractor{
}
func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
var mediaList []*models.Media
client := util.GetHTTPClient(ctx.Extractor.CodeName)
response, err := GetVideo(
@ -109,11 +108,7 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
}
}
if len(media.Formats) > 0 {
mediaList = append(mediaList, media)
}
return mediaList, nil
return []*models.Media{media}, nil
}
func GetVideo(

8
ext/redgifs/util.go
View file

@ -12,22 +12,22 @@ import (
var accessToken *Token
func GetAccessToken(client models.HTTPClient) (*Token, error) {
func GetAccessToken(session models.HTTPClient) (*Token, error) {
if accessToken == nil || time.Now().Unix() >= accessToken.ExpiresIn {
if err := RefreshAccessToken(client); err != nil {
if err := RefreshAccessToken(session); err != nil {
return nil, err
}
}
return accessToken, nil
}
func RefreshAccessToken(client models.HTTPClient) error {
func RefreshAccessToken(session models.HTTPClient) error {
req, err := http.NewRequest(http.MethodGet, tokenEndpoint, nil)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", util.ChromeUA)
res, err := client.Do(req)
res, err := session.Do(req)
if err != nil {
return fmt.Errorf("failed to send request: %w", err)
}

10
ext/tiktok/main.go
View file

@ -78,8 +78,6 @@ var Extractor = &models.Extractor{
func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
client := util.GetHTTPClient(ctx.Extractor.CodeName)
var mediaList []*models.Media
details, err := GetVideoAPI(
client, ctx.MatchedContentID)
if err != nil {
@ -119,10 +117,12 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
}
media.AddFormat(format)
}
mediaList = append(mediaList, media)
return []*models.Media{media}, nil
} else {
images := details.ImagePostInfo.Images
for _, image := range images {
mediaList := make([]*models.Media, 0, len(images))
for i := range images {
image := images[i]
media := ctx.Extractor.NewMedia(
ctx.MatchedContentID,
ctx.MatchedContentURL,
@ -135,9 +135,9 @@ func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
})
mediaList = append(mediaList, media)
}
}
return mediaList, nil
}
}
func GetVideoAPI(
client models.HTTPClient,