Init

.env.example

@@ -0,0 +1,10 @@
+BOT_API_URL=https://api.telegram.org
+BOT_TOKEN=12345678:ABC-DEF1234ghIkl-zyx57W2P0s
+
+DB_HOST=localhost
+DB_PORT=3306
+DB_NAME=govd
+DB_USER=govd
+DB_PASSWORD=password
+
+REPO_URL=https://github.com/govdbot/govd

.gitignore

@@ -0,0 +1,16 @@
+*.exe
+*.json
+*.txt
+*.py
+*.html
+
+old/
+
+.env
+.idea/
+
+downloads
+
+govd
+
+.DS_Store

LICENSE

@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

README.md

@@ -0,0 +1,53 @@
+# govd
+
+a telegram bot for downloading media from various platforms
+
+this project was born after the discontinuation of a highly popular bot known as UVD, and draws significant inspiration from [yt-dlp](https://github.com/yt-dlp/yt-dlp)
+
+- official instance: [@govd_bot](https://t.me/govd_bot)
+- support group: [govdsupport](https://t.me/govdsupport)
+
+## features
+
+- download media from various platforms
+- download videos, photos, and audio
+- inline mode support
+- group chat support with customizable settings
+- media caption support
+
+## dependencies
+
+- ffmpeg >= 6.1.1
+- libheif >= 1.19.7
+- pkg-config
+- mysql db
+
+
+## botapi
+
+to avoid limits on files, you should host your own telegram botapi. public bot instance is currently running under a botapi fork, [tdlight-telegram-bot-api](https://github.com/tdlight-team/tdlight-telegram-bot-api)
+
+## installation
+
+```bash
+git clone https://github.com/govdbot/govd.git
+cd govd
+# edit .env file with your bot token and database credentials
+sh build.sh
+```
+
+## cookies
+
+some extractors require cookies for download. to add your cookies, just insert a txt file in cookies folder (netscape format)
+
+## todo
+
+- [ ] add more extractors
+- [ ] switch to sonic json parser
+- [ ] switch to native libav
+- [ ] add tests
+- [ ] add dockerfile and compose
+- [ ] improve error handling
+- [ ] add support for telegram wehbhooks
+- [ ] switch to pgsql (?)
+- [ ] better API (?)

bot/core/default.go

@@ -0,0 +1,134 @@
+package core
+
+import (
+	"fmt"
+	"govd/database"
+	"govd/models"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+func HandleDefaultFormatDownload(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+) error {
+	storedMedias, err := database.GetDefaultMedias(
+		dlCtx.Extractor.CodeName,
+		dlCtx.MatchedContentID,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to get default medias: %w", err)
+	}
+
+	if len(storedMedias) > 0 {
+		return HandleDefaultStoredFormatDownload(
+			bot, ctx, dlCtx, storedMedias,
+		)
+	}
+
+	response, err := dlCtx.Extractor.Run(dlCtx)
+	if err != nil {
+		return fmt.Errorf("extractor fetch run failed: %w", err)
+	}
+
+	mediaList := response.MediaList
+	if len(mediaList) == 0 {
+		return fmt.Errorf("no media found for content ID: %s", dlCtx.MatchedContentID)
+	}
+
+	for i := range mediaList {
+		defaultFormat := mediaList[i].GetDefaultFormat()
+		if defaultFormat == nil {
+			return fmt.Errorf("no default format found for media at index %d", i)
+		}
+		if len(defaultFormat.URL) == 0 {
+			return fmt.Errorf("media format at index %d has no URL", i)
+		}
+		// ensure we can merge video and audio formats
+		ensureMergeFormats(mediaList[i], defaultFormat)
+		mediaList[i].Format = defaultFormat
+	}
+
+	medias, err := DownloadMedias(mediaList, nil)
+	if err != nil {
+		return fmt.Errorf("failed to download media list: %w", err)
+	}
+
+	if len(medias) == 0 {
+		return fmt.Errorf("no formats downloaded")
+	}
+
+	isCaptionEnabled := true
+	if dlCtx.GroupSettings != nil && !*dlCtx.GroupSettings.Captions {
+		isCaptionEnabled = false
+	}
+	messageCaption := FormatCaption(
+		mediaList[0],
+		isCaptionEnabled,
+	)
+
+	// plugins act as post-processing for the media.
+	// they are run after the media is downloaded
+	// and before it is sent to the user
+	// this allows for things like merging audio and video, etc.
+	for _, media := range medias {
+		for _, plugin := range media.Media.Format.Plugins {
+			err = plugin(media)
+			if err != nil {
+				return fmt.Errorf("failed to run plugin: %w", err)
+			}
+		}
+	}
+
+	_, err = SendMedias(
+		bot, ctx, dlCtx,
+		medias,
+		&models.SendMediaFormatsOptions{
+			Caption:  messageCaption,
+			IsStored: false,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to send formats: %w", err)
+	}
+
+	return nil
+}
+
+func HandleDefaultStoredFormatDownload(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+	storedMedias []*models.Media,
+) error {
+	isCaptionEnabled := true
+	if dlCtx.GroupSettings != nil && !*dlCtx.GroupSettings.Captions {
+		isCaptionEnabled = false
+	}
+	messageCaption := FormatCaption(
+		storedMedias[0],
+		isCaptionEnabled,
+	)
+	var formats []*models.DownloadedMedia
+	for _, media := range storedMedias {
+		formats = append(formats, &models.DownloadedMedia{
+			FilePath:          "",
+			ThumbnailFilePath: "",
+			Media:             media,
+		})
+	}
+	_, err := SendMedias(
+		bot, ctx, dlCtx,
+		formats,
+		&models.SendMediaFormatsOptions{
+			Caption:  messageCaption,
+			IsStored: true,
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to send media: %w", err)
+	}
+	return nil
+}

bot/core/download.go

@@ -0,0 +1,187 @@
+package core
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"sort"
+	"sync"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+)
+
+func downloadMediaItem(
+	ctx context.Context,
+	media *models.Media,
+	config *models.DownloadConfig,
+	idx int,
+) (*models.DownloadedMedia, error) {
+	if config == nil {
+		config = util.DefaultConfig()
+	}
+
+	format := media.Format
+	if format == nil {
+		return nil, fmt.Errorf("media format is nil")
+	}
+
+	fileName := format.GetFileName()
+	var filePath string
+	var thumbnailFilePath string
+
+	if format.Type != enums.MediaTypePhoto {
+		if len(format.Segments) == 0 {
+			path, err := util.DownloadFile(
+				ctx, format.URL,
+				fileName, config,
+			)
+			if err != nil {
+				return nil, fmt.Errorf("failed to download file: %w", err)
+			}
+			filePath = path
+		} else {
+			path, err := util.DownloadFileWithSegments(
+				ctx, format.Segments,
+				fileName, config,
+			)
+			if err != nil {
+				return nil, fmt.Errorf("failed to download segments: %w", err)
+			}
+			filePath = path
+		}
+
+		if format.Type == enums.MediaTypeVideo || format.Type == enums.MediaTypeAudio {
+			path, err := getFileThumbnail(format, filePath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get thumbnail: %w", err)
+			}
+			thumbnailFilePath = path
+		}
+
+		if format.Type == enums.MediaTypeVideo {
+			if format.Width == 0 || format.Height == 0 || format.Duration == 0 {
+				insertVideoInfo(format, filePath)
+			}
+		}
+	} else {
+		file, err := util.DownloadFileInMemory(ctx, format.URL, config)
+		if err != nil {
+			return nil, fmt.Errorf("failed to download image: %w", err)
+		}
+		path := filepath.Join(config.DownloadDir, fileName)
+		if err := util.ImgToJPEG(file, path); err != nil {
+			return nil, fmt.Errorf("failed to convert image: %w", err)
+		}
+		filePath = path
+	}
+
+	return &models.DownloadedMedia{
+		FilePath:          filePath,
+		ThumbnailFilePath: thumbnailFilePath,
+		Media:             media,
+		Index:             idx,
+	}, nil
+}
+
+func StartDownloadTask(
+	media *models.Media,
+	idx int,
+	config *models.DownloadConfig,
+) (*models.DownloadedMedia, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	return downloadMediaItem(ctx, media, config, idx)
+}
+
+func StartConcurrentDownload(
+	media *models.Media,
+	resultsChan chan<- models.DownloadedMedia,
+	config *models.DownloadConfig,
+	errChan chan<- error,
+	wg *sync.WaitGroup,
+	idx int,
+) {
+	defer wg.Done()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	result, err := downloadMediaItem(ctx, media, config, idx)
+	if err != nil {
+		errChan <- err
+		return
+	}
+
+	resultsChan <- *result
+}
+
+func DownloadMedia(
+	media *models.Media,
+	config *models.DownloadConfig,
+) (*models.DownloadedMedia, error) {
+	return StartDownloadTask(media, 0, config)
+}
+
+func DownloadMedias(
+	medias []*models.Media,
+	config *models.DownloadConfig,
+) ([]*models.DownloadedMedia, error) {
+	if len(medias) == 0 {
+		return []*models.DownloadedMedia{}, nil
+	}
+
+	if len(medias) == 1 {
+		result, err := DownloadMedia(medias[0], config)
+		if err != nil {
+			return nil, err
+		}
+		return []*models.DownloadedMedia{result}, nil
+	}
+
+	resultsChan := make(chan models.DownloadedMedia, len(medias))
+	errChan := make(chan error, len(medias))
+	var wg sync.WaitGroup
+
+	for idx, media := range medias {
+		wg.Add(1)
+		go StartConcurrentDownload(media, resultsChan, config, errChan, &wg, idx)
+	}
+
+	go func() {
+		wg.Wait()
+		close(resultsChan)
+		close(errChan)
+	}()
+
+	var results []*models.DownloadedMedia
+	var firstError error
+
+	select {
+	case err := <-errChan:
+		if err != nil {
+			firstError = err
+		}
+	default:
+		// no errors (yet)
+	}
+
+	for result := range resultsChan {
+		resultCopy := result // create a copy to avoid pointer issues
+		results = append(results, &resultCopy)
+	}
+
+	if firstError != nil {
+		return results, firstError
+	}
+
+	if len(results) > 1 {
+		sort.SliceStable(results, func(i, j int) bool {
+			return results[i].Index < results[j].Index
+		})
+	}
+
+	return results, nil
+}

bot/core/inline.go

@@ -0,0 +1,251 @@
+package core
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+
+	"govd/database"
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+var InlineTasks = make(map[string]*models.DownloadContext)
+
+func HandleInline(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+) error {
+	if dlCtx.Extractor.Type != enums.ExtractorTypeSingle {
+		return util.ErrNotImplemented
+	}
+	contentID := dlCtx.MatchedContentID
+	cached, err := database.GetDefaultMedias(
+		dlCtx.Extractor.CodeName,
+		contentID,
+	)
+	if err != nil {
+		return err
+	}
+	if len(cached) > 0 {
+		if len(cached) > 1 {
+			return util.ErrInlineMediaGroup
+		}
+		err = HandleInlineCached(
+			bot, ctx,
+			dlCtx, cached[0],
+		)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	err = StartInlineTask(bot, ctx, dlCtx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func HandleInlineCached(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+	media *models.Media,
+) error {
+	var result gotgbot.InlineQueryResult
+
+	format := media.Format
+	resultID := fmt.Sprintf("%d:%s", ctx.EffectiveUser.Id, format.FormatID)
+	resultTitle := "share"
+	mediaCaption := FormatCaption(media, true)
+	_, inputFileType := format.GetFormatInfo()
+
+	switch inputFileType {
+	case "photo":
+		result = &gotgbot.InlineQueryResultCachedPhoto{
+			Id:          resultID,
+			PhotoFileId: format.FileID,
+			Title:       resultTitle,
+			Caption:     mediaCaption,
+			ParseMode:   "HTML",
+		}
+	case "video":
+		result = &gotgbot.InlineQueryResultCachedVideo{
+			Id:          resultID,
+			VideoFileId: format.FileID,
+			Title:       resultTitle,
+			Caption:     mediaCaption,
+			ParseMode:   "HTML",
+		}
+	case "audio":
+		result = &gotgbot.InlineQueryResultCachedAudio{
+			Id:          resultID,
+			AudioFileId: format.FileID,
+			Caption:     mediaCaption,
+			ParseMode:   "HTML",
+		}
+	case "document":
+		result = &gotgbot.InlineQueryResultCachedDocument{
+			Id:             resultID,
+			DocumentFileId: format.FileID,
+			Title:          resultTitle,
+			Caption:        mediaCaption,
+			ParseMode:      "HTML",
+		}
+	default:
+		return errors.New("unsupported input file type")
+	}
+	ctx.InlineQuery.Answer(
+		bot, []gotgbot.InlineQueryResult{result},
+		&gotgbot.AnswerInlineQueryOpts{
+			CacheTime:  1,
+			IsPersonal: true,
+		},
+	)
+	return nil
+}
+
+func HandleInlineCachedResult(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+	media *models.Media,
+) error {
+	format := media.Format
+	messageCaption := FormatCaption(media, true)
+	inputMedia, err := format.GetInputMediaWithFileID(messageCaption)
+	if err != nil {
+		return err
+	}
+
+	_, _, err = bot.EditMessageMedia(
+		inputMedia,
+		&gotgbot.EditMessageMediaOpts{
+			InlineMessageId: ctx.ChosenInlineResult.InlineMessageId,
+		},
+	)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func StartInlineTask(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+) error {
+	randomID, err := uuid.NewUUID()
+	if err != nil {
+		return errors.New("could not generate task ID")
+	}
+	taskID := randomID.String()
+	inlineResult := &gotgbot.InlineQueryResultArticle{
+		Id:    taskID,
+		Title: "share",
+		InputMessageContent: &gotgbot.InputTextMessageContent{
+			MessageText: "loading media plese wait...",
+			ParseMode:   "HTML",
+			LinkPreviewOptions: &gotgbot.LinkPreviewOptions{
+				IsDisabled: true,
+			},
+		},
+		ReplyMarkup: &gotgbot.InlineKeyboardMarkup{
+			InlineKeyboard: [][]gotgbot.InlineKeyboardButton{
+				{
+					{
+						Text:         "...",
+						CallbackData: "inline:loading",
+					},
+				},
+			},
+		},
+	}
+	ok, err := ctx.InlineQuery.Answer(
+		bot, []gotgbot.InlineQueryResult{inlineResult},
+		&gotgbot.AnswerInlineQueryOpts{
+			CacheTime:  1,
+			IsPersonal: true,
+		},
+	)
+	if err != nil {
+		log.Println("failed to answer inline query:", err)
+	}
+	if !ok {
+		log.Println("failed to answer inline query")
+		return nil
+	}
+	InlineTasks[taskID] = dlCtx
+	return nil
+}
+
+func GetInlineFormat(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+	mediaChan chan<- *models.Media,
+	errChan chan<- error,
+) {
+	response, err := dlCtx.Extractor.Run(dlCtx)
+	if err != nil {
+		errChan <- fmt.Errorf("failed to get media: %w", err)
+		return
+	}
+	mediaList := response.MediaList
+	if len(mediaList) == 0 {
+		errChan <- fmt.Errorf("no media found for content ID: %s", dlCtx.MatchedContentID)
+	}
+	if len(mediaList) > 1 {
+		errChan <- util.ErrInlineMediaGroup
+		return
+	}
+	for i := range mediaList {
+		defaultFormat := mediaList[i].GetDefaultFormat()
+		if defaultFormat == nil {
+			errChan <- fmt.Errorf("no default format found for media at index %d", i)
+			return
+		}
+		if len(defaultFormat.URL) == 0 {
+			errChan <- fmt.Errorf("media format at index %d has no URL", i)
+			return
+		}
+		// ensure we can merge video and audio formats
+		ensureMergeFormats(mediaList[i], defaultFormat)
+		mediaList[i].Format = defaultFormat
+	}
+	messageCaption := FormatCaption(mediaList[0], true)
+	medias, err := DownloadMedias(mediaList, nil)
+	if err != nil {
+		errChan <- fmt.Errorf("failed to download medias: %w", err)
+		return
+	}
+	msgs, err := SendMedias(
+		bot, ctx, dlCtx,
+		medias, &models.SendMediaFormatsOptions{
+			Caption: messageCaption,
+		},
+	)
+	if err != nil {
+		errChan <- fmt.Errorf("failed to send media: %w", err)
+		return
+	}
+	msg := &msgs[0]
+	msg.Delete(bot, nil)
+	err = StoreMedias(
+		dlCtx, msgs,
+		medias,
+	)
+	if err != nil {
+		errChan <- fmt.Errorf("failed to store media: %w", err)
+		return
+	}
+	mediaChan <- medias[0].Media
+}

bot/core/main.go

@@ -0,0 +1,147 @@
+package core
+
+import (
+	"fmt"
+	"os"
+	"slices"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+func HandleDownloadRequest(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+) error {
+	chatID := ctx.EffectiveMessage.Chat.Id
+	if dlCtx.Extractor.Type == enums.ExtractorTypeSingle {
+		TypingEffect(bot, ctx, chatID)
+		err := HandleDefaultFormatDownload(bot, ctx, dlCtx)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	return util.ErrUnsupportedExtractorType
+}
+
+func SendMedias(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	dlCtx *models.DownloadContext,
+	medias []*models.DownloadedMedia,
+	options *models.SendMediaFormatsOptions,
+) ([]gotgbot.Message, error) {
+	var chatID int64
+	var messageOptions *gotgbot.SendMediaGroupOpts
+
+	if dlCtx.GroupSettings != nil {
+		if len(medias) > dlCtx.GroupSettings.MediaGroupLimit {
+			return nil, util.ErrMediaGroupLimitExceeded
+		}
+		if !*dlCtx.GroupSettings.NSFW {
+			for _, media := range medias {
+				if media.Media.NSFW {
+					return nil, util.ErrNSFWNotAllowed
+				}
+			}
+		}
+	}
+
+	switch {
+	case ctx.Message != nil:
+		chatID = ctx.EffectiveMessage.Chat.Id
+		messageOptions = &gotgbot.SendMediaGroupOpts{
+			ReplyParameters: &gotgbot.ReplyParameters{
+				MessageId: ctx.EffectiveMessage.MessageId,
+			},
+		}
+	case ctx.CallbackQuery != nil:
+		chatID = ctx.CallbackQuery.Message.GetChat().Id
+		messageOptions = nil
+	case ctx.InlineQuery != nil:
+		chatID = ctx.InlineQuery.From.Id
+		messageOptions = nil
+	case ctx.ChosenInlineResult != nil:
+		chatID = ctx.ChosenInlineResult.From.Id
+		messageOptions = &gotgbot.SendMediaGroupOpts{
+			DisableNotification: true,
+		}
+	default:
+		return nil, errors.New("failed to get chat id")
+	}
+
+	var sentMessages []gotgbot.Message
+
+	mediaGroupChunks := slices.Collect(
+		slices.Chunk(medias, 10),
+	)
+
+	for _, chunk := range mediaGroupChunks {
+		var inputMediaList []gotgbot.InputMedia
+		for idx, media := range chunk {
+			var caption string
+
+			if idx == 0 {
+				caption = options.Caption
+			}
+			inputMedia, err := media.Media.Format.GetInputMedia(
+				media.FilePath,
+				media.ThumbnailFilePath,
+				caption,
+			)
+			if err != nil {
+				return nil, fmt.Errorf("failed to get input media: %w", err)
+			}
+			inputMediaList = append(inputMediaList, inputMedia)
+		}
+		mediaType := chunk[0].Media.Format.Type
+		SendingEffect(bot, ctx, chatID, mediaType)
+		msgs, err := bot.SendMediaGroup(
+			chatID,
+			inputMediaList,
+			messageOptions,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, media := range chunk {
+			if media.FilePath != "" {
+				os.Remove(media.FilePath)
+			}
+			if media.ThumbnailFilePath != "" {
+				os.Remove(media.ThumbnailFilePath)
+			}
+		}
+
+		sentMessages = append(sentMessages, msgs...)
+		if sentMessages[0].Chat.Type != "private" {
+			if len(mediaGroupChunks) > 1 {
+				time.Sleep(3 * time.Second)
+			} // avoid floodwait?
+		}
+	}
+	if len(sentMessages) == 0 {
+		return nil, errors.New("no messages sent")
+	}
+	if !options.IsStored {
+		err := StoreMedias(
+			dlCtx,
+			sentMessages,
+			medias,
+		)
+		if err != nil {
+			return nil, fmt.Errorf("failed to cache formats: %w", err)
+		}
+	}
+	return sentMessages, nil
+}

bot/core/util.go

@@ -0,0 +1,280 @@
+package core
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"govd/database"
+	"govd/enums"
+	"govd/models"
+	"govd/plugins"
+	"govd/util"
+	"govd/util/av"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+func getFileThumbnail(
+	format *models.MediaFormat,
+	filePath string,
+) (string, error) {
+	fileDir := filepath.Dir(filePath)
+	fileName := filepath.Base(filePath)
+	fileExt := filepath.Ext(fileName)
+	fileBaseName := fileName[:len(fileName)-len(fileExt)]
+	thumbnailFilePath := filepath.Join(fileDir, fileBaseName+".thumb.jpeg")
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	if len(format.Thumbnail) > 0 {
+		file, err := util.DownloadFileInMemory(ctx, format.Thumbnail, nil)
+		if err != nil {
+			return "", fmt.Errorf("failed to download file in memory: %w", err)
+		}
+		err = util.ImgToJPEG(file, thumbnailFilePath)
+		if err != nil {
+			return "", fmt.Errorf("failed to convert to JPEG: %w", err)
+		}
+		return thumbnailFilePath, nil
+	}
+	if format.Type == enums.MediaTypeVideo {
+		err := av.ExtractVideoThumbnail(filePath, thumbnailFilePath)
+		if err != nil {
+			return "", fmt.Errorf("failed to extract video thumbnail: %w", err)
+		}
+		return thumbnailFilePath, nil
+	}
+	return "", nil
+}
+
+func insertVideoInfo(
+	format *models.MediaFormat,
+	filePath string,
+) {
+	width, height, duration := av.GetVideoInfo(filePath)
+	format.Width = width
+	format.Height = height
+	format.Duration = duration
+}
+
+func GetMessageFileID(msg *gotgbot.Message) string {
+	switch {
+	case msg.Video != nil:
+		return msg.Video.FileId
+	case msg.Animation != nil:
+		return msg.Animation.FileId
+	case msg.Photo != nil:
+		return msg.Photo[len(msg.Photo)-1].FileId
+	case msg.Document != nil:
+		return msg.Document.FileId
+	case msg.Audio != nil:
+		return msg.Audio.FileId
+	case msg.Voice != nil:
+		return msg.Voice.FileId
+	default:
+		return ""
+	}
+}
+
+func GetMessageFileSize(msg *gotgbot.Message) int64 {
+	switch {
+	case msg.Video != nil:
+		return msg.Video.FileSize
+	case msg.Animation != nil:
+		return msg.Animation.FileSize
+	case msg.Photo != nil:
+		return msg.Photo[len(msg.Photo)-1].FileSize
+	case msg.Document != nil:
+		return msg.Document.FileSize
+	case msg.Audio != nil:
+		return msg.Audio.FileSize
+	case msg.Voice != nil:
+		return msg.Voice.FileSize
+	default:
+		return 0
+	}
+}
+
+func StoreMedias(
+	dlCtx *models.DownloadContext,
+	msgs []gotgbot.Message,
+	medias []*models.DownloadedMedia,
+) error {
+	var storedMedias []*models.Media
+	if len(medias) == 0 {
+		return fmt.Errorf("no media to store")
+	}
+	for idx, msg := range msgs {
+		fileID := GetMessageFileID(&msg)
+		if len(fileID) == 0 {
+			return fmt.Errorf("no file ID found for media at index %d", idx)
+		}
+		fileSize := GetMessageFileSize(&msg)
+		medias[idx].Media.Format.FileID = fileID
+		medias[idx].Media.Format.FileSize = fileSize
+		storedMedias = append(
+			storedMedias,
+			medias[idx].Media,
+		)
+	}
+	for _, media := range storedMedias {
+		err := database.StoreMedia(
+			dlCtx.Extractor.CodeName,
+			media.ContentID,
+			media,
+		)
+		if err != nil {
+			return fmt.Errorf("failed to store media: %w", err)
+		}
+	}
+	return nil
+}
+
+func FormatCaption(
+	media *models.Media,
+	isEnabled bool,
+) string {
+	newCaption := fmt.Sprintf(
+		"<a href='%s'>source</a> - @govd_bot\n",
+		media.ContentURL,
+	)
+	if isEnabled && media.Caption.Valid {
+		text := media.Caption.String
+		if len(text) > 600 {
+			text = text[:600] + "..."
+		}
+		newCaption += fmt.Sprintf(
+			"<blockquote expandable>%s</blockquote>\n",
+			util.EscapeCaption(text),
+		)
+	}
+	return newCaption
+}
+
+func TypingEffect(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	chatID int64,
+) {
+	bot.SendChatAction(
+		chatID,
+		"typing",
+		nil,
+	)
+}
+
+func SendingEffect(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	chatID int64,
+	mediaType enums.MediaType,
+) {
+	action := "upload_document"
+	if mediaType == enums.MediaTypeVideo {
+		action = "upload_video"
+	}
+	if mediaType == enums.MediaTypeAudio {
+		action = "upload_audio"
+	}
+	if mediaType == enums.MediaTypePhoto {
+		action = "upload_photo"
+	}
+	bot.SendChatAction(
+		chatID,
+		action,
+		nil,
+	)
+}
+
+func HandleErrorMessage(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	err error,
+) {
+	currentError := err
+	for currentError != nil {
+		var botError *util.Error
+		if errors.As(currentError, &botError) {
+			SendErrorMessage(bot, ctx, fmt.Sprintf(
+				"error occurred when downloading: %s",
+				currentError.Error(),
+			))
+			return
+		}
+		currentError = errors.Unwrap(currentError)
+	}
+
+	lastError := util.GetLastError(err)
+	errorMessage := fmt.Sprintf(
+		"error occurred when downloading: %s",
+		lastError.Error(),
+	)
+
+	if strings.Contains(errorMessage, bot.Token) {
+		errorMessage = "telegram related error, probably connection issue"
+	}
+
+	SendErrorMessage(bot, ctx, errorMessage)
+
+}
+
+func SendErrorMessage(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+	errorMessage string,
+) {
+	log.Println(errorMessage)
+
+	switch {
+	case ctx.Update.Message != nil:
+		ctx.EffectiveMessage.Reply(
+			bot,
+			errorMessage,
+			nil,
+		)
+	case ctx.Update.InlineQuery != nil:
+		ctx.InlineQuery.Answer(
+			bot,
+			nil,
+			&gotgbot.AnswerInlineQueryOpts{
+				CacheTime: 1,
+				Button: &gotgbot.InlineQueryResultsButton{
+					Text:           errorMessage,
+					StartParameter: "start",
+				},
+			},
+		)
+	case ctx.ChosenInlineResult != nil:
+		bot.EditMessageText(
+			errorMessage,
+			&gotgbot.EditMessageTextOpts{
+				InlineMessageId: ctx.ChosenInlineResult.InlineMessageId,
+			})
+	}
+}
+
+func ensureMergeFormats(
+	media *models.Media,
+	videoFormat *models.MediaFormat,
+) {
+	if videoFormat.Type != enums.MediaTypeVideo {
+		return
+	}
+	if videoFormat.AudioCodec != "" {
+		return
+	}
+	// video with no audio
+	audioFormat := media.GetDefaultAudioFormat()
+	if audioFormat == nil {
+		return
+	}
+	videoFormat.AudioCodec = audioFormat.AudioCodec
+	videoFormat.Plugins = append(videoFormat.Plugins, plugins.MergeAudio)
+}

bot/handlers/ext.go

@@ -0,0 +1,44 @@
+package handlers
+
+import (
+	extractors "govd/ext"
+	"strings"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+func ExtractorsHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	ctx.CallbackQuery.Answer(bot, nil)
+
+	messageText := "available extractors:\n"
+	extractorNames := make([]string, 0, len(extractors.List))
+	for _, extractor := range extractors.List {
+		if extractor.IsRedirect {
+			continue
+		}
+		extractorNames = append(extractorNames, extractor.Name)
+	}
+	messageText += strings.Join(extractorNames, ", ")
+
+	ctx.EffectiveMessage.EditText(
+		bot,
+		messageText,
+		&gotgbot.EditMessageTextOpts{
+			LinkPreviewOptions: &gotgbot.LinkPreviewOptions{
+				IsDisabled: true,
+			},
+			ReplyMarkup: gotgbot.InlineKeyboardMarkup{
+				InlineKeyboard: [][]gotgbot.InlineKeyboardButton{
+					{
+						{
+							Text:         "back",
+							CallbackData: "start",
+						},
+					},
+				},
+			},
+		},
+	)
+	return nil
+}

bot/handlers/help.go

@@ -0,0 +1,45 @@
+package handlers
+
+import (
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+var helpMessage = "usage:\n" +
+	"- you can add the bot to a group " +
+	"to start catching sent links\n" +
+	"- you can send a link to the bot privately " +
+	"to download the media too\n\n" +
+	"group commands:\n" +
+	"- /settings = show current settings\n" +
+	"- /captions (true|false) = enable/disable descriptions\n" +
+	"- /nsfw (true|false) = enable/disable nsfw content\n" +
+	"- /limit (int) = set max items in media groups\n\n" +
+	"note: the bot is still in beta, " +
+	"so expect some bugs and missing features.\n"
+
+var helpKeyboard = gotgbot.InlineKeyboardMarkup{
+	InlineKeyboard: [][]gotgbot.InlineKeyboardButton{
+		{
+			{
+				Text:         "back",
+				CallbackData: "start",
+			},
+		},
+	},
+}
+
+func HelpHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	ctx.CallbackQuery.Answer(bot, nil)
+	ctx.EffectiveMessage.EditText(
+		bot,
+		helpMessage,
+		&gotgbot.EditMessageTextOpts{
+			LinkPreviewOptions: &gotgbot.LinkPreviewOptions{
+				IsDisabled: true,
+			},
+			ReplyMarkup: helpKeyboard,
+		},
+	)
+	return nil
+}

bot/handlers/inline.go

@@ -0,0 +1,91 @@
+package handlers
+
+import (
+	"context"
+	"govd/bot/core"
+	"govd/models"
+	"govd/util"
+	"strings"
+	"time"
+
+	extractors "govd/ext"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+func InlineDownloadHandler(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+) error {
+	url := strings.TrimSpace(ctx.InlineQuery.Query)
+	if url == "" {
+		ctx.InlineQuery.Answer(bot, []gotgbot.InlineQueryResult{}, &gotgbot.AnswerInlineQueryOpts{
+			CacheTime:  1,
+			IsPersonal: true,
+		})
+		return nil
+	}
+	dlCtx, err := extractors.CtxByURL(url)
+	if err != nil || dlCtx == nil || dlCtx.Extractor == nil {
+		ctx.InlineQuery.Answer(bot, []gotgbot.InlineQueryResult{}, &gotgbot.AnswerInlineQueryOpts{
+			CacheTime:  1,
+			IsPersonal: true,
+		})
+		return nil
+	}
+	return core.HandleInline(bot, ctx, dlCtx)
+}
+
+func InlineDownloadResultHandler(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+) error {
+	dlCtx, ok := core.InlineTasks[ctx.ChosenInlineResult.ResultId]
+	if !ok {
+		return nil
+	}
+	defer delete(core.InlineTasks, ctx.ChosenInlineResult.ResultId)
+
+	mediaChan := make(chan *models.Media, 1)
+	errChan := make(chan error, 1)
+	timeout, cancel := context.WithTimeout(
+		context.Background(),
+		5*time.Minute,
+	)
+	defer cancel()
+
+	go core.GetInlineFormat(
+		bot, ctx, dlCtx,
+		mediaChan, errChan,
+	)
+	select {
+	case media := <-mediaChan:
+		err := core.HandleInlineCachedResult(
+			bot, ctx,
+			dlCtx, media,
+		)
+		if err != nil {
+			core.HandleErrorMessage(bot, ctx, err)
+			return nil
+		}
+	case err := <-errChan:
+		core.HandleErrorMessage(bot, ctx, err)
+		return nil
+	case <-timeout.Done():
+		core.HandleErrorMessage(bot, ctx, util.ErrTimeout)
+		return nil
+	}
+	return nil
+}
+
+func InlineLoadingHandler(
+	bot *gotgbot.Bot,
+	ctx *ext.Context,
+) error {
+	ctx.CallbackQuery.Answer(bot, &gotgbot.AnswerCallbackQueryOpts{
+		Text:      "wait !",
+		ShowAlert: true,
+	})
+	return nil
+}

bot/handlers/instances.go

@@ -0,0 +1,72 @@
+package handlers
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strings"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+var buildHash = "unknown"
+var branchName = "unknown"
+
+func getInstanceMessage() string {
+	return "current instance\n" +
+		"go version: %s\n" +
+		"build: <a href='%s'>%s</a>\n" +
+		"branch: <a href='%s'>%s</a>\n\n" +
+		"public instances\n" +
+		"- @govd_bot | main official instance\n" +
+		"\nwant to add your own instance? reach us on @govdsupport"
+}
+
+func InstancesHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	var commitURL string
+	var branchURL string
+
+	repoURL := os.Getenv("REPO_URL")
+	if repoURL != "" {
+		commitURL = fmt.Sprintf(
+			"%s/tree/%s",
+			repoURL,
+			buildHash,
+		)
+		branchURL = fmt.Sprintf(
+			"%s/tree/%s",
+			repoURL,
+			branchName,
+		)
+	}
+	messageText := fmt.Sprintf(
+		getInstanceMessage(),
+		strings.TrimPrefix(runtime.Version(), "go"),
+		commitURL,
+		buildHash,
+		branchURL,
+		branchName,
+	)
+	ctx.CallbackQuery.Answer(bot, nil)
+	ctx.EffectiveMessage.EditText(
+		bot,
+		messageText,
+		&gotgbot.EditMessageTextOpts{
+			LinkPreviewOptions: &gotgbot.LinkPreviewOptions{
+				IsDisabled: true,
+			},
+			ReplyMarkup: gotgbot.InlineKeyboardMarkup{
+				InlineKeyboard: [][]gotgbot.InlineKeyboardButton{
+					{
+						{
+							Text:         "back",
+							CallbackData: "start",
+						},
+					},
+				},
+			},
+		},
+	)
+	return nil
+}

bot/handlers/settings.go

@@ -0,0 +1,213 @@
+package handlers
+
+import (
+	"fmt"
+	"govd/database"
+	"govd/util"
+	"strconv"
+	"strings"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+func SettingsHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	if ctx.EffectiveMessage.Chat.Type == "private" {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"use this command in group chats only",
+			nil,
+		)
+		return nil
+	}
+	settings, err := database.GetGroupSettings(ctx.EffectiveMessage.Chat.Id)
+	if err != nil {
+		return err
+	}
+	ctx.EffectiveMessage.Reply(
+		bot,
+		fmt.Sprintf(
+			"settings for this group\n\ncaptions: %s\nnsfw: %s\nmedia group limit: %d",
+			strconv.FormatBool(*settings.Captions),
+			strconv.FormatBool(*settings.NSFW),
+			settings.MediaGroupLimit,
+		),
+		nil,
+	)
+	return nil
+}
+
+func CaptionsHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	if ctx.EffectiveMessage.Chat.Type == "private" {
+		return nil
+	}
+
+	chatID := ctx.EffectiveMessage.Chat.Id
+	userID := ctx.EffectiveMessage.From.Id
+
+	args := ctx.Args()
+	if len(args) != 2 {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"usage: /captions (true|false)",
+			nil,
+		)
+		return nil
+	}
+	if !util.IsUserAdmin(bot, chatID, userID) {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"you don't have permission to change settings",
+			nil,
+		)
+		return nil
+	}
+	userInput := strings.ToLower(args[1])
+	value, err := strconv.ParseBool(userInput)
+	if err != nil {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			fmt.Sprintf("invalid value (%s), use true or false", userInput),
+			nil,
+		)
+		return nil
+	}
+	settings, err := database.GetGroupSettings(chatID)
+	if err != nil {
+		return err
+	}
+	settings.Captions = &value
+	err = database.UpdateGroupSettings(chatID, settings)
+	if err != nil {
+		return err
+	}
+	var message string
+	if value {
+		message = "captions enabled"
+	} else {
+		message = "captions disabled"
+	}
+	ctx.EffectiveMessage.Reply(
+		bot,
+		message,
+		nil,
+	)
+	return nil
+}
+
+func NSFWHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	if ctx.EffectiveMessage.Chat.Type == "private" {
+		return nil
+	}
+
+	chatID := ctx.EffectiveMessage.Chat.Id
+	userID := ctx.EffectiveMessage.From.Id
+
+	args := ctx.Args()
+	if len(args) != 2 {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"usage: /nsfw (true|false)",
+			nil,
+		)
+		return nil
+	}
+	if !util.IsUserAdmin(bot, chatID, userID) {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"you don't have permission to change settings",
+			nil,
+		)
+		return nil
+	}
+	userInput := strings.ToLower(args[1])
+	value, err := strconv.ParseBool(userInput)
+	if err != nil {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			fmt.Sprintf("invalid value (%s), use true or false", userInput),
+			nil,
+		)
+		return nil
+	}
+	settings, err := database.GetGroupSettings(chatID)
+	if err != nil {
+		return err
+	}
+	settings.NSFW = &value
+	err = database.UpdateGroupSettings(chatID, settings)
+	if err != nil {
+		return err
+	}
+	var message string
+	if value {
+		message = "nsfw enabled"
+	} else {
+		message = "nsfw disabled"
+	}
+	ctx.EffectiveMessage.Reply(
+		bot,
+		message,
+		nil,
+	)
+	return nil
+}
+
+func MediaGroupLimitHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	if ctx.EffectiveMessage.Chat.Type == "private" {
+		return nil
+	}
+
+	chatID := ctx.EffectiveMessage.Chat.Id
+	userID := ctx.EffectiveMessage.From.Id
+
+	args := ctx.Args()
+	if len(args) != 2 {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"usage: /limit (int)",
+			nil,
+		)
+		return nil
+	}
+	if !util.IsUserAdmin(bot, chatID, userID) {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"you don't have permission to change settings",
+			nil,
+		)
+		return nil
+	}
+	value, err := strconv.Atoi(args[1])
+	if err != nil {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			fmt.Sprintf("invalid value (%s), use a number", args[1]),
+			nil,
+		)
+		return nil
+	}
+	if value < 1 || value > 20 {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			"media group limit must be between 1 and 20",
+			nil,
+		)
+		return nil
+	}
+	settings, err := database.GetGroupSettings(chatID)
+	if err != nil {
+		return err
+	}
+	settings.MediaGroupLimit = value
+	err = database.UpdateGroupSettings(chatID, settings)
+	if err != nil {
+		return err
+	}
+	ctx.EffectiveMessage.Reply(
+		bot,
+		fmt.Sprintf("media group limit set to %d", value),
+		nil,
+	)
+	return nil
+}

bot/handlers/start.go

@@ -0,0 +1,88 @@
+package handlers
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+var startMessage = "govd is an open-source telegram bot " +
+	"that allows you to download medias from " +
+	"various platforms. the project born after " +
+	"the discontinuation of an " +
+	"highly popular bot, known as UVD."
+
+func getStartKeyboard(bot *gotgbot.Bot) gotgbot.InlineKeyboardMarkup {
+	return gotgbot.InlineKeyboardMarkup{
+		InlineKeyboard: [][]gotgbot.InlineKeyboardButton{
+			{
+				{
+					Text: "add to group",
+					Url: fmt.Sprintf(
+						"https://t.me/%s?startgroup=true",
+						bot.Username,
+					),
+				},
+			},
+			{
+				{
+					Text:         "usage",
+					CallbackData: "help",
+				},
+				{
+					Text:         "stats",
+					CallbackData: "stats",
+				},
+			},
+			{
+				{
+					Text:         "extractors",
+					CallbackData: "extractors",
+				},
+				{
+					Text: "support",
+					Url:  "https://t.me/govdsupport",
+				},
+			},
+			{
+				{
+					Text:         "instances",
+					CallbackData: "instances",
+				},
+				{
+					Text: "github",
+					Url:  os.Getenv("REPO_URL"),
+				},
+			},
+		},
+	}
+}
+
+func StartHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	if ctx.EffectiveMessage.Chat.Type != "private" {
+		return nil
+	}
+	keyboard := getStartKeyboard(bot)
+	if ctx.Update.Message != nil {
+		ctx.EffectiveMessage.Reply(
+			bot,
+			startMessage,
+			&gotgbot.SendMessageOpts{
+				ReplyMarkup: &keyboard,
+			},
+		)
+	} else if ctx.Update.CallbackQuery != nil {
+		ctx.CallbackQuery.Answer(bot, nil)
+		ctx.EffectiveMessage.EditText(
+			bot,
+			startMessage,
+			&gotgbot.EditMessageTextOpts{
+				ReplyMarkup: keyboard,
+			},
+		)
+
+	}
+	return nil
+}

bot/handlers/stats.go

@@ -0,0 +1,89 @@
+package handlers
+
+import (
+	"fmt"
+	"govd/database"
+	"time"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+)
+
+type Stats struct {
+	TotalUsers      int64
+	TotalGroups     int64
+	TotalDailyUsers int64
+	TotalMedia      int64
+	UpdatedAt       time.Time
+}
+
+var lastSavedStats *Stats
+
+var statsMessage = "users: %d\nusers today: %d\ngroups: %d\ndownloads: %d\n\nupdates every 10 minutes"
+
+func StatsHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	if ctx.EffectiveMessage.Chat.Type != "private" {
+		return nil
+	}
+	ctx.CallbackQuery.Answer(bot, nil)
+	stats := GetStats()
+	ctx.EffectiveMessage.EditText(
+		bot,
+		fmt.Sprintf(
+			statsMessage,
+			stats.TotalUsers,
+			stats.TotalDailyUsers,
+			stats.TotalGroups,
+			stats.TotalMedia,
+		),
+		&gotgbot.EditMessageTextOpts{
+			ReplyMarkup: gotgbot.InlineKeyboardMarkup{
+				InlineKeyboard: [][]gotgbot.InlineKeyboardButton{
+					{
+						{
+							Text:         "back",
+							CallbackData: "start",
+						},
+					},
+				},
+			},
+		},
+	)
+	return nil
+}
+
+func UpdateStats() {
+	totalUsers, err := database.GetUsersCount()
+	if err != nil {
+		return
+	}
+	totalGroups, err := database.GetGroupsCount()
+	if err != nil {
+		return
+	}
+	totalDailyUsers, err := database.GetDailyUserCount()
+	if err != nil {
+		return
+	}
+	totalMedia, err := database.GetMediaCount()
+	if err != nil {
+		return
+	}
+	lastSavedStats = &Stats{
+		TotalUsers:      totalUsers,
+		TotalGroups:     totalGroups,
+		TotalDailyUsers: totalDailyUsers,
+		TotalMedia:      totalMedia,
+		UpdatedAt:       time.Now(),
+	}
+}
+
+func GetStats() *Stats {
+	if lastSavedStats == nil {
+		UpdateStats()
+	}
+	if lastSavedStats.UpdatedAt.Add(10 * time.Minute).Before(time.Now()) {
+		UpdateStats()
+	}
+	return lastSavedStats
+}

bot/handlers/url.go

@@ -0,0 +1,65 @@
+package handlers
+
+import (
+	"govd/bot/core"
+	"govd/database"
+	extractors "govd/ext"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext/handlers/filters/message"
+)
+
+func URLHandler(bot *gotgbot.Bot, ctx *ext.Context) error {
+	messageURL := getMessageURL(ctx.EffectiveMessage)
+	if messageURL == "" {
+		return nil
+	}
+	dlCtx, err := extractors.CtxByURL(messageURL)
+	if err != nil {
+		core.HandleErrorMessage(
+			bot, ctx, err)
+		return nil
+	}
+	if dlCtx == nil || dlCtx.Extractor == nil {
+		return nil
+	}
+	userID := ctx.EffectiveMessage.From.Id
+	if ctx.EffectiveMessage.Chat.Type != "private" {
+		settings, err := database.GetGroupSettings(ctx.EffectiveMessage.Chat.Id)
+		if err != nil {
+			return err
+		}
+		dlCtx.GroupSettings = settings
+	}
+	if userID != 1087968824 {
+		// groupAnonymousBot
+		_, err = database.GetUser(userID)
+		if err != nil {
+			return err
+		}
+	}
+	err = core.HandleDownloadRequest(bot, ctx, dlCtx)
+	if err != nil {
+		core.HandleErrorMessage(
+			bot, ctx, err)
+	}
+	return nil
+}
+
+func URLFilter(msg *gotgbot.Message) bool {
+	return message.Text(msg) && !message.Command(msg) && containsURL(msg)
+}
+
+func containsURL(msg *gotgbot.Message) bool {
+	return message.Entity("url")(msg)
+}
+
+func getMessageURL(msg *gotgbot.Message) string {
+	for _, entity := range msg.Entities {
+		if entity.Type == "url" {
+			return msg.Text[entity.Offset : entity.Offset+entity.Length]
+		}
+	}
+	return ""
+}

bot/main.go

@@ -0,0 +1,118 @@
+package bot
+
+import (
+	"log"
+	"os"
+	"time"
+
+	botHandlers "govd/bot/handlers"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext/handlers"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext/handlers/filters/callbackquery"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext/handlers/filters/choseninlineresult"
+	"github.com/PaulSonOfLars/gotgbot/v2/ext/handlers/filters/inlinequery"
+)
+
+var AllowedUpdates = []string{
+	"message",
+	"callback_query",
+	"inline_query",
+	"chosen_inline_result",
+}
+
+func Start() {
+	token := os.Getenv("BOT_TOKEN")
+	if token == "" {
+		log.Fatalf("BOT_TOKEN is not provided")
+	}
+	b, err := gotgbot.NewBot(token, &gotgbot.BotOpts{
+		BotClient: NewBotClient(),
+	})
+	if err != nil {
+		log.Fatalf("failed to create bot: %v", err)
+	}
+	dispatcher := ext.NewDispatcher(&ext.DispatcherOpts{
+		Error: func(b *gotgbot.Bot, ctx *ext.Context, err error) ext.DispatcherAction {
+			log.Println("an error occurred while handling update:", err.Error())
+			return ext.DispatcherActionNoop
+		},
+		MaxRoutines: ext.DefaultMaxRoutines,
+	})
+	updater := ext.NewUpdater(dispatcher, nil)
+	registerHandlers(dispatcher)
+	err = updater.StartPolling(b, &ext.PollingOpts{
+		DropPendingUpdates: true,
+		GetUpdatesOpts: &gotgbot.GetUpdatesOpts{
+			Timeout: 9 * 60,
+			RequestOpts: &gotgbot.RequestOpts{
+				Timeout: time.Minute * 10,
+			},
+			AllowedUpdates: AllowedUpdates,
+		},
+	})
+	if err != nil {
+		log.Fatalf("failed to start polling: %v", err)
+	}
+	log.Printf("bot started on: %s\n", b.User.Username)
+}
+
+func registerHandlers(dispatcher *ext.Dispatcher) {
+	dispatcher.AddHandler(handlers.NewMessage(
+		botHandlers.URLFilter,
+		botHandlers.URLHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCommand(
+		"start",
+		botHandlers.StartHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCallback(
+		callbackquery.Equal("start"),
+		botHandlers.StartHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCallback(
+		callbackquery.Equal("help"),
+		botHandlers.HelpHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCommand(
+		"settings",
+		botHandlers.SettingsHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCommand(
+		"captions",
+		botHandlers.CaptionsHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCommand(
+		"nsfw",
+		botHandlers.NSFWHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCommand(
+		"limit",
+		botHandlers.MediaGroupLimitHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCallback(
+		callbackquery.Equal("stats"),
+		botHandlers.StatsHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCallback(
+		callbackquery.Equal("extractors"),
+		botHandlers.ExtractorsHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCallback(
+		callbackquery.Equal("instances"),
+		botHandlers.InstancesHandler,
+	))
+	dispatcher.AddHandler(handlers.NewInlineQuery(
+		inlinequery.All,
+		botHandlers.InlineDownloadHandler,
+	))
+	dispatcher.AddHandler(handlers.NewChosenInlineResult(
+		choseninlineresult.All,
+		botHandlers.InlineDownloadResultHandler,
+	))
+	dispatcher.AddHandler(handlers.NewCallback(
+		callbackquery.Equal("inline:loading"),
+		botHandlers.InlineLoadingHandler,
+	))
+}

bot/middleware.go

@@ -0,0 +1,56 @@
+package bot
+
+import (
+	"context"
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+)
+
+type BotClient struct {
+	gotgbot.BotClient
+}
+
+func (b BotClient) RequestWithContext(
+	ctx context.Context,
+	token string,
+	method string,
+	params map[string]string,
+	data map[string]gotgbot.FileReader,
+	opts *gotgbot.RequestOpts,
+) (json.RawMessage, error) {
+	if strings.HasPrefix(method, "send") || method == "copyMessage" {
+		params["allow_sending_without_reply"] = "true"
+	}
+	if strings.HasPrefix(method, "send") || strings.HasPrefix(method, "edit") {
+		params["parse_mode"] = "HTML"
+	}
+	val, err := b.BotClient.RequestWithContext(ctx, token, method, params, data, opts)
+	if err != nil {
+		return nil, err
+	}
+	return val, err
+}
+
+func NewBotClient() BotClient {
+	botAPIURL := os.Getenv("BOT_API_URL")
+	if botAPIURL == "" {
+		log.Println("BOT_API_URL is not provided, using default")
+		botAPIURL = gotgbot.DefaultAPIURL
+	}
+	return BotClient{
+		BotClient: &gotgbot.BaseBotClient{
+			Client:             http.Client{},
+			UseTestEnvironment: false,
+			DefaultRequestOpts: &gotgbot.RequestOpts{
+				Timeout: 10 * time.Minute,
+				APIURL:  botAPIURL,
+			},
+		},
+	}
+}

build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+COMMIT_HASH=$(git rev-parse --short HEAD)
+BRANCH_NAME=$(git branch --show-current)
+
+PACKAGE_PATH="govd/bot/handlers"
+
+echo "Building with commit hash: ${COMMIT_HASH}"
+echo "Branch name: ${BRANCH_NAME}"
+go build -ldflags="-X '${PACKAGE_PATH}.buildHash=${COMMIT_HASH}' -X '${PACKAGE_PATH}.branchName=${BRANCH_NAME}'"
+
+if [ $? -eq 0 ]; then
+    echo "Build completed successfully"
+else
+    echo "Build failed"
+    exit 1
+fi

cookies/.gitignore

@@ -0,0 +1 @@
+*.txt

database/main.go

@@ -0,0 +1,65 @@
+package database
+
+import (
+	"govd/models"
+
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"gorm.io/driver/mysql"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+var DB *gorm.DB
+
+func Start() {
+	host := os.Getenv("DB_HOST")
+	port := os.Getenv("DB_PORT")
+	user := os.Getenv("DB_USER")
+	password := os.Getenv("DB_PASSWORD")
+	dbname := os.Getenv("DB_NAME")
+
+	connectionString := fmt.Sprintf(
+		"%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True",
+		user, password, host, port, dbname,
+	)
+	db, err := gorm.Open(mysql.Open(connectionString), &gorm.Config{
+		Logger: logger.Default.LogMode(logger.Silent),
+		NowFunc: func() time.Time {
+			utc, _ := time.LoadLocation("Europe/Rome")
+			return time.Now().In(utc)
+		},
+	})
+	if err != nil {
+		log.Fatalf("failed to connect to database: %v", err)
+	}
+	DB = db
+	sqlDB, err := DB.DB()
+	if err != nil {
+		log.Fatalf("failed to get database connection: %v", err)
+	}
+	err = sqlDB.Ping()
+	if err != nil {
+		log.Fatalf("failed to ping database: %v", err)
+	}
+	err = migrateDatabase()
+	if err != nil {
+		log.Fatalf("failed to migrate database: %v", err)
+	}
+}
+
+func migrateDatabase() error {
+	err := DB.AutoMigrate(
+		&models.Media{},
+		&models.MediaFormat{},
+		&models.GroupSettings{},
+		&models.User{},
+	)
+	if err != nil {
+		return err
+	}
+	return nil
+}

database/media.go

@@ -0,0 +1,60 @@
+package database
+
+import (
+	"fmt"
+
+	"govd/models"
+
+	"gorm.io/gorm"
+)
+
+func GetDefaultMedias(
+	extractorCodeName string,
+	contentID string,
+) ([]*models.Media, error) {
+	var mediaList []*models.Media
+
+	err := DB.
+		Where(&models.Media{
+			ExtractorCodeName: extractorCodeName,
+			ContentID:         contentID,
+		}).
+		Preload("Format", "is_default = ?", true).
+		Find(&mediaList).
+		Error
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to get stored media list: %w", err)
+	}
+
+	return mediaList, nil
+}
+
+func StoreMedia(
+	extractorCodeName string,
+	contentID string,
+	media *models.Media,
+) error {
+	return DB.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Where(models.Media{
+			ExtractorCodeName: extractorCodeName,
+			ContentID:         contentID,
+		}).FirstOrCreate(&media).Error; err != nil {
+			return fmt.Errorf("failed to get or create media: %w", err)
+		}
+		if media.Format != nil {
+			format := media.Format
+			format.MediaID = media.ID
+
+			if err := tx.Where(models.MediaFormat{
+				MediaID:  format.MediaID,
+				FormatID: format.FormatID,
+				Type:     format.Type,
+			}).FirstOrCreate(format).Error; err != nil {
+				return fmt.Errorf("failed to get or create format: %w", err)
+			}
+		}
+
+		return nil
+	})
+}

database/settings.go

@@ -0,0 +1,37 @@
+package database
+
+import (
+	"govd/models"
+)
+
+func GetGroupSettings(
+	chatID int64,
+) (*models.GroupSettings, error) {
+	var groupSettings models.GroupSettings
+	err := DB.
+		Where(&models.GroupSettings{
+			ChatID: chatID,
+		}).
+		FirstOrCreate(&groupSettings).
+		Error
+	if err != nil {
+		return nil, err
+	}
+	return &groupSettings, nil
+}
+
+func UpdateGroupSettings(
+	chatID int64,
+	settings *models.GroupSettings,
+) error {
+	err := DB.
+		Where(&models.GroupSettings{
+			ChatID: chatID,
+		}).
+		Updates(settings).
+		Error
+	if err != nil {
+		return err
+	}
+	return nil
+}

database/stats.go

@@ -0,0 +1,52 @@
+package database
+
+import "govd/models"
+
+func GetMediaCount() (int64, error) {
+	var count int64
+	err := DB.
+		Model(&models.Media{}).
+		Count(&count).
+		Error
+	if err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
+func GetUsersCount() (int64, error) {
+	var count int64
+	err := DB.
+		Model(&models.User{}).
+		Count(&count).
+		Error
+	if err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
+func GetGroupsCount() (int64, error) {
+	var count int64
+	err := DB.
+		Model(&models.GroupSettings{}).
+		Count(&count).
+		Error
+	if err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
+func GetDailyUserCount() (int64, error) {
+	var count int64
+	err := DB.
+		Model(&models.User{}).
+		Where("DATE(last_used) = DATE(NOW())").
+		Count(&count).
+		Error
+	if err != nil {
+		return 0, err
+	}
+	return count, nil
+}

database/user.go

@@ -0,0 +1,38 @@
+package database
+
+import "govd/models"
+
+func GetUser(
+	userID int64,
+) (*models.User, error) {
+	var user models.User
+	err := DB.
+		Where(&models.User{
+			UserID: userID,
+		}).
+		FirstOrCreate(&user).
+		Error
+	if err != nil {
+		return nil, err
+	}
+	go UpdateUserStatus(userID)
+	return &user, nil
+}
+
+func UpdateUserStatus(
+	userID int64,
+) error {
+	err := DB.
+		Model(&models.User{}).
+		Where(&models.User{
+			UserID: userID,
+		}).
+		Updates(&models.User{
+			LastUsed: DB.NowFunc(),
+		}).
+		Error
+	if err != nil {
+		return err
+	}
+	return nil
+}

enums/chat_type.go

@@ -0,0 +1,8 @@
+package enums
+
+type ChatType string
+
+const (
+	ChatTypePrivate ChatType = "private"
+	ChatTypeGroup   ChatType = "group"
+)

enums/extractor_category.go

@@ -0,0 +1,9 @@
+package enums
+
+type ExtractorCategory string
+
+const (
+	ExtractorCategorySocial    ExtractorCategory = "social"
+	ExtractorCategoryStreaming ExtractorCategory = "streaming"
+	ExtractorCategoryMusic     ExtractorCategory = "music"
+)

enums/extractor_type.go

@@ -0,0 +1,7 @@
+package enums
+
+type ExtractorType string
+
+const (
+	ExtractorTypeSingle ExtractorType = "single"
+)

enums/media_codec.go

@@ -0,0 +1,17 @@
+package enums
+
+type MediaCodec string
+
+const (
+	MediaCodecAVC    MediaCodec = "avc"
+	MediaCodecHEVC   MediaCodec = "hevc"
+	MediaCodecVP9    MediaCodec = "vp9"
+	MediaCodecVP8    MediaCodec = "vp8"
+	MediaCodecAV1    MediaCodec = "av1"
+	MediaCodecAAC    MediaCodec = "aac"
+	MediaCodecOpus   MediaCodec = "opus"
+	MediaCodecVorbis MediaCodec = "vorbis"
+	MediaCodecMP3    MediaCodec = "mp3"
+	MediaCodecFLAC   MediaCodec = "flac"
+	MediaCodecWebP   MediaCodec = "webp"
+)

enums/media_type.go

@@ -0,0 +1,9 @@
+package enums
+
+type MediaType string
+
+const (
+	MediaTypeVideo MediaType = "video"
+	MediaTypeAudio MediaType = "audio"
+	MediaTypePhoto MediaType = "photo"
+)

ext/instagram/main.go

@@ -0,0 +1,169 @@
+package instagram
+
+import (
+	"crypto/tls"
+	"fmt"
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+	"io"
+	"net/http"
+	"regexp"
+
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+)
+
+// as a public service, we can't use the official API
+// so we use igram.world API, a third-party service
+// that provides a similar functionality
+// feel free to open PR, if you want to
+// add support for the official Instagram API
+
+const (
+	apiHostname  = "api.igram.world"
+	apiKey       = "aaeaf2805cea6abef3f9d2b6a666fce62fd9d612a43ab772bb50ce81455112e0"
+	apiTimestamp = "1742201548873"
+
+	// todo: Implement a proper way
+	// to get the API key and timestamp
+)
+
+var HTTPClient = &http.Client{
+	Transport: &http3.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+		QUICConfig: &quic.Config{
+			MaxIncomingStreams: -1,
+			EnableDatagrams:    true,
+		},
+	},
+}
+
+var Extractor = &models.Extractor{
+	Name:       "Instagram",
+	CodeName:   "instagram",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https:\/\/www\.instagram\.com\/(reel|p|tv)\/(?P<id>[a-zA-Z0-9_-]+)`),
+	IsRedirect: false,
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		mediaList, err := MediaListFromAPI(ctx, false)
+		return &models.ExtractorResponse{
+			MediaList: mediaList,
+		}, err
+	},
+}
+
+var StoriesExtractor = &models.Extractor{
+	Name:       "Instagram Stories",
+	CodeName:   "instagram:stories",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https:\/\/www\.instagram\.com\/stories\/[a-zA-Z0-9._]+\/(?P<id>\d+)`),
+	IsRedirect: false,
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		mediaList, err := MediaListFromAPI(ctx, true)
+		return &models.ExtractorResponse{
+			MediaList: mediaList,
+		}, err
+	},
+}
+
+func MediaListFromAPI(
+	ctx *models.DownloadContext,
+	stories bool,
+) ([]*models.Media, error) {
+	var mediaList []*models.Media
+	postURL := ctx.MatchedContentURL
+	details, err := GetVideoAPI(postURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get post: %w", err)
+	}
+	var caption string
+	if !stories {
+		caption, err = GetPostCaption(postURL)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get caption: %w", err)
+		}
+	}
+	for _, item := range details.Items {
+		media := ctx.Extractor.NewMedia(
+			ctx.MatchedContentID,
+			ctx.MatchedContentURL,
+		)
+		media.SetCaption(caption)
+		urlObj := item.URL[0]
+		contentURL, err := GetCDNURL(urlObj.URL)
+		if err != nil {
+			return nil, err
+		}
+		thumbnailURL, err := GetCDNURL(item.Thumb)
+		if err != nil {
+			return nil, err
+		}
+		fileExt := urlObj.Ext
+		formatID := urlObj.Type
+		switch fileExt {
+		case "mp4":
+			media.AddFormat(&models.MediaFormat{
+				Type:       enums.MediaTypeVideo,
+				FormatID:   formatID,
+				URL:        []string{contentURL},
+				VideoCodec: enums.MediaCodecAVC,
+				AudioCodec: enums.MediaCodecAAC,
+				Thumbnail:  []string{thumbnailURL},
+			},
+			)
+		case "jpg", "webp", "heic", "jpeg":
+			media.AddFormat(&models.MediaFormat{
+				Type:     enums.MediaTypePhoto,
+				FormatID: formatID,
+				URL:      []string{contentURL},
+			})
+		default:
+			return nil, fmt.Errorf("unknown format: %s", fileExt)
+		}
+		mediaList = append(mediaList, media)
+	}
+
+	return mediaList, nil
+}
+
+func GetVideoAPI(contentURL string) (*IGramResponse, error) {
+	apiURL := fmt.Sprintf(
+		"https://%s/api/convert",
+		apiHostname,
+	)
+	payload, err := BuildSignedPayload(contentURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build signed payload: %w", err)
+	}
+	req, err := http.NewRequest("POST", apiURL, payload)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", util.ChromeUA)
+
+	resp, err := HTTPClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get response: %s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	response, err := ParseIGramResponse(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+	return response, nil
+}

ext/instagram/models.go

@@ -0,0 +1,19 @@
+package instagram
+
+type IGramResponse struct {
+	Items []*IGramMedia `json:"items"`
+}
+
+type IGramMedia struct {
+	URL       []*MediaURL `json:"url"`
+	Thumb     string      `json:"thumb"`
+	Hosting   string      `json:"hosting"`
+	Timestamp int         `json:"timestamp"`
+}
+
+type MediaURL struct {
+	URL  string `json:"url"`
+	Name string `json:"name"`
+	Type string `json:"type"`
+	Ext  string `json:"ext"`
+}

ext/instagram/util.go

@@ -0,0 +1,139 @@
+package instagram
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"govd/util"
+	"html"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+)
+
+var captionPattern = regexp.MustCompile(
+	`(?s)<meta property="og:title" content=".*?: &quot;(.*?)&quot;"`,
+)
+
+func BuildSignedPayload(contentURL string) (io.Reader, error) {
+	timestamp := fmt.Sprintf("%d", time.Now().UnixMilli())
+	hash := sha256.New()
+	_, err := io.WriteString(
+		hash,
+		contentURL+timestamp+apiKey,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("error writing to SHA256 hash: %w", err)
+	}
+	secretBytes := hash.Sum(nil)
+	secretString := fmt.Sprintf("%x", secretBytes)
+	secretString = strings.ToLower(secretString)
+	payload := map[string]string{
+		"url":  contentURL,
+		"ts":   timestamp,
+		"_ts":  apiTimestamp,
+		"_tsc": "0", // ?
+		"_s":   secretString,
+	}
+	parsedPayload, err := json.Marshal(payload)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling payload: %w", err)
+	}
+	reader := strings.NewReader(string(parsedPayload))
+	return reader, nil
+}
+
+func ParseIGramResponse(body []byte) (*IGramResponse, error) {
+	var rawResponse interface{}
+	if err := json.Unmarshal(body, &rawResponse); err != nil {
+		return nil, fmt.Errorf("failed to decode response: %w", err)
+	}
+	switch rawResponse.(type) {
+	case []interface{}:
+		// array of IGramMedia
+		var media []*IGramMedia
+		if err := json.Unmarshal(body, &media); err != nil {
+			return nil, fmt.Errorf("failed to decode response: %w", err)
+		}
+		return &IGramResponse{
+			Items: media,
+		}, nil
+	case map[string]interface{}:
+		// single IGramMedia
+		var media IGramMedia
+		if err := json.Unmarshal(body, &media); err != nil {
+			return nil, fmt.Errorf("failed to decode response: %w", err)
+		}
+		return &IGramResponse{
+			Items: []*IGramMedia{&media},
+		}, nil
+	default:
+		return nil, fmt.Errorf("unexpected response type: %T", rawResponse)
+	}
+}
+
+func GetCDNURL(contentURL string) (string, error) {
+	parsedUrl, err := url.Parse(contentURL)
+	if err != nil {
+		return "", fmt.Errorf("can't parse igram URL: %v", err)
+	}
+	queryParams, err := url.ParseQuery(parsedUrl.RawQuery)
+	if err != nil {
+		return "", fmt.Errorf("can't unescape igram URL: %v", err)
+	}
+	cdnURL := queryParams.Get("uri")
+	return cdnURL, nil
+}
+
+func GetPostCaption(
+	postURL string,
+) (string, error) {
+	req, err := http.NewRequest(
+		http.MethodGet,
+		postURL,
+		nil,
+	)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("User-Agent", util.ChromeUA)
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "it-IT,it;q=0.8,en-US;q=0.5,en;q=0.3")
+	req.Header.Set("Referer", "https://www.instagram.com/accounts/onetap/?next=%2F")
+	req.Header.Set("Alt-Used", "www.instagram.com")
+	req.Header.Set("Connection", "keep-alive")
+	req.Header.Set("Cookie", `csrftoken=Ib2Zuvf1y9HkDwXFxkdang; sessionid=8569455296%3AIFQiov2eYfTdSd%3A19%3AAYfVHnaxecWGWhyzxvz60vu5qLn05DyKgN_tTZUXTA; ds_user_id=8569455296; mid=Z_j1vQAEAAGVUE3KuxMR7vBonGBw; ig_did=BC48C8B7-D71B-49EF-8195-F9DE37A57B49; rur="CLN\0548569455296\0541775905137:01f7ebda5b896815e9279bb86a572db6bdc8ebccf3e1f8d5327e2bc5ca187fd5cd932b66"; wd=513x594; datr=x_X4Z_CHqpwtjaRKq7PtCNu3`)
+	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	req.Header.Set("Sec-Fetch-Dest", "document")
+	req.Header.Set("Sec-Fetch-Mode", "navigate")
+	req.Header.Set("Sec-Fetch-Site", "same-origin")
+	req.Header.Set("Priority", "u=0, i")
+	req.Header.Set("Pragma", "no-cache")
+	req.Header.Set("Cache-Control", "no-cache")
+	req.Header.Set("TE", "trailers")
+
+	resp, err := HTTPClient.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to get response: %s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	matches := captionPattern.FindStringSubmatch(string(body))
+	if len(matches) < 2 {
+		// post has no caption most likely
+		return "", nil
+	} else {
+		return html.UnescapeString(matches[1]), nil
+	}
+}

ext/main.go

@@ -0,0 +1,24 @@
+package ext
+
+import (
+	"govd/ext/instagram"
+	"govd/ext/pinterest"
+	"govd/ext/reddit"
+	"govd/ext/tiktok"
+	"govd/ext/twitter"
+	"govd/models"
+)
+
+var List = []*models.Extractor{
+	tiktok.Extractor,
+	tiktok.VMExtractor,
+	instagram.Extractor,
+	instagram.StoriesExtractor,
+	twitter.Extractor,
+	twitter.ShortExtractor,
+	pinterest.Extractor,
+	pinterest.ShortExtractor,
+	reddit.Extractor,
+	reddit.ShortExtractor,
+	// todo: add every ext lol
+}

ext/pinterest/main.go

@@ -0,0 +1,172 @@
+package pinterest
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+)
+
+const (
+	pinResourceEndpoint = "https://www.pinterest.com/resource/PinResource/get/"
+	shortenerAPIFormat  = "https://api.pinterest.com/url_shortener/%s/redirect/"
+)
+
+var ShortExtractor = &models.Extractor{
+	Name:       "Pinterest (Short)",
+	CodeName:   "pinterest:short",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?://(\w+\.)?pin\.\w+/(?P<id>\w+)`),
+	IsRedirect: true,
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		shortURL := fmt.Sprintf(shortenerAPIFormat, ctx.MatchedContentID)
+		location, err := util.GetLocationURL(shortURL, "")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get real url: %w", err)
+		}
+		return &models.ExtractorResponse{
+			URL: location,
+		}, nil
+	},
+}
+
+var Extractor = &models.Extractor{
+	Name:       "Pinterest",
+	CodeName:   "pinterest",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?://(\w+\.)?pinterest[\.\w]+/pin/(?P<id>\d+)`),
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		media, err := ExtractPinMedia(ctx)
+		if err != nil {
+			return nil, err
+		}
+		return &models.ExtractorResponse{
+			MediaList: media,
+		}, nil
+	},
+}
+
+func ExtractPinMedia(ctx *models.DownloadContext) ([]*models.Media, error) {
+	pinID := ctx.MatchedContentID
+	contentURL := ctx.MatchedContentURL
+
+	pinData, err := GetPinData(pinID)
+	if err != nil {
+		return nil, err
+	}
+
+	media := ctx.Extractor.NewMedia(pinID, contentURL)
+	media.SetCaption(pinData.Title)
+
+	if pinData.Videos != nil && pinData.Videos.VideoList != nil {
+		formats, err := ParseVideoObject(pinData.Videos)
+		if err != nil {
+			return nil, err
+		}
+		for _, format := range formats {
+			media.AddFormat(format)
+		}
+		return []*models.Media{media}, nil
+	}
+
+	if pinData.StoryPinData != nil && len(pinData.StoryPinData.Pages) > 0 {
+		for _, page := range pinData.StoryPinData.Pages {
+			for _, block := range page.Blocks {
+				if block.BlockType == 3 && block.Video != nil { // blockType 3 = Video
+					formats, err := ParseVideoObject(block.Video)
+					if err != nil {
+						return nil, err
+					}
+					for _, format := range formats {
+						media.AddFormat(format)
+					}
+					return []*models.Media{media}, nil
+				}
+			}
+		}
+	}
+
+	if pinData.Images != nil && pinData.Images.Orig != nil {
+		imageURL := pinData.Images.Orig.URL
+		media.AddFormat(&models.MediaFormat{
+			FormatID: "photo",
+			Type:     enums.MediaTypePhoto,
+			URL:      []string{imageURL},
+		})
+		return []*models.Media{media}, nil
+	} else if pinData.StoryPinData != nil && len(pinData.StoryPinData.Pages) > 0 {
+		for _, page := range pinData.StoryPinData.Pages {
+			if page.Image != nil && page.Image.Images.Originals != nil {
+				media.AddFormat(&models.MediaFormat{
+					FormatID: "photo",
+					Type:     enums.MediaTypePhoto,
+					URL:      []string{page.Image.Images.Originals.URL},
+				})
+				return []*models.Media{media}, nil
+			}
+		}
+	}
+
+	if pinData.Embed != nil && pinData.Embed.Type == "gif" {
+		media.AddFormat(&models.MediaFormat{
+			FormatID:   "gif",
+			Type:       enums.MediaTypeVideo,
+			VideoCodec: enums.MediaCodecAVC,
+			URL:        []string{pinData.Embed.Src},
+		})
+		return []*models.Media{media}, nil
+	}
+
+	return nil, fmt.Errorf("no media found for pin ID: %s", pinID)
+}
+
+func GetPinData(pinID string) (*PinData, error) {
+	params := BuildPinRequestParams(pinID)
+
+	req, err := http.NewRequest("GET", pinResourceEndpoint, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	q := req.URL.Query()
+	for key, value := range params {
+		q.Add(key, value)
+	}
+	req.URL.RawQuery = q.Encode()
+	req.Header.Set("User-Agent", util.ChromeUA)
+
+	// fix 403 error
+	req.Header.Set("X-Pinterest-PWS-Handler", "www/[username].js")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("bad response: %s", resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var pinResponse PinResponse
+	err = json.Unmarshal(body, &pinResponse)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+
+	return &pinResponse.ResourceResponse.Data, nil
+}

ext/pinterest/models.go

@@ -0,0 +1,62 @@
+package pinterest
+
+type PinResponse struct {
+	ResourceResponse struct {
+		Data PinData `json:"data"`
+	} `json:"resource_response"`
+}
+
+type PinData struct {
+	ID           string    `json:"id"`
+	Title        string    `json:"title"`
+	Description  string    `json:"description"`
+	Images       *Images   `json:"images,omitempty"`
+	Videos       *Videos   `json:"videos,omitempty"`
+	StoryPinData *StoryPin `json:"story_pin_data,omitempty"`
+	Embed        *Embed    `json:"embed,omitempty"`
+}
+
+type Images struct {
+	Orig *ImageObject `json:"orig"`
+}
+
+type ImageObject struct {
+	URL    string `json:"url"`
+	Width  int    `json:"width"`
+	Height int    `json:"height"`
+}
+
+type Videos struct {
+	VideoList map[string]*VideoObject `json:"video_list"`
+}
+
+type VideoObject struct {
+	URL       string `json:"url"`
+	Width     int64  `json:"width"`
+	Height    int64  `json:"height"`
+	Duration  int64  `json:"duration"`
+	Thumbnail string `json:"thumbnail"`
+}
+
+type StoryPin struct {
+	Pages []Page `json:"pages"`
+}
+
+type Page struct {
+	Blocks []Block `json:"blocks"`
+	Image  *struct {
+		Images struct {
+			Originals *ImageObject `json:"originals"`
+		} `json:"images"`
+	} `json:"image,omitempty"`
+}
+
+type Block struct {
+	BlockType int     `json:"block_type"`
+	Video     *Videos `json:"video,omitempty"`
+}
+
+type Embed struct {
+	Type string `json:"type"`
+	Src  string `json:"src"`
+}

ext/pinterest/util.go

@@ -0,0 +1,55 @@
+package pinterest
+
+import (
+	"encoding/json"
+	"fmt"
+	"govd/enums"
+	"govd/models"
+	"govd/util/parser"
+)
+
+func ParseVideoObject(videoObj *Videos) ([]*models.MediaFormat, error) {
+	var formats []*models.MediaFormat
+
+	for key, video := range videoObj.VideoList {
+		if key != "HLS" {
+			formats = append(formats, &models.MediaFormat{
+				FormatID:   key,
+				URL:        []string{video.URL},
+				Type:       enums.MediaTypeVideo,
+				VideoCodec: enums.MediaCodecAVC,
+				AudioCodec: enums.MediaCodecAAC,
+				Width:      video.Width,
+				Height:     video.Height,
+				Duration:   video.Duration / 1000,
+				Thumbnail:  []string{video.Thumbnail},
+			})
+		} else {
+			hlsFormats, err := parser.ParseM3U8FromURL(video.URL)
+			if err != nil {
+				return nil, fmt.Errorf("failed to extract hls formats: %w", err)
+			}
+			for _, hlsFormat := range hlsFormats {
+				hlsFormat.Duration = video.Duration / 1000
+				hlsFormat.Thumbnail = []string{video.Thumbnail}
+				formats = append(formats, hlsFormat)
+			}
+		}
+	}
+
+	return formats, nil
+}
+
+func BuildPinRequestParams(pinID string) map[string]string {
+	options := map[string]interface{}{
+		"options": map[string]interface{}{
+			"field_set_key": "unauth_react_main_pin",
+			"id":            pinID,
+		},
+	}
+
+	jsonData, _ := json.Marshal(options)
+	return map[string]string{
+		"data": string(jsonData),
+	}
+}

ext/reddit/main.go

@@ -0,0 +1,267 @@
+package reddit
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+)
+
+var HTTPClient = &http.Client{
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		if len(via) >= 10 {
+			return fmt.Errorf("stopped after 10 redirects")
+		}
+		return nil
+	},
+}
+
+var ShortExtractor = &models.Extractor{
+	Name:       "Reddit (Short)",
+	CodeName:   "reddit:short",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?://(?P<host>(?:\w+\.)?reddit(?:media)?\.com)/(?P<slug>(?:(?:r|user)/[^/]+/)?s/(?P<id>[^/?#&]+))`),
+	IsRedirect: true,
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		req, err := http.NewRequest("GET", ctx.MatchedContentURL, nil)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		req.Header.Set("User-Agent", util.ChromeUA)
+		cookies, err := util.ParseCookieFile("reddit.txt")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get cookies: %w", err)
+		}
+		for _, cookie := range cookies {
+			req.AddCookie(cookie)
+		}
+
+		res, err := HTTPClient.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("failed to send request: %w", err)
+		}
+		defer res.Body.Close()
+
+		location := res.Request.URL.String()
+
+		return &models.ExtractorResponse{
+			URL: location,
+		}, nil
+	},
+}
+
+var Extractor = &models.Extractor{
+	Name:       "Reddit",
+	CodeName:   "reddit",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?://(?P<host>(?:\w+\.)?reddit(?:media)?\.com)/(?P<slug>(?:(?:r|user)/[^/]+/)?comments/(?P<id>[^/?#&]+))`),
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		mediaList, err := MediaListFromAPI(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get media: %w", err)
+		}
+		return &models.ExtractorResponse{
+			MediaList: mediaList,
+		}, nil
+	},
+}
+
+func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
+	host := ctx.MatchedGroups["host"]
+	slug := ctx.MatchedGroups["slug"]
+
+	contentID := ctx.MatchedContentID
+	contentURL := ctx.MatchedContentURL
+
+	manifest, err := GetRedditData(host, slug)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(manifest) == 0 || len(manifest[0].Data.Children) == 0 {
+		return nil, fmt.Errorf("no data found in response")
+	}
+
+	data := manifest[0].Data.Children[0].Data
+	title := data.Title
+	isNsfw := data.Over18
+	var mediaList []*models.Media
+
+	if !data.IsVideo {
+		// check for single photo
+		if data.Preview != nil && len(data.Preview.Images) > 0 {
+			media := ctx.Extractor.NewMedia(contentID, contentURL)
+			media.SetCaption(title)
+			if isNsfw {
+				media.NSFW = true
+			}
+
+			image := data.Preview.Images[0]
+
+			// check for video preview (GIF)
+			if data.Preview.RedditVideoPreview != nil {
+				formats, err := GetHLSFormats(
+					data.Preview.RedditVideoPreview.FallbackURL,
+					image.Source.URL,
+					data.Preview.RedditVideoPreview.Duration,
+				)
+				if err != nil {
+					return nil, err
+				}
+
+				for _, format := range formats {
+					media.AddFormat(format)
+				}
+
+				mediaList = append(mediaList, media)
+				return mediaList, nil
+			}
+
+			// check for MP4 variant (animated GIF)
+			if image.Variants.MP4 != nil {
+				media.AddFormat(&models.MediaFormat{
+					FormatID:   "gif",
+					Type:       enums.MediaTypeVideo,
+					VideoCodec: enums.MediaCodecAVC,
+					AudioCodec: enums.MediaCodecAAC,
+					URL:        []string{util.FixURL(image.Variants.MP4.Source.URL)},
+					Thumbnail:  []string{util.FixURL(image.Source.URL)},
+				})
+
+				mediaList = append(mediaList, media)
+				return mediaList, nil
+			}
+
+			// regular photo
+			media.AddFormat(&models.MediaFormat{
+				FormatID: "photo",
+				Type:     enums.MediaTypePhoto,
+				URL:      []string{util.FixURL(image.Source.URL)},
+			})
+
+			mediaList = append(mediaList, media)
+			return mediaList, nil
+		}
+
+		// check for gallery/collection
+		if len(data.MediaMetadata) > 0 {
+			for key, obj := range data.MediaMetadata {
+				if obj.E == "Image" {
+					media := ctx.Extractor.NewMedia(key, contentURL)
+					media.SetCaption(title)
+					if isNsfw {
+						media.NSFW = true
+					}
+
+					media.AddFormat(&models.MediaFormat{
+						FormatID: "photo",
+						Type:     enums.MediaTypePhoto,
+						URL:      []string{util.FixURL(obj.S.U)},
+					})
+
+					mediaList = append(mediaList, media)
+				}
+			}
+
+			return mediaList, nil
+		}
+	} else {
+		// video
+		media := ctx.Extractor.NewMedia(contentID, contentURL)
+		media.SetCaption(title)
+		if isNsfw {
+			media.NSFW = true
+		}
+
+		var redditVideo *RedditVideo
+
+		if data.Media != nil && data.Media.RedditVideo != nil {
+			redditVideo = data.Media.RedditVideo
+		} else if data.SecureMedia != nil && data.SecureMedia.RedditVideo != nil {
+			redditVideo = data.SecureMedia.RedditVideo
+		}
+
+		if redditVideo != nil {
+			thumbnail := data.Thumbnail
+
+			if (thumbnail == "nsfw" || thumbnail == "spoiler") && data.Preview != nil && len(data.Preview.Images) > 0 {
+				thumbnail = data.Preview.Images[0].Source.URL
+			}
+
+			formats, err := GetHLSFormats(
+				redditVideo.FallbackURL,
+				thumbnail,
+				redditVideo.Duration,
+			)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, format := range formats {
+				media.AddFormat(format)
+			}
+
+			mediaList = append(mediaList, media)
+			return mediaList, nil
+		}
+	}
+
+	return mediaList, nil
+}
+
+func GetRedditData(host string, slug string) (RedditResponse, error) {
+	url := fmt.Sprintf("https://%s/%s/.json", host, slug)
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", util.ChromeUA)
+	cookies, err := util.ParseCookieFile("reddit.txt")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cookies: %w", err)
+	}
+	for _, cookie := range cookies {
+		req.AddCookie(cookie)
+	}
+
+	res, err := HTTPClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		// try with alternative domain
+		altHost := "old.reddit.com"
+		if host == "old.reddit.com" {
+			altHost = "www.reddit.com"
+		}
+
+		return GetRedditData(altHost, slug)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var response RedditResponse
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+
+	return response, nil
+}

ext/reddit/models.go

@@ -0,0 +1,74 @@
+package reddit
+
+type RedditResponse []struct {
+	Data struct {
+		Children []struct {
+			Data PostData `json:"data"`
+		} `json:"children"`
+	} `json:"data"`
+}
+
+type PostData struct {
+	ID            string                   `json:"id"`
+	Title         string                   `json:"title"`
+	IsVideo       bool                     `json:"is_video"`
+	Thumbnail     string                   `json:"thumbnail"`
+	Media         *Media                   `json:"media"`
+	Preview       *Preview                 `json:"preview"`
+	MediaMetadata map[string]MediaMetadata `json:"media_metadata"`
+	SecureMedia   *Media                   `json:"secure_media"`
+	Over18        bool                     `json:"over_18"`
+}
+
+type Media struct {
+	RedditVideo *RedditVideo `json:"reddit_video"`
+}
+
+type RedditVideo struct {
+	FallbackURL      string `json:"fallback_url"`
+	HLSURL           string `json:"hls_url"`
+	DashURL          string `json:"dash_url"`
+	Duration         int64  `json:"duration"`
+	Height           int64  `json:"height"`
+	Width            int64  `json:"width"`
+	ScrubberMediaURL string `json:"scrubber_media_url"`
+}
+
+type Preview struct {
+	Images             []Image             `json:"images"`
+	RedditVideoPreview *RedditVideoPreview `json:"reddit_video_preview"`
+}
+
+type Image struct {
+	Source   ImageSource   `json:"source"`
+	Variants ImageVariants `json:"variants"`
+}
+
+type ImageSource struct {
+	URL    string `json:"url"`
+	Width  int64  `json:"width"`
+	Height int64  `json:"height"`
+}
+
+type ImageVariants struct {
+	MP4 *MP4Variant `json:"mp4"`
+}
+
+type MP4Variant struct {
+	Source ImageSource `json:"source"`
+}
+
+type RedditVideoPreview struct {
+	FallbackURL string `json:"fallback_url"`
+	Duration    int64  `json:"duration"`
+}
+
+type MediaMetadata struct {
+	Status string `json:"status"`
+	E      string `json:"e"`
+	S      struct {
+		U string `json:"u"`
+		X int64  `json:"x"`
+		Y int64  `json:"y"`
+	} `json:"s"`
+}

ext/reddit/util.go

@@ -0,0 +1,39 @@
+package reddit
+
+import (
+	"fmt"
+	"govd/models"
+	"govd/util"
+	"govd/util/parser"
+	"regexp"
+)
+
+const (
+	hlsURLFormat = "https://v.redd.it/%s/HLSPlaylist.m3u8"
+)
+
+var videoURLPattern = regexp.MustCompile(`https?://v\.redd\.it/([^/]+)`)
+
+func GetHLSFormats(videoURL string, thumbnail string, duration int64) ([]*models.MediaFormat, error) {
+	matches := videoURLPattern.FindStringSubmatch(videoURL)
+	if len(matches) < 2 {
+		return nil, nil
+	}
+
+	videoID := matches[1]
+	hlsURL := fmt.Sprintf(hlsURLFormat, videoID)
+
+	formats, err := parser.ParseM3U8FromURL(hlsURL)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, format := range formats {
+		format.Duration = duration
+		if thumbnail != "" {
+			format.Thumbnail = []string{util.FixURL(thumbnail)}
+		}
+	}
+
+	return formats, nil
+}

ext/tiktok/main.go

@@ -0,0 +1,184 @@
+package tiktok
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+
+	"github.com/quic-go/quic-go"
+	"github.com/quic-go/quic-go/http3"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+)
+
+const (
+	apiHostname        = "api16-normal-c-useast1a.tiktokv.com"
+	installationID     = "7127307272354596614"
+	appName            = "musical_ly"
+	appID              = "1233"
+	appVersion         = "37.1.4"
+	manifestAppVersion = "2023508030"
+	packageID          = "com.zhiliaoapp.musically/" + manifestAppVersion
+	appUserAgent       = packageID + " (Linux; U; Android 13; en_US; Pixel 7; Build/TD1A.220804.031; Cronet/58.0.2991.0)"
+)
+
+var HTTPClient = &http.Client{
+	Transport: &http3.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+		QUICConfig: &quic.Config{
+			MaxIncomingStreams: -1,
+			EnableDatagrams:    true,
+		},
+	},
+}
+
+var VMExtractor = &models.Extractor{
+	Name:       "TikTok VM",
+	CodeName:   "tiktokvm",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https:\/\/((?:vm|vt|www)\.)?(vx)?tiktok\.com\/(?:t\/)?(?P<id>[a-zA-Z0-9]+)`),
+	IsRedirect: true,
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		location, err := util.GetLocationURL(ctx.MatchedContentURL, "")
+		if err != nil {
+			return nil, fmt.Errorf("failed to get url location: %w", err)
+		}
+		return &models.ExtractorResponse{
+			URL: location,
+		}, nil
+	},
+}
+
+var Extractor = &models.Extractor{
+	Name:       "TikTok",
+	CodeName:   "tiktok",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?:\/\/((www|m)\.)?(vx)?tiktok\.com\/((?:embed|@[\w\.-]+)\/)?(v(ideo)?|p(hoto)?)\/(?P<id>[0-9]+)`),
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		mediaList, err := MediaListFromAPI(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get media: %w", err)
+		}
+		return &models.ExtractorResponse{
+			MediaList: mediaList,
+		}, nil
+	},
+}
+
+func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
+	var mediaList []*models.Media
+
+	details, err := GetVideoAPI(ctx.MatchedContentID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get from api: %w", err)
+	}
+	caption := details.Desc
+	isImageSlide := details.ImagePostInfo != nil
+	if !isImageSlide {
+		media := ctx.Extractor.NewMedia(
+			ctx.MatchedContentID,
+			ctx.MatchedContentURL,
+		)
+		media.SetCaption(caption)
+		video := details.Video
+
+		// generic PlayAddr
+		if video.PlayAddr != nil {
+			format, err := ParsePlayAddr(video, video.PlayAddr)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse playaddr: %w", err)
+			}
+			media.AddFormat(format)
+		}
+		// hevc PlayAddr
+		if video.PlayAddrBytevc1 != nil {
+			format, err := ParsePlayAddr(video, video.PlayAddrBytevc1)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse playaddr: %w", err)
+			}
+			media.AddFormat(format)
+		}
+		// h264 PlayAddr
+		if video.PlayAddrH264 != nil {
+			format, err := ParsePlayAddr(video, video.PlayAddrH264)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse playaddr: %w", err)
+			}
+			media.AddFormat(format)
+		}
+		mediaList = append(mediaList, media)
+	} else {
+		images := details.ImagePostInfo.Images
+		for _, image := range images {
+			media := ctx.Extractor.NewMedia(
+				ctx.MatchedContentID,
+				ctx.MatchedContentURL,
+			)
+			media.SetCaption(caption)
+			media.AddFormat(&models.MediaFormat{
+				FormatID: "image",
+				Type:     enums.MediaTypePhoto,
+				URL:      image.DisplayImage.URLList,
+			})
+			mediaList = append(mediaList, media)
+		}
+	}
+	return mediaList, nil
+}
+
+func GetVideoAPI(awemeID string) (*AwemeDetails, error) {
+	apiURL := fmt.Sprintf(
+		"https://%s/aweme/v1/multi/aweme/detail/",
+		apiHostname,
+	)
+	queryParams, err := BuildAPIQuery()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build api query: %w", err)
+	}
+	postData := BuildPostData(awemeID)
+
+	req, err := http.NewRequest(
+		http.MethodPost,
+		apiURL,
+		postData,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.URL.RawQuery = queryParams.Encode()
+	req.Header.Set("User-Agent", appUserAgent)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("X-Argus", "")
+
+	resp, err := HTTPClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	var data *Response
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+	}
+	videoData, err := FindVideoData(data, awemeID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find video data: %w", err)
+	}
+	return videoData, nil
+}

ext/tiktok/models.go

@@ -0,0 +1,65 @@
+package tiktok
+
+type Response struct {
+	AwemeDetails []AwemeDetails `json:"aweme_details"`
+	StatusCode   int            `json:"status_code"`
+	StatusMsg    string         `json:"status_msg"`
+}
+
+type Cover struct {
+	Height    int64    `json:"height"`
+	URI       string   `json:"uri"`
+	URLList   []string `json:"url_list"`
+	URLPrefix any      `json:"url_prefix"`
+	Width     int64    `json:"width"`
+}
+
+type PlayAddr struct {
+	DataSize int64    `json:"data_size"`
+	FileCs   string   `json:"file_cs"`
+	FileHash string   `json:"file_hash"`
+	Height   int64    `json:"height"`
+	URI      string   `json:"uri"`
+	URLKey   string   `json:"url_key"`
+	URLList  []string `json:"url_list"`
+	Width    int64    `json:"width"`
+}
+
+type Image struct {
+	DisplayImage *DisplayImage `json:"display_image"`
+}
+
+type DisplayImage struct {
+	Height    int      `json:"height"`
+	URI       string   `json:"uri"`
+	URLList   []string `json:"url_list"`
+	URLPrefix any      `json:"url_prefix"`
+	Width     int      `json:"width"`
+}
+
+type ImagePostInfo struct {
+	Images      []Image `json:"images"`
+	MusicVolume float64 `json:"music_volume"`
+	PostExtra   string  `json:"post_extra"`
+	Title       string  `json:"title"`
+}
+
+type Video struct {
+	CdnURLExpired   int64     `json:"cdn_url_expired"`
+	Cover           Cover     `json:"cover"`
+	Duration        int64     `json:"duration"`
+	HasWatermark    bool      `json:"has_watermark"`
+	Height          int64     `json:"height"`
+	PlayAddr        *PlayAddr `json:"play_addr"`
+	PlayAddrBytevc1 *PlayAddr `json:"play_addr_bytevc1"`
+	PlayAddrH264    *PlayAddr `json:"play_addr_h264"`
+	Width           int64     `json:"width"`
+}
+
+type AwemeDetails struct {
+	AwemeID       string         `json:"aweme_id"`
+	AwemeType     int            `json:"aweme_type"`
+	Desc          string         `json:"desc"`
+	Video         *Video         `json:"video"`
+	ImagePostInfo *ImagePostInfo `json:"image_post_info"`
+}

ext/tiktok/util.go

@@ -0,0 +1,177 @@
+package tiktok
+
+import (
+	"crypto/rand"
+	"fmt"
+	"math/big"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+
+	"github.com/google/uuid"
+)
+
+func BuildAPIQuery() (url.Values, error) {
+	requestTicket := strconv.Itoa(int(time.Now().Unix()) * 1000)
+	clientDeviceID := uuid.New().String()
+	versionCode, err := GetAppVersionCode(appVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get app version code: %w", err)
+	}
+	return url.Values{
+		"device_platform":       []string{"android"},
+		"os":                    []string{"android"},
+		"ssmix":                 []string{"0"}, // what is this?
+		"_rticket":              []string{requestTicket},
+		"cdid":                  []string{clientDeviceID},
+		"channel":               []string{"googleplay"},
+		"aid":                   []string{appID},
+		"app_name":              []string{appName},
+		"version_code":          []string{versionCode},
+		"version_name":          []string{appVersion},
+		"manifest_version_code": []string{manifestAppVersion},
+		"update_version_code":   []string{manifestAppVersion},
+		"ab_version":            []string{appVersion},
+		"resolution":            []string{"1080*2400"},
+		"dpi":                   []string{"420"},
+		"device_type":           []string{"Pixel 7"},
+		"device_brand":          []string{"Google"},
+		"language":              []string{"en"},
+		"os_api":                []string{"29"},
+		"os_version":            []string{"13"},
+		"ac":                    []string{"wifi"},
+		"is_pad":                []string{"0"},
+		"current_region":        []string{"US"},
+		"app_type":              []string{"normal"},
+		"last_install_time":     []string{GetRandomInstallTime()},
+		"timezone_name":         []string{"America/New_York"},
+		"residence":             []string{"US"},
+		"app_language":          []string{"en"},
+		"timezone_offset":       []string{"-14400"},
+		"host_abi":              []string{"armeabi-v7a"},
+		"locale":                []string{"en"},
+		"ac2":                   []string{"wifi5g"},
+		"uoo":                   []string{"1"}, // what is this?
+		"carrier_region":        []string{"US"},
+		"build_number":          []string{appVersion},
+		"region":                []string{"US"},
+		"ts":                    []string{strconv.Itoa(int(time.Now().Unix()))},
+		"iid":                   []string{installationID},
+		"device_id":             []string{GetRandomDeviceID()},
+		"openudid":              []string{GetRandomUdid()},
+	}, nil
+}
+
+func ParsePlayAddr(
+	video *Video,
+	playAddr *PlayAddr,
+) (*models.MediaFormat, error) {
+	formatID := playAddr.URLKey
+	if formatID == "" {
+		return nil, errors.New("url_key not found")
+	}
+	videoCodec := enums.MediaCodecHEVC
+	if strings.Contains(formatID, "h264") {
+		videoCodec = enums.MediaCodecAVC
+	}
+	videoURL := playAddr.URLList
+	videoDuration := video.Duration / 1000
+	videoWidth := playAddr.Width
+	videoHeight := playAddr.Height
+	videoCover := &video.Cover
+	videoThumbnailURLs := videoCover.URLList
+
+	return &models.MediaFormat{
+		Type:       enums.MediaTypeVideo,
+		FormatID:   formatID,
+		URL:        videoURL,
+		VideoCodec: videoCodec,
+		AudioCodec: enums.MediaCodecAAC,
+		Duration:   videoDuration,
+		Thumbnail:  videoThumbnailURLs,
+		Width:      videoWidth,
+		Height:     videoHeight,
+	}, nil
+}
+
+func GetRandomInstallTime() string {
+	currentTime := int(time.Now().Unix())
+	minOffset := big.NewInt(86400)
+	maxOffset := big.NewInt(1123200)
+	diff := new(big.Int).Sub(maxOffset, minOffset)
+	randomOffset, _ := rand.Int(rand.Reader, diff)
+	randomOffset.Add(randomOffset, minOffset)
+	result := currentTime - int(randomOffset.Int64())
+	return strconv.Itoa(result)
+}
+
+func GetRandomUdid() string {
+	const charset = "0123456789abcdef"
+	result := make([]byte, 16)
+
+	for i := range result {
+		index, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
+		result[i] = charset[index.Int64()]
+	}
+	return string(result)
+}
+
+func GetRandomDeviceID() string {
+	minNum := big.NewInt(7250000000000000000)
+	maxNum := big.NewInt(7351147085025500000)
+	diff := new(big.Int).Sub(maxNum, minNum)
+	randNum, _ := rand.Int(rand.Reader, diff)
+	result := new(big.Int).Add(randNum, minNum)
+	return result.String()
+}
+
+func BuildPostData(awemeID string) *strings.Reader {
+	data := url.Values{
+		"aweme_ids":      []string{fmt.Sprintf("[%s]", awemeID)},
+		"request_source": []string{"0"},
+	}
+	return strings.NewReader(data.Encode())
+
+}
+
+func GetAppVersionCode(version string) (string, error) {
+	parts := strings.Split(version, ".")
+
+	var result strings.Builder
+	for _, part := range parts {
+		num, err := strconv.Atoi(part)
+		if err != nil {
+			return "", fmt.Errorf("failed to parse version part: %w", err)
+		}
+		_, err = fmt.Fprintf(&result, "%02d", num)
+		if err != nil {
+			return "", fmt.Errorf("failed to format version part: %w", err)
+		}
+	}
+	return result.String(), nil
+}
+
+func FindVideoData(
+	resp *Response,
+	expectedAwemeID string,
+) (*AwemeDetails, error) {
+	if resp.StatusCode == 2053 {
+		return nil, util.ErrUnavailable
+	}
+	if resp.AwemeDetails == nil {
+		return nil, errors.New("aweme_details is nil")
+	}
+	for _, item := range resp.AwemeDetails {
+		if item.AwemeID == expectedAwemeID {
+			return &item, nil
+		}
+	}
+	return nil, errors.New("matching aweme_id not found")
+}

ext/twitter/main.go

@@ -0,0 +1,181 @@
+package twitter
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+)
+
+const (
+	apiHostname = "x.com"
+	apiEndpoint = "https://x.com/i/api/graphql/zZXycP0V6H7m-2r0mOnFcA/TweetDetail"
+)
+
+var HTTPClient = &http.Client{}
+
+var ShortExtractor = &models.Extractor{
+	Name:       "Twitter (Short)",
+	CodeName:   "twitter:short",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?://t\.co/(?P<id>\w+)`),
+	IsRedirect: true,
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		req, err := http.NewRequest("GET", ctx.MatchedContentURL, nil)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create req: %w", err)
+		}
+		req.Header.Set("User-Agent", util.ChromeUA)
+		res, err := HTTPClient.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("failed to send request: %w", err)
+		}
+		defer res.Body.Close()
+		body, err := io.ReadAll(res.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read body: %w", err)
+		}
+		matchedURL := Extractor.URLPattern.FindStringSubmatch(string(body))
+		if matchedURL == nil {
+			return nil, fmt.Errorf("failed to find url in body")
+		}
+		return &models.ExtractorResponse{
+			URL: matchedURL[0],
+		}, nil
+	},
+}
+
+var Extractor = &models.Extractor{
+	Name:       "Twitter",
+	CodeName:   "twitter",
+	Type:       enums.ExtractorTypeSingle,
+	Category:   enums.ExtractorCategorySocial,
+	URLPattern: regexp.MustCompile(`https?:\/\/(vx)?(twitter|x)\.com\/([^\/]+)\/status\/(?P<id>\d+)`),
+
+	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
+		mediaList, err := MediaListFromAPI(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get media: %w", err)
+		}
+		return &models.ExtractorResponse{
+			MediaList: mediaList,
+		}, nil
+	},
+}
+
+func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
+	var mediaList []*models.Media
+
+	tweetData, err := GetTweetAPI(ctx.MatchedContentID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get tweet data: %w", err)
+	}
+
+	caption := CleanCaption(tweetData.FullText)
+
+	var mediaEntities []MediaEntity
+	if tweetData.ExtendedEntities != nil && len(tweetData.ExtendedEntities.Media) > 0 {
+		mediaEntities = tweetData.ExtendedEntities.Media
+	} else if tweetData.Entities != nil && len(tweetData.Entities.Media) > 0 {
+		mediaEntities = tweetData.Entities.Media
+	} else {
+		return nil, fmt.Errorf("no media found in tweet")
+	}
+
+	for _, mediaEntity := range mediaEntities {
+		media := ctx.Extractor.NewMedia(
+			ctx.MatchedContentID,
+			ctx.MatchedContentURL,
+		)
+		media.SetCaption(caption)
+
+		switch mediaEntity.Type {
+		case "video", "animated_gif":
+			formats, err := ExtractVideoFormats(&mediaEntity)
+			if err != nil {
+				return nil, err
+			}
+			for _, format := range formats {
+				media.AddFormat(format)
+			}
+		case "photo":
+			media.AddFormat(&models.MediaFormat{
+				Type:     enums.MediaTypePhoto,
+				FormatID: "photo",
+				URL:      []string{mediaEntity.MediaURLHTTPS},
+			})
+		}
+
+		if len(media.Formats) > 0 {
+			mediaList = append(mediaList, media)
+		}
+	}
+
+	return mediaList, nil
+}
+
+func GetTweetAPI(tweetID string) (*Tweet, error) {
+	cookies, err := util.ParseCookieFile("twitter.txt")
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cookies: %w", err)
+	}
+	headers := BuildAPIHeaders(cookies)
+	if headers == nil {
+		return nil, fmt.Errorf("failed to build headers. check cookies")
+	}
+	query := BuildAPIQuery(tweetID)
+
+	req, err := http.NewRequest("GET", apiEndpoint, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create req: %w", err)
+	}
+
+	for key, value := range headers {
+		req.Header.Set(key, value)
+	}
+
+	for _, cookie := range cookies {
+		req.AddCookie(cookie)
+	}
+
+	q := req.URL.Query()
+	for key, value := range query {
+		q.Add(key, value)
+	}
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := HTTPClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("invalid response code: %s", resp.Status)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read body: %w", err)
+	}
+
+	var apiResponse APIResponse
+	err = json.Unmarshal(body, &apiResponse)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+
+	tweet, err := FindTweetData(&apiResponse, tweetID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get tweet data: %w", err)
+	}
+
+	return tweet, nil
+}

ext/twitter/models.go

@@ -0,0 +1,72 @@
+package twitter
+
+type APIResponse struct {
+	Data struct {
+		ThreadedConversationWithInjectionsV2 struct {
+			Instructions []struct {
+				Entries []struct {
+					EntryID string `json:"entryId"`
+					Content struct {
+						ItemContent struct {
+							TweetResults struct {
+								Result TweetResult `json:"result"`
+							} `json:"tweet_results"`
+						} `json:"itemContent"`
+					} `json:"content"`
+				} `json:"entries"`
+			} `json:"instructions"`
+		} `json:"threaded_conversation_with_injections_v2"`
+	} `json:"data"`
+}
+
+type TweetResult struct {
+	Tweet  *Tweet `json:"tweet,omitempty"`
+	Legacy *Tweet `json:"legacy,omitempty"`
+	RestID string `json:"rest_id,omitempty"`
+	Core   *Core  `json:"core,omitempty"`
+}
+
+type Core struct {
+	UserResults struct {
+		Result struct {
+			Legacy *UserLegacy `json:"legacy,omitempty"`
+		} `json:"result"`
+	} `json:"user_results"`
+}
+
+type UserLegacy struct {
+	ScreenName string `json:"screen_name"`
+	Name       string `json:"name"`
+}
+
+type Tweet struct {
+	FullText         string            `json:"full_text"`
+	ExtendedEntities *ExtendedEntities `json:"extended_entities,omitempty"`
+	Entities         *ExtendedEntities `json:"entities,omitempty"`
+	CreatedAt        string            `json:"created_at"`
+	ID               string            `json:"id_str"`
+}
+
+type ExtendedEntities struct {
+	Media []MediaEntity `json:"media,omitempty"`
+}
+
+type MediaEntity struct {
+	Type          string     `json:"type"`
+	MediaURLHTTPS string     `json:"media_url_https"`
+	ExpandedURL   string     `json:"expanded_url"`
+	URL           string     `json:"url"`
+	VideoInfo     *VideoInfo `json:"video_info,omitempty"`
+}
+
+type VideoInfo struct {
+	DurationMillis int       `json:"duration_millis"`
+	Variants       []Variant `json:"variants"`
+	AspectRatio    []int     `json:"aspect_ratio"`
+}
+
+type Variant struct {
+	Bitrate     int    `json:"bitrate,omitempty"`
+	ContentType string `json:"content_type"`
+	URL         string `json:"url"`
+}

ext/twitter/util.go

@@ -0,0 +1,162 @@
+package twitter
+
+import (
+	"encoding/json"
+	"fmt"
+	"govd/enums"
+	"govd/models"
+	"govd/util"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+const authToken = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
+
+var resolutionRegex = regexp.MustCompile(`(\d+)x(\d+)`)
+
+func BuildAPIHeaders(cookies []*http.Cookie) map[string]string {
+	var csrfToken string
+	for _, cookie := range cookies {
+		if cookie.Name == "ct0" {
+			csrfToken = cookie.Value
+			break
+		}
+	}
+	if csrfToken == "" {
+		return nil
+	}
+	headers := map[string]string{
+		"authorization":             fmt.Sprintf("Bearer %s", authToken),
+		"user-agent":                util.ChromeUA,
+		"x-twitter-auth-type":       "OAuth2Session",
+		"x-twitter-client-language": "en",
+		"x-twitter-active-user":     "yes",
+	}
+
+	if csrfToken != "" {
+		headers["x-csrf-token"] = csrfToken
+	}
+
+	return headers
+}
+
+func BuildAPIQuery(tweetID string) map[string]string {
+	variables := map[string]interface{}{
+		"focalTweetId":                           tweetID,
+		"includePromotedContent":                 true,
+		"with_rux_injections":                    false,
+		"withBirdwatchNotes":                     true,
+		"withCommunity":                          true,
+		"withDownvotePerspective":                false,
+		"withQuickPromoteEligibilityTweetFields": true,
+		"withReactionsMetadata":                  false,
+		"withReactionsPerspective":               false,
+		"withSuperFollowsTweetFields":            true,
+		"withSuperFollowsUserFields":             true,
+		"withV2Timeline":                         true,
+		"withVoice":                              true,
+	}
+
+	features := map[string]interface{}{
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              false,
+		"interactive_text_enabled":                                                true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"responsive_web_enhance_cards_enabled":                                    true,
+		"responsive_web_graphql_timeline_navigation_enabled":                      false,
+		"responsive_web_text_conversations_enabled":                               false,
+		"responsive_web_uc_gql_enabled":                                           true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+		"tweetypie_unmention_optimization_enabled":                                true,
+		"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":  true,
+		"verified_phone_label_enabled":                                            false,
+		"vibe_api_enabled":                                                        true,
+	}
+
+	variablesJSON, _ := json.Marshal(variables)
+	featuresJSON, _ := json.Marshal(features)
+
+	return map[string]string{
+		"variables": string(variablesJSON),
+		"features":  string(featuresJSON),
+	}
+}
+
+func CleanCaption(caption string) string {
+	if caption == "" {
+		return ""
+	}
+	regex := regexp.MustCompile(`https?://t\.co/\S+`)
+	return strings.TrimSpace(regex.ReplaceAllString(caption, ""))
+}
+
+func ExtractVideoFormats(media *MediaEntity) ([]*models.MediaFormat, error) {
+	var formats []*models.MediaFormat
+
+	if media.VideoInfo == nil {
+		return formats, nil
+	}
+
+	duration := int64(media.VideoInfo.DurationMillis / 1000)
+
+	for _, variant := range media.VideoInfo.Variants {
+		if variant.ContentType == "video/mp4" {
+			width, height := extractResolution(variant.URL)
+
+			formats = append(formats, &models.MediaFormat{
+				Type:       enums.MediaTypeVideo,
+				FormatID:   fmt.Sprintf("mp4_%d", variant.Bitrate),
+				URL:        []string{variant.URL},
+				VideoCodec: enums.MediaCodecAVC,
+				AudioCodec: enums.MediaCodecAAC,
+				Duration:   duration,
+				Thumbnail:  []string{media.MediaURLHTTPS},
+				Width:      width,
+				Height:     height,
+				Bitrate:    int64(variant.Bitrate),
+			})
+		}
+	}
+
+	return formats, nil
+}
+
+func extractResolution(url string) (int64, int64) {
+	matches := resolutionRegex.FindStringSubmatch(url)
+	if len(matches) >= 3 {
+		width, _ := strconv.ParseInt(matches[1], 10, 64)
+		height, _ := strconv.ParseInt(matches[2], 10, 64)
+		return width, height
+	}
+	return 0, 0
+}
+
+func FindTweetData(resp *APIResponse, tweetID string) (*Tweet, error) {
+	instructions := resp.Data.ThreadedConversationWithInjectionsV2.Instructions
+	if len(instructions) == 0 {
+		return nil, fmt.Errorf("nessuna istruzione trovata nella risposta")
+	}
+
+	entries := instructions[0].Entries
+	entryID := fmt.Sprintf("tweet-%s", tweetID)
+
+	for _, entry := range entries {
+		if entry.EntryID == entryID {
+			result := entry.Content.ItemContent.TweetResults.Result
+
+			if result.Tweet != nil {
+				return result.Tweet, nil
+			}
+
+			if result.Legacy != nil {
+				return result.Legacy, nil
+			}
+
+			return nil, fmt.Errorf("struttura del tweet non valida")
+		}
+	}
+
+	return nil, fmt.Errorf("tweet non trovato nella risposta")
+}

ext/util.go

@@ -0,0 +1,74 @@
+package ext
+
+import (
+	"fmt"
+	"govd/models"
+)
+
+var maxRedirects = 5
+
+func CtxByURL(url string) (*models.DownloadContext, error) {
+	var redirectCount int
+
+	currentURL := url
+
+	for redirectCount <= maxRedirects {
+		for _, extractor := range List {
+			matches := extractor.URLPattern.FindStringSubmatch(currentURL)
+			if matches == nil {
+				continue
+			}
+
+			groupNames := extractor.URLPattern.SubexpNames()
+			if len(matches) == 0 {
+				continue
+			}
+
+			groups := make(map[string]string)
+			for i, name := range groupNames {
+				if name != "" {
+					groups[name] = matches[i]
+				}
+			}
+			groups["match"] = matches[0]
+
+			ctx := &models.DownloadContext{
+				MatchedContentID:  groups["id"],
+				MatchedContentURL: groups["match"],
+				MatchedGroups:     groups,
+				Extractor:         extractor,
+			}
+
+			if !extractor.IsRedirect {
+				return ctx, nil
+			}
+
+			response, err := extractor.Run(ctx)
+			if err != nil {
+				return nil, err
+			}
+			if response.URL == "" {
+				return nil, fmt.Errorf("no URL found in response")
+			}
+
+			currentURL = response.URL
+			redirectCount++
+
+			break
+		}
+
+		if redirectCount > maxRedirects {
+			return nil, fmt.Errorf("exceeded maximum number of redirects (%d)", maxRedirects)
+		}
+	}
+	return nil, nil
+}
+
+func ByCodeName(codeName string) *models.Extractor {
+	for _, extractor := range List {
+		if extractor.CodeName == codeName {
+			return extractor
+		}
+	}
+	return nil
+}

go.mod

@@ -0,0 +1,52 @@
+module govd
+
+go 1.24.0
+
+require (
+	github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.31
+	github.com/google/uuid v1.6.0
+	github.com/guregu/null/v6 v6.0.0
+	github.com/joho/godotenv v1.5.1
+	github.com/quic-go/quic-go v0.50.1
+	github.com/strukturag/libheif v1.19.7
+	github.com/tidwall/gjson v1.18.0
+	github.com/u2takey/ffmpeg-go v0.5.0
+	golang.org/x/image v0.26.0
+	gorm.io/gorm v1.25.12
+)
+
+require (
+	github.com/Eyevinn/dash-mpd v0.12.0 // indirect
+	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/unki2aut/go-xsd-types v0.0.0-20200220223938-30e5405398f8 // indirect
+)
+
+require (
+	github.com/aki237/nscjar v0.0.0-20210417074043-bbb606196143
+	github.com/aws/aws-sdk-go v1.55.6 // indirect
+	github.com/etherlabsio/go-m3u8 v1.0.0
+	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
+	github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect
+	github.com/grafov/m3u8 v0.12.1
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/onsi/ginkgo/v2 v2.9.5 // indirect
+	github.com/pkg/errors v0.9.1
+	github.com/quic-go/qpack v0.5.1 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/u2takey/go-utils v0.3.1 // indirect
+	github.com/unki2aut/go-mpd v0.0.0-20250218132413-c6a2d2d492f4
+	go.uber.org/mock v0.5.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
+	golang.org/x/mod v0.18.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/sync v0.13.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	golang.org/x/tools v0.22.0 // indirect
+	gorm.io/driver/mysql v1.5.7
+)

go.sum

@@ -0,0 +1,162 @@
+github.com/AlekSi/pointer v1.0.0/go.mod h1:1kjywbfcPFCmncIxtk6fIEub6LKrfMz3gc5QKVOSOA8=
+github.com/Eyevinn/dash-mpd v0.12.0 h1:fFNE9KPLqe4OG79fYyT/KalmFbQT2vG4Z01ppmEC4Aw=
+github.com/Eyevinn/dash-mpd v0.12.0/go.mod h1:yym2itvB74evfJFDZB99p700LQddQFsN1YCbk9t6mAA=
+github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.31 h1:SIkzqC6Nv+znY4NGbWlJceWdns8QVmf9cwAYXd7Cg8k=
+github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.31/go.mod h1:kL1v4iIjlalwm3gCYGvF4NLa3hs+aKEfRkNJvj4aoDU=
+github.com/aki237/nscjar v0.0.0-20210417074043-bbb606196143 h1:PqRkQZW8lAlK2DnH9iSBfISmDxSChaoNJHwP0p7SD2Y=
+github.com/aki237/nscjar v0.0.0-20210417074043-bbb606196143/go.mod h1:l0r3UsMujHR1bAYL7R0+6NXkHo/vIe+ja3xLZbUZNb8=
+github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk=
+github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0=
+github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
+github.com/etherlabsio/go-m3u8 v1.0.0 h1:d3HJVr8wlbvJO20ksKEyvDYf4bcM7v8YV3W83fHswL0=
+github.com/etherlabsio/go-m3u8 v1.0.0/go.mod h1:RzDiaXgaYnIEzZUmVUD/xMRFR7bY7U5JaCnp8XYLmXU=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
+github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
+github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grafov/m3u8 v0.12.1 h1:DuP1uA1kvRRmGNAZ0m+ObLv1dvrfNO0TPx0c/enNk0s=
+github.com/grafov/m3u8 v0.12.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080=
+github.com/guregu/null/v6 v6.0.0 h1:N14VRS+4di81i1PXRiprbQJ9EM9gqBa0+KVMeS/QSjQ=
+github.com/guregu/null/v6 v6.0.0/go.mod h1:hrMIhIfrOZeLPZhROSn149tpw2gHkidAqxoXNyeX3iQ=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
+github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
+github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
+github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
+github.com/quic-go/quic-go v0.50.1 h1:unsgjFIUqW8a2oopkY7YNONpV1gYND6Nt9hnt1PN94Q=
+github.com/quic-go/quic-go v0.50.1/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/strukturag/libheif v1.19.7 h1:XMfSJvmnucTbiS6CSxxZmpx5XSPjdqkpA3wiL6+I2Iw=
+github.com/strukturag/libheif v1.19.7/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks=
+github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
+github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU=
+github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc=
+github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys=
+github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs=
+github.com/unki2aut/go-mpd v0.0.0-20250218132413-c6a2d2d492f4 h1:yPsATZRcBhrBQkxK9hsGo1cPHroobmw7Bptt+UJV0D8=
+github.com/unki2aut/go-mpd v0.0.0-20250218132413-c6a2d2d492f4/go.mod h1:LITqXLCxxmcoHtOMgZh5NbcfS4RCrrADQXPVkYwF/cc=
+github.com/unki2aut/go-xsd-types v0.0.0-20200220223938-30e5405398f8 h1:u0Bi6Mf8BKPQnxGJ7QubdMyhb0SJjnQU7kX0BA9eASk=
+github.com/unki2aut/go-xsd-types v0.0.0-20200220223938-30e5405398f8/go.mod h1:uIeMfpmWIZ8SGp+fTfwDBWiiRn3aJm4b7rFSro9s++Q=
+go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
+go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
+gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
+golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.26.0 h1:4XjIFEZWQmCZi6Wv8BoxsDhRU3RVnLX04dToTDAEPlY=
+golang.org/x/image v0.26.0/go.mod h1:lcxbMFAovzpnJxzXS3nyL83K27tmqtKzIJpctK8YO5c=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
+golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
+golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
+golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
+golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
+gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
+gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
+gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8=
+gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"govd/bot"
+	"govd/database"
+	"govd/util"
+	"log"
+
+	"github.com/joho/godotenv"
+)
+
+func main() {
+	err := godotenv.Load()
+	if err != nil {
+		log.Fatal("error loading .env file")
+	}
+	ok := util.CheckFFmpeg()
+	if !ok {
+		log.Fatal("ffmpeg executable not found. please install it or add it to your PATH")
+	}
+	database.Start()
+	go bot.Start()
+
+	select {} // keep the main goroutine alive
+}

models/ctx.go

@@ -0,0 +1,9 @@
+package models
+
+type DownloadContext struct {
+	MatchedContentID  string
+	MatchedContentURL string
+	MatchedGroups     map[string]string
+	GroupSettings     *GroupSettings
+	Extractor         *Extractor
+}

models/download.go

@@ -0,0 +1,14 @@
+package models
+
+import "time"
+
+type DownloadConfig struct {
+	ChunkSize       int           // size of each chunk in bytes
+	Concurrency     int           // maximum number of concurrent downloads
+	Timeout         time.Duration // timeout for individual HTTP requests
+	DownloadDir     string        // directory to save downloaded files
+	RetryAttempts   int           // number of retry attempts per chunk
+	RetryDelay      time.Duration // delay between retries
+	Remux           bool          // whether to remux the downloaded file with ffmpeg
+	ProgressUpdater func(float64) // optional function to report download progress
+}

models/ext.go

@@ -0,0 +1,34 @@
+package models
+
+import (
+	"govd/enums"
+	"regexp"
+)
+
+type Extractor struct {
+	Name       string
+	CodeName   string
+	Type       enums.ExtractorType
+	Category   enums.ExtractorCategory
+	URLPattern *regexp.Regexp
+	IsDRM      bool
+	IsRedirect bool
+
+	Run func(*DownloadContext) (*ExtractorResponse, error)
+}
+
+type ExtractorResponse struct {
+	MediaList []*Media
+	URL       string // redirected URL
+}
+
+func (extractor *Extractor) NewMedia(
+	contentID string,
+	contentURL string,
+) *Media {
+	return &Media{
+		ContentID:         contentID,
+		ContentURL:        contentURL,
+		ExtractorCodeName: extractor.CodeName,
+	}
+}

models/media.go

@@ -0,0 +1,506 @@
+package models
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"govd/enums"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/google/uuid"
+	"github.com/guregu/null/v6/zero"
+	"gorm.io/gorm"
+)
+
+type Media struct {
+	ID                uint           `json:"-"`
+	ContentID         string         `gorm:"not null;index" json:"content_id"`
+	ContentURL        string         `gorm:"not null" json:"content_url"`
+	ExtractorCodeName string         `gorm:"not null;index" json:"extractor_code_name"`
+	Caption           zero.String    `json:"caption"`
+	NSFW              bool           `gorm:"default:false" json:"nsfw"`
+	CreatedAt         time.Time      `json:"-"`
+	UpdatedAt         time.Time      `json:"-"`
+	DeletedAt         gorm.DeletedAt `gorm:"index" json:"-"`
+
+	Format *MediaFormat `json:"-"`
+
+	Formats []*MediaFormat `gorm:"-" json:"formats"`
+}
+
+type MediaFormat struct {
+	ID         uint             `json:"-"`
+	MediaID    uint             `gorm:"index:idx_media_format,priority:1;not null" json:"-"`
+	Type       enums.MediaType  `gorm:"not null;index:idx_media_type" json:"type"`
+	FormatID   string           `gorm:"not null;index" json:"format_id"`
+	FileID     string           `gorm:"not null;index" json:"-"`
+	VideoCodec enums.MediaCodec `json:"video_codec"`
+	AudioCodec enums.MediaCodec `json:"audio_codec"`
+	Duration   int64            `json:"duration"`
+	Width      int64            `json:"width"`
+	Height     int64            `json:"height"`
+	Bitrate    int64            `json:"bitrate"`
+	Title      string           `json:"title"`
+	Artist     string           `json:"artist"`
+	IsDefault  bool             `gorm:"default:false;index" json:"is_default"`
+	Segments   []string         `gorm:"-" json:"segments"`
+	FileSize   int64            `json:"-"`
+	Plugins    []Plugin         `gorm:"-" json:"-"`
+
+	CreatedAt time.Time      `json:"-"`
+	UpdatedAt time.Time      `json:"-"`
+	DeletedAt gorm.DeletedAt `gorm:"index" json:"-"`
+
+	// api use only, not stored in database
+	URL       []string `gorm:"-" json:"url"`
+	Thumbnail []string `gorm:"-" json:"thumbnail"`
+
+	Media *Media `gorm:"foreignKey:MediaID" json:"-"`
+}
+
+type DownloadedMedia struct {
+	FilePath          string
+	ThumbnailFilePath string
+	Media             *Media
+	Index             int
+}
+
+func (media *Media) GetFormat(formatID string) *MediaFormat {
+	for _, format := range media.Formats {
+		if format.FormatID == formatID {
+			return format
+		}
+	}
+	return nil
+}
+
+func (media *Media) GetDefaultFormat() *MediaFormat {
+	format := media.GetDefaultVideoFormat()
+	if format != nil {
+		return format
+	}
+	format = media.GetDefaultAudioFormat()
+	if format != nil {
+		return format
+	}
+	format = media.GetDefaultPhotoFormat()
+	if format != nil {
+		return format
+	}
+	return nil
+}
+
+func (media *Media) GetDefaultVideoFormat() *MediaFormat {
+	filtered := filterFormats(media.Formats, func(format *MediaFormat) bool {
+		return format.VideoCodec == enums.MediaCodecAVC
+	})
+	if len(filtered) == 0 {
+		filtered = filterFormats(media.Formats, func(format *MediaFormat) bool {
+			return format.VideoCodec != ""
+		})
+	}
+	if len(filtered) == 0 {
+		return nil
+	}
+	bestFormat := filtered[0]
+	for _, format := range filtered {
+		if format.Bitrate > bestFormat.Bitrate {
+			bestFormat = format
+		}
+	}
+	bestFormat.IsDefault = true
+	return bestFormat
+}
+
+func (media *Media) GetDefaultAudioFormat() *MediaFormat {
+	filtered := filterFormats(media.Formats, func(format *MediaFormat) bool {
+		return format.VideoCodec == "" &&
+			(format.AudioCodec == enums.MediaCodecAAC ||
+				format.AudioCodec == enums.MediaCodecMP3)
+	})
+	if len(filtered) == 0 {
+		filtered = filterFormats(media.Formats, func(format *MediaFormat) bool {
+			return format.VideoCodec == "" && format.AudioCodec != ""
+		})
+	}
+	if len(filtered) == 0 {
+		return nil
+	}
+	bestFormat := filtered[0]
+	for _, format := range filtered {
+		if format.Bitrate > bestFormat.Bitrate {
+			bestFormat = format
+		}
+	}
+	bestFormat.IsDefault = true
+	return bestFormat
+}
+
+func (media *Media) GetDefaultPhotoFormat() *MediaFormat {
+	filtered := filterFormats(media.Formats, func(format *MediaFormat) bool {
+		return format.Type == enums.MediaTypePhoto
+	})
+	if len(filtered) == 0 {
+		return nil
+	}
+	filtered[0].IsDefault = true
+	return filtered[0]
+}
+
+func (media *Media) GetAudioFromVideoFormat() *MediaFormat {
+	videoFormat := media.GetDefaultVideoFormat()
+
+	if videoFormat == nil {
+		return nil
+	}
+
+	return &MediaFormat{
+		Type:       enums.MediaTypeAudio,
+		FormatID:   "AudioFromVideo",
+		URL:        videoFormat.URL,
+		AudioCodec: enums.MediaCodecAAC,
+		Thumbnail:  videoFormat.Thumbnail,
+		Duration:   videoFormat.Duration,
+		Title:      videoFormat.Title,
+		Artist:     videoFormat.Artist,
+	}
+}
+
+func (media *Media) SetCaption(caption string) {
+	if len(caption) == 0 {
+		return
+	}
+	media.Caption = zero.StringFrom(caption)
+}
+
+func (media *Media) AddFormat(fmt *MediaFormat) {
+	media.Formats = append(media.Formats, fmt)
+}
+
+func (media *Media) GetSortedFormats() []*MediaFormat {
+	// group by video format (codec, width, height)
+	groupedVideos := make(map[[3]int64]*MediaFormat)
+	for _, format := range media.Formats {
+		if format.Type == enums.MediaTypeVideo {
+			key := [3]int64{
+				getCodecPriority(format.VideoCodec),
+				format.Width,
+				format.Height,
+			}
+			existing, ok := groupedVideos[key]
+			if !ok || format.Bitrate > existing.Bitrate {
+				groupedVideos[key] = format
+			}
+		}
+	}
+
+	// group by audio format (codec, bitrate)
+	groupedAudios := make(map[[2]int64]*MediaFormat)
+	for _, format := range media.Formats {
+		if format.Type == enums.MediaTypeAudio {
+			key := [2]int64{
+				getCodecPriority(format.AudioCodec),
+				format.Bitrate,
+			}
+			_, exists := groupedAudios[key]
+			if !exists {
+				groupedAudios[key] = format
+			}
+		}
+	}
+
+	// combine the best video and audio into a final list
+	var finalSortedList []*MediaFormat
+	for _, best := range groupedVideos {
+		finalSortedList = append(finalSortedList, best)
+	}
+	for _, best := range groupedAudios {
+		finalSortedList = append(finalSortedList, best)
+	}
+
+	for _, format := range media.Formats {
+		if format.Type != enums.MediaTypeVideo && format.Type != enums.MediaTypeAudio {
+			finalSortedList = append(finalSortedList, format) // for non-video and non-audio formats
+		}
+	}
+
+	// sort the final list
+	sort.Slice(finalSortedList, func(i, j int) bool {
+		a, b := finalSortedList[i], finalSortedList[j]
+		// compare by type priority (video, audio, photo, etc.)
+		if cmp := getTypePriority(a.Type) - getTypePriority(b.Type); cmp != 0 {
+			return cmp < 0
+		}
+		// compare by codec priority (for both video and audio)
+		if a.Type == enums.MediaTypeVideo {
+			if cmp := getCodecPriority(a.VideoCodec) - getCodecPriority(b.VideoCodec); cmp != 0 {
+				return cmp < 0
+			}
+		} else if a.Type == enums.MediaTypeAudio {
+			if cmp := getCodecPriority(a.AudioCodec) - getCodecPriority(b.AudioCodec); cmp != 0 {
+				return cmp < 0
+			}
+		}
+		// compare by width for videos
+		if cmp := a.Width - b.Width; cmp != 0 {
+			return cmp < 0
+		}
+		// compare by height for videos
+		if cmp := a.Height - b.Height; cmp != 0 {
+			return cmp < 0
+		}
+		// compare by bitrate (lower bitrate first)
+		return a.Bitrate-b.Bitrate < 0
+	})
+
+	return finalSortedList
+}
+
+func filterFormats(
+	formats []*MediaFormat,
+	condition func(*MediaFormat) bool,
+) []*MediaFormat {
+	var filtered []*MediaFormat
+	for _, format := range formats {
+		if condition(format) {
+			filtered = append(filtered, format)
+		}
+	}
+	return filtered
+}
+
+func getCodecPriority(codec enums.MediaCodec) int64 {
+	codecPriority := map[enums.MediaCodec]int64{
+		enums.MediaCodecAVC:  1,
+		enums.MediaCodecHEVC: 2,
+		enums.MediaCodecMP3:  3,
+		enums.MediaCodecAAC:  4,
+	}
+	return codecPriority[codec]
+}
+
+func getTypePriority(mediaType enums.MediaType) int64 {
+	typePriority := map[enums.MediaType]int64{
+		enums.MediaTypeVideo: 1,
+		enums.MediaTypeAudio: 2,
+		enums.MediaTypePhoto: 3,
+	}
+	return typePriority[mediaType]
+}
+
+// getFormatInfo returns the file extension and the InputMedia type.
+func (format *MediaFormat) GetFormatInfo() (string, string) {
+	if format.Type == enums.MediaTypePhoto {
+		return "jpeg", "photo"
+	}
+
+	videoCodec := format.VideoCodec
+	audioCodec := format.AudioCodec
+
+	switch {
+	case videoCodec == enums.MediaCodecAVC && audioCodec == enums.MediaCodecAAC:
+		return "mp4", "video"
+	case videoCodec == enums.MediaCodecAVC && audioCodec == enums.MediaCodecMP3:
+		return "mp4", "video"
+	case videoCodec == enums.MediaCodecHEVC && audioCodec == enums.MediaCodecAAC:
+		return "mp4", "document"
+	case videoCodec == enums.MediaCodecHEVC && audioCodec == enums.MediaCodecMP3:
+		return "mp4", "document"
+	case videoCodec == enums.MediaCodecAV1 && audioCodec == enums.MediaCodecOpus:
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecAV1 && audioCodec == enums.MediaCodecFLAC:
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecVP9 && audioCodec == enums.MediaCodecOpus:
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecVP9 && audioCodec == enums.MediaCodecFLAC:
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecAVC && audioCodec == "":
+		return "mp4", "video"
+	case videoCodec == enums.MediaCodecHEVC && audioCodec == "":
+		return "mp4", "document"
+	case videoCodec == enums.MediaCodecAV1 && audioCodec == "":
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecVP9 && audioCodec == "":
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecVP8 && audioCodec == "":
+		return "webm", "document"
+	case videoCodec == enums.MediaCodecWebP && audioCodec == "":
+		return "webp", "video"
+	case videoCodec == "" && audioCodec == enums.MediaCodecMP3:
+		return "mp3", "audio"
+	case videoCodec == "" && audioCodec == enums.MediaCodecAAC:
+		return "m4a", "audio"
+	case videoCodec == "" && audioCodec == enums.MediaCodecOpus:
+		return "webm", "document"
+	case videoCodec == "" && audioCodec == enums.MediaCodecFLAC:
+		return "flac", "document"
+	case videoCodec == "" && audioCodec == enums.MediaCodecVorbis:
+		return "oga", "document"
+	default:
+		return "webm", "document"
+	}
+}
+
+func (format *MediaFormat) GetInputMedia(
+	filePath string,
+	thumbnailFilePath string,
+	messageCaption string,
+) (gotgbot.InputMedia, error) {
+	if format.FileID != "" {
+		return format.GetInputMediaWithFileID(messageCaption)
+	}
+
+	_, inputMediaType := format.GetFormatInfo()
+
+	fileObj, err := os.Open(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %w", err)
+	}
+	fileInputMedia := gotgbot.InputFileByReader(
+		filepath.Base(filePath),
+		fileObj,
+	)
+
+	var thumbnailFileInputMedia gotgbot.InputFile
+	if thumbnailFilePath != "" {
+		thumbnailFileObj, err := os.Open(thumbnailFilePath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open file: %w", err)
+		}
+		thumbnailFileInputMedia = gotgbot.InputFileByReader(
+			filepath.Base(thumbnailFilePath),
+			thumbnailFileObj,
+		)
+	}
+
+	if inputMediaType == "video" {
+		return &gotgbot.InputMediaVideo{
+			Media:             fileInputMedia,
+			Thumbnail:         thumbnailFileInputMedia,
+			Width:             format.Width,
+			Height:            format.Height,
+			Duration:          format.Duration,
+			Caption:           messageCaption,
+			SupportsStreaming: true,
+			ParseMode:         "HTML",
+		}, nil
+	}
+	if inputMediaType == "audio" {
+		return &gotgbot.InputMediaAudio{
+			Media:     fileInputMedia,
+			Thumbnail: thumbnailFileInputMedia,
+			Duration:  format.Duration,
+			Performer: format.Artist,
+			Title:     format.Title,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	if inputMediaType == "photo" {
+		return &gotgbot.InputMediaPhoto{
+			Media:     fileInputMedia,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	if inputMediaType == "document" {
+		return &gotgbot.InputMediaDocument{
+			Media:     fileInputMedia,
+			Thumbnail: thumbnailFileInputMedia,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	return nil, fmt.Errorf("unknown input type: %s", inputMediaType)
+}
+
+func (format *MediaFormat) GetInputMediaWithFileID(
+	messageCaption string,
+) (gotgbot.InputMedia, error) {
+	_, inputMediaType := format.GetFormatInfo()
+	fileInputMedia := gotgbot.InputFileByID(format.FileID)
+	if inputMediaType == "video" {
+		return &gotgbot.InputMediaVideo{
+			Media:     fileInputMedia,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	if inputMediaType == "audio" {
+		return &gotgbot.InputMediaAudio{
+			Media:     fileInputMedia,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	if inputMediaType == "photo" {
+		return &gotgbot.InputMediaPhoto{
+			Media:     fileInputMedia,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	if inputMediaType == "document" {
+		return &gotgbot.InputMediaDocument{
+			Media:     fileInputMedia,
+			Caption:   messageCaption,
+			ParseMode: "HTML",
+		}, nil
+	}
+	return nil, fmt.Errorf("unknown input type: %s", inputMediaType)
+}
+
+func (format *MediaFormat) GetFileName() string {
+	extension, _ := format.GetFormatInfo()
+	if format.Type == enums.MediaTypeAudio && format.Title != "" && format.Artist != "" {
+		return fmt.Sprintf("%s - %s.%s", format.Artist, format.Title, extension)
+	} else {
+		name := uuid.New().String()
+		name = strings.ReplaceAll(name, "-", "")
+		return fmt.Sprintf("%s.%s", name, extension)
+	}
+}
+
+func (media *Media) HasVideo() bool {
+	for _, format := range media.Formats {
+		if format.Type == enums.MediaTypeVideo {
+			return true
+		}
+	}
+	return false
+}
+
+func (media *Media) HasAudio() bool {
+	for _, format := range media.Formats {
+		if format.Type == enums.MediaTypeAudio {
+			return true
+		}
+	}
+	return false
+}
+
+func (media *Media) HasPhoto() bool {
+	for _, format := range media.Formats {
+		if format.Type == enums.MediaTypePhoto {
+			return true
+		}
+	}
+	return false
+}
+
+func (media *Media) SupportsAudio() bool {
+	for _, format := range media.Formats {
+		if format.AudioCodec != "" {
+			return true
+		}
+	}
+	return false
+}
+
+func (media *Media) SupportsAudioFromVideo() bool {
+	return !media.HasAudio() && media.HasVideo() && media.SupportsAudio()
+}

models/misc.go

@@ -0,0 +1,11 @@
+package models
+
+type SendMediaFormatsOptions struct {
+	IsStored bool
+	Caption  string
+}
+
+type Chunk struct {
+	Data []byte
+	Idx  int
+}

models/plugin.go

@@ -0,0 +1,3 @@
+package models
+
+type Plugin = func(*DownloadedMedia) error

models/settings.go

@@ -0,0 +1,12 @@
+package models
+
+import "gorm.io/gorm"
+
+type GroupSettings struct {
+	gorm.Model
+
+	ChatID          int64 `gorm:"primaryKey"`
+	NSFW            *bool `gorm:"default:false"`
+	Captions        *bool `gorm:"default:false"`
+	MediaGroupLimit int   `gorm:"default:10"`
+}

models/user.go

@@ -0,0 +1,14 @@
+package models
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+type User struct {
+	gorm.Model
+
+	UserID   int64     `gorm:"primaryKey"`
+	LastUsed time.Time `gorm:"autoCreateTime"`
+}

plugins/main.go

@@ -0,0 +1,7 @@
+package plugins
+
+import "govd/models"
+
+var List = []models.Plugin{
+	MergeAudio,
+}

plugins/merge_audio.go

@@ -0,0 +1,40 @@
+package plugins
+
+import (
+	"context"
+	"fmt"
+	"govd/models"
+	"govd/util"
+	"govd/util/av"
+
+	"github.com/pkg/errors"
+)
+
+func MergeAudio(media *models.DownloadedMedia) error {
+	audioFormat := media.Media.GetDefaultAudioFormat()
+	if audioFormat == nil {
+		return errors.New("no audio format found")
+	}
+
+	// download the audio file
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	audioFile, err := util.DownloadFile(
+		ctx, audioFormat.URL,
+		audioFormat.GetFileName(), nil,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to download audio file: %w", err)
+	}
+
+	err = av.MergeVideoWithAudio(
+		media.FilePath,
+		audioFile,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to merge video with audio: %w", err)
+	}
+
+	return nil
+}

util/av/audio.go

@@ -0,0 +1,23 @@
+package av
+
+import (
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+)
+
+func AudioFromVideo(videoPath string, audioPath string) error {
+	err := ffmpeg.
+		Input(videoPath).
+		Output(audioPath, ffmpeg.KwArgs{
+			"map": "a",
+			"vn":  nil,
+			"f":   "mp3",
+			"ab":  "128k",
+		}).
+		Silent(true).
+		OverWriteOutput().
+		Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}

util/av/merge_audio.go

@@ -0,0 +1,46 @@
+package av
+
+import (
+	"fmt"
+	"os"
+
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+)
+
+func MergeVideoWithAudio(
+	videoFile string,
+	audioFile string,
+) error {
+	tempFileName := videoFile + ".temp"
+	outputFile := videoFile
+
+	err := os.Rename(videoFile, tempFileName)
+	if err != nil {
+		return fmt.Errorf("failed to rename file: %w", err)
+	}
+
+	defer os.Remove(tempFileName)
+	defer os.Remove(audioFile)
+
+	videoStream := ffmpeg.Input(tempFileName)
+	audioStream := ffmpeg.Input(audioFile)
+
+	err = ffmpeg.Output(
+		[]*ffmpeg.Stream{videoStream, audioStream},
+		outputFile,
+		ffmpeg.KwArgs{
+			"map":      []string{"0:v:0", "1:a:0"},
+			"movflags": "+faststart",
+			"c:v":      "copy",
+			"c:a":      "copy",
+		}).
+		Silent(true).
+		OverWriteOutput().
+		Run()
+
+	if err != nil {
+		return fmt.Errorf("failed to merge files: %w", err)
+	}
+
+	return nil
+}

util/av/remux.go

@@ -0,0 +1,35 @@
+package av
+
+import (
+	"fmt"
+	"os"
+
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+)
+
+func RemuxFile(
+	inputFile string,
+) error {
+	tempFileName := inputFile + ".temp"
+	outputFile := inputFile
+	err := os.Rename(inputFile, tempFileName)
+	if err != nil {
+		return fmt.Errorf("failed to rename file: %v", err)
+	}
+	err = ffmpeg.
+		Input(tempFileName).
+		Output(outputFile, ffmpeg.KwArgs{
+			"c": "copy",
+		}).
+		Silent(true).
+		OverWriteOutput().
+		Run()
+	if err != nil {
+		return fmt.Errorf("failed to remux file: %v", err)
+	}
+	err = os.Remove(tempFileName)
+	if err != nil {
+		return fmt.Errorf("failed to remove temp file: %v", err)
+	}
+	return nil
+}

util/av/thumbnail.go

@@ -0,0 +1,27 @@
+package av
+
+import (
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+)
+
+func ExtractVideoThumbnail(
+	videoPath string,
+	thumbnailPath string,
+) error {
+	err := ffmpeg.
+		Input(videoPath).
+		Output(thumbnailPath, ffmpeg.KwArgs{
+			"vframes": 1,
+			"f":       "image2",
+			"ss":      "00:00:01",
+			"c:v":     "mjpeg",
+			"q:v":     10, // not sure
+		}).
+		Silent(true).
+		OverWriteOutput().
+		Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}

util/av/videoinfo.go

@@ -0,0 +1,18 @@
+package av
+
+import (
+	"github.com/tidwall/gjson"
+	ffmpeg "github.com/u2takey/ffmpeg-go"
+)
+
+func GetVideoInfo(filePath string) (int64, int64, int64) {
+	probeData, err := ffmpeg.Probe(filePath)
+	if err != nil {
+		return 0, 0, 0
+	}
+	duration := gjson.Get(probeData, "format.duration").Int()
+	width := gjson.Get(probeData, "streams.0.width").Int()
+	height := gjson.Get(probeData, "streams.0.height").Int()
+
+	return duration, width, height
+}

util/consts.go

@@ -0,0 +1,5 @@
+package util
+
+const (
+	ChromeUA = "Mozilla/5.0 (Linux; Android 10; SM-G960U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.181 Mobile Safari/537.36"
+)

util/download.go

@@ -0,0 +1,512 @@
+package util
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"govd/models"
+	"govd/util/av"
+)
+
+func DefaultConfig() *models.DownloadConfig {
+	return &models.DownloadConfig{
+		ChunkSize:     10 * 1024 * 1024, // 10MB
+		Concurrency:   4,
+		Timeout:       30 * time.Second,
+		DownloadDir:   "downloads",
+		RetryAttempts: 3,
+		RetryDelay:    2 * time.Second,
+		Remux:         true,
+	}
+}
+
+func DownloadFile(
+	ctx context.Context,
+	URLList []string,
+	fileName string,
+	config *models.DownloadConfig,
+) (string, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+
+	var errs []error
+	for _, fileURL := range URLList {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		default:
+			// create the download directory if it doesn't exist
+			if err := ensureDownloadDir(config.DownloadDir); err != nil {
+				return "", err
+			}
+
+			filePath := filepath.Join(config.DownloadDir, fileName)
+			err := runChunkedDownload(ctx, fileURL, filePath, config)
+			if err != nil {
+				errs = append(errs, err)
+				continue
+			}
+
+			if config.Remux {
+				err := av.RemuxFile(filePath)
+				if err != nil {
+					return "", fmt.Errorf("remuxing failed: %w", err)
+				}
+			}
+			return filePath, nil
+		}
+	}
+
+	return "", fmt.Errorf("%w: %v", ErrDownloadFailed, errs)
+}
+
+func DownloadFileWithSegments(
+	ctx context.Context,
+	segmentURLs []string,
+	fileName string,
+	config *models.DownloadConfig,
+) (string, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+	if err := ensureDownloadDir(config.DownloadDir); err != nil {
+		return "", err
+	}
+	tempDir := filepath.Join(config.DownloadDir, "segments_"+time.Now().Format("20060102_150405"))
+	if err := os.MkdirAll(tempDir, 0755); err != nil {
+		return "", fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+	downloadedFiles, err := DownloadSegments(ctx, segmentURLs, config)
+	if err != nil {
+		os.RemoveAll(tempDir)
+		return "", fmt.Errorf("failed to download segments: %w", err)
+	}
+	mergedFilePath, err := MergeSegmentFiles(ctx, downloadedFiles, fileName, config)
+	if err != nil {
+		os.RemoveAll(tempDir)
+		return "", fmt.Errorf("failed to merge segments: %w", err)
+	}
+	if err := os.RemoveAll(tempDir); err != nil {
+		return "", fmt.Errorf("failed to remove temporary directory: %w", err)
+	}
+	return mergedFilePath, nil
+}
+
+func DownloadFileInMemory(
+	ctx context.Context,
+	URLList []string,
+	config *models.DownloadConfig,
+) (*bytes.Reader, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+
+	var errs []error
+	for _, fileURL := range URLList {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+			data, err := downloadInMemory(ctx, fileURL, config.Timeout)
+			if err != nil {
+				errs = append(errs, err)
+				continue
+			}
+			return bytes.NewReader(data), nil
+		}
+	}
+
+	return nil, fmt.Errorf("%w: %v", ErrDownloadFailed, errs)
+}
+
+func downloadInMemory(ctx context.Context, fileURL string, timeout time.Duration) ([]byte, error) {
+	reqCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, fileURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	session := GetHTTPSession()
+	resp, err := session.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to download file: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	return io.ReadAll(resp.Body)
+}
+
+func ensureDownloadDir(dir string) error {
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return fmt.Errorf("failed to create downloads directory: %w", err)
+		}
+	}
+	return nil
+}
+
+func runChunkedDownload(
+	ctx context.Context,
+	fileURL string,
+	filePath string,
+	config *models.DownloadConfig,
+) error {
+	fileSize, err := getFileSize(ctx, fileURL, config.Timeout)
+	if err != nil {
+		return err
+	}
+
+	file, err := os.Create(filePath)
+	if err != nil {
+		return fmt.Errorf("failed to create file: %w", err)
+	}
+	defer file.Close()
+
+	// pre-allocate file size if possible
+	if fileSize > 0 {
+		if err := file.Truncate(int64(fileSize)); err != nil {
+			return fmt.Errorf("failed to allocate file space: %w", err)
+		}
+	}
+
+	chunks := createChunks(fileSize, config.ChunkSize)
+
+	semaphore := make(chan struct{}, config.Concurrency)
+	var wg sync.WaitGroup
+
+	errChan := make(chan error, 1)
+	var downloadErr error
+	var errOnce sync.Once
+
+	var completedChunks int64
+	var completedBytes int64
+	var progressMutex sync.Mutex
+
+	downloadCtx, cancelDownload := context.WithCancel(ctx)
+	defer cancelDownload()
+
+	for idx, chunk := range chunks {
+		wg.Add(1)
+
+		go func(idx int, chunk [2]int) {
+			defer wg.Done()
+
+			// respect concurrency limit
+			select {
+			case semaphore <- struct{}{}:
+				defer func() { <-semaphore }()
+			case <-downloadCtx.Done():
+				return
+			}
+
+			chunkData, err := downloadChunkWithRetry(downloadCtx, fileURL, chunk, config)
+			if err != nil {
+				errOnce.Do(func() {
+					downloadErr = fmt.Errorf("chunk %d: %w", idx, err)
+					cancelDownload() // cancel all other downloads
+					errChan <- downloadErr
+				})
+				return
+			}
+
+			if err := writeChunkToFile(file, chunkData, chunk[0]); err != nil {
+				errOnce.Do(func() {
+					downloadErr = fmt.Errorf("failed to write chunk %d: %w", idx, err)
+					cancelDownload()
+					errChan <- downloadErr
+				})
+				return
+			}
+
+			// update progress
+			chunkSize := chunk[1] - chunk[0] + 1
+			progressMutex.Lock()
+			completedChunks++
+			completedBytes += int64(chunkSize)
+			progress := float64(completedBytes) / float64(fileSize)
+			progressMutex.Unlock()
+
+			// report progress if handler exists
+			if config.ProgressUpdater != nil {
+				config.ProgressUpdater(progress)
+			}
+		}(idx, chunk)
+	}
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	select {
+	case err := <-errChan:
+		if err != nil {
+			// clean up partial download
+			os.Remove(filePath)
+			return err
+		}
+	case <-ctx.Done():
+		cancelDownload()
+		os.Remove(filePath)
+		return ctx.Err()
+	}
+
+	return nil
+}
+
+func getFileSize(ctx context.Context, fileURL string, timeout time.Duration) (int, error) {
+	reqCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(reqCtx, http.MethodHead, fileURL, nil)
+	if err != nil {
+		return 0, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	session := GetHTTPSession()
+	resp, err := session.Do(req)
+	if err != nil {
+		return 0, fmt.Errorf("failed to get file size: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, fmt.Errorf("failed to get file info: status code %d", resp.StatusCode)
+	}
+
+	return int(resp.ContentLength), nil
+}
+
+func downloadChunkWithRetry(
+	ctx context.Context,
+	fileURL string,
+	chunk [2]int,
+	config *models.DownloadConfig,
+) ([]byte, error) {
+	var lastErr error
+
+	for attempt := 0; attempt <= config.RetryAttempts; attempt++ {
+		if attempt > 0 {
+			// wait before retry
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(config.RetryDelay):
+			}
+		}
+
+		data, err := downloadChunk(ctx, fileURL, chunk, config.Timeout)
+		if err == nil {
+			return data, nil
+		}
+
+		lastErr = err
+	}
+
+	return nil, fmt.Errorf("all %d attempts failed: %w", config.RetryAttempts+1, lastErr)
+}
+
+func downloadChunk(
+	ctx context.Context,
+	fileURL string,
+	chunk [2]int,
+	timeout time.Duration,
+) ([]byte, error) {
+	reqCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, fileURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", chunk[0], chunk[1]))
+
+	session := GetHTTPSession()
+	resp, err := session.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("download failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	return io.ReadAll(resp.Body)
+}
+
+func writeChunkToFile(file *os.File, data []byte, offset int) error {
+	_, err := file.WriteAt(data, int64(offset))
+	return err
+}
+
+func createChunks(fileSize int, chunkSize int) [][2]int {
+	if fileSize <= 0 {
+		return [][2]int{{0, 0}}
+	}
+
+	numChunks := int(math.Ceil(float64(fileSize) / float64(chunkSize)))
+	chunks := make([][2]int, numChunks)
+
+	for i := 0; i < numChunks; i++ {
+		start := i * chunkSize
+		end := start + chunkSize - 1
+		if end >= fileSize {
+			end = fileSize - 1
+		}
+		chunks[i] = [2]int{start, end}
+	}
+
+	return chunks
+}
+
+func DownloadSegments(
+	ctx context.Context,
+	segmentURLs []string,
+	config *models.DownloadConfig,
+) ([]string, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+
+	tempDir := filepath.Join(config.DownloadDir, "segments_"+time.Now().Format("20060102_150405"))
+	if err := os.MkdirAll(tempDir, 0755); err != nil {
+		return nil, fmt.Errorf("failed to create temporary directory: %w", err)
+	}
+
+	semaphore := make(chan struct{}, config.Concurrency)
+	var wg sync.WaitGroup
+
+	errChan := make(chan error, len(segmentURLs))
+
+	downloadedFiles := make([]string, len(segmentURLs))
+
+	for i, segmentURL := range segmentURLs {
+		wg.Add(1)
+		go func(idx int, url string) {
+			defer wg.Done()
+
+			// acquire semaphore slot
+			semaphore <- struct{}{}
+			defer func() { <-semaphore }()
+
+			segmentFileName := fmt.Sprintf("segment_%05d", idx)
+			segmentPath := filepath.Join(tempDir, segmentFileName)
+
+			_, err := DownloadFile(ctx, []string{url}, segmentFileName, &models.DownloadConfig{
+				ChunkSize:       config.ChunkSize,
+				Concurrency:     3, // segments are typically small
+				Timeout:         config.Timeout,
+				DownloadDir:     tempDir,
+				RetryAttempts:   config.RetryAttempts,
+				RetryDelay:      config.RetryDelay,
+				Remux:           false, // don't remux individual segments
+				ProgressUpdater: nil,   // no progress updates for individual segments
+			})
+
+			if err != nil {
+				errChan <- fmt.Errorf("failed to download segment %d: %w", idx, err)
+				return
+			}
+
+			downloadedFiles[idx] = segmentPath
+		}(i, segmentURL)
+	}
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	for err := range errChan {
+		if err != nil {
+			os.RemoveAll(tempDir)
+			return nil, err
+		}
+	}
+
+	return downloadedFiles, nil
+}
+
+func MergeSegmentFiles(
+	ctx context.Context,
+	segmentPaths []string,
+	outputFileName string,
+	config *models.DownloadConfig,
+) (string, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+
+	if err := ensureDownloadDir(config.DownloadDir); err != nil {
+		return "", err
+	}
+
+	outputPath := filepath.Join(config.DownloadDir, outputFileName)
+	outputFile, err := os.Create(outputPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to create output file: %w", err)
+	}
+	defer outputFile.Close()
+
+	var totalBytes int64
+	var processedBytes int64
+
+	if config.ProgressUpdater != nil {
+		for _, segmentPath := range segmentPaths {
+			fileInfo, err := os.Stat(segmentPath)
+			if err == nil {
+				totalBytes += fileInfo.Size()
+			}
+		}
+	}
+
+	for i, segmentPath := range segmentPaths {
+		select {
+		case <-ctx.Done():
+			return "", ctx.Err()
+		default:
+			segmentFile, err := os.Open(segmentPath)
+			if err != nil {
+				return "", fmt.Errorf("failed to open segment %d: %w", i, err)
+			}
+
+			written, err := io.Copy(outputFile, segmentFile)
+			segmentFile.Close()
+
+			if err != nil {
+				return "", fmt.Errorf("failed to copy segment %d: %w", i, err)
+			}
+
+			if config.ProgressUpdater != nil && totalBytes > 0 {
+				processedBytes += written
+				progress := float64(processedBytes) / float64(totalBytes)
+				config.ProgressUpdater(progress)
+			}
+		}
+	}
+
+	if config.Remux {
+		err := av.RemuxFile(outputPath)
+		if err != nil {
+			return "", fmt.Errorf("remuxing failed: %w", err)
+		}
+	}
+
+	return outputPath, nil
+}

util/errors.go

@@ -0,0 +1,23 @@
+package util
+
+type Error struct {
+	Message string
+}
+
+func (err *Error) Error() string {
+	return err.Message
+}
+
+var (
+	ErrUnavailable              = &Error{Message: "this content is unavailable"}
+	ErrNotImplemented           = &Error{Message: "this feature is not implemented"}
+	ErrTimeout                  = &Error{Message: "timeout error when downloading. try again"}
+	ErrUnknownRIFF              = &Error{Message: "uknown RIFF format"}
+	ErrUnsupportedImageFormat   = &Error{Message: "unsupported image format"}
+	ErrFileTooShort             = &Error{Message: "file too short"}
+	ErrDownloadFailed           = &Error{Message: "download failed"}
+	ErrUnsupportedExtractorType = &Error{Message: "unsupported extractor type"}
+	ErrMediaGroupLimitExceeded  = &Error{Message: "media group limit exceeded for this group. try changing /settings"}
+	ErrNSFWNotAllowed           = &Error{Message: "this content is marked as nsfw and can't be downloaded in this group. try changing /settings or use me privately"}
+	ErrInlineMediaGroup         = &Error{Message: "you can't download media groups in inline mode. try using me in a private chat"}
+)

util/http.go

@@ -0,0 +1,14 @@
+package util
+
+import (
+	"net/http"
+	"time"
+)
+
+var httpSession = &http.Client{
+	Timeout: 20 * time.Second,
+}
+
+func GetHTTPSession() *http.Client {
+	return httpSession
+}

util/img.go

@@ -0,0 +1,126 @@
+package util
+
+import (
+	"bytes"
+	"fmt"
+	"image"
+	"image/jpeg"
+	"io"
+	"os"
+
+	_ "image/gif"
+	_ "image/png"
+
+	_ "github.com/strukturag/libheif/go/heif"
+	_ "golang.org/x/image/webp"
+)
+
+var (
+	jpegMagic = []byte{0xFF, 0xD8, 0xFF}
+	pngMagic  = []byte{0x89, 0x50, 0x4E, 0x47}
+	gifMagic  = []byte{0x47, 0x49, 0x46}
+	riffMagic = []byte{0x52, 0x49, 0x46, 0x46}
+	webpMagic = []byte{0x57, 0x45, 0x42, 0x50}
+)
+
+func ImgToJPEG(file io.ReadSeeker, outputPath string) error {
+	format, err := DetectImageFormat(file)
+	if err != nil {
+		return fmt.Errorf("failed to detect image format: %w", err)
+	}
+
+	outputFile, err := os.Create(outputPath)
+	if err != nil {
+		return fmt.Errorf("failed to create output file: %w", err)
+	}
+	defer outputFile.Close()
+
+	if format == "jpeg" {
+		if _, err = file.Seek(0, io.SeekStart); err != nil {
+			return fmt.Errorf("failed to reset file position: %w", err)
+		}
+
+		if _, err = io.Copy(outputFile, file); err != nil {
+			os.Remove(outputPath)
+			return fmt.Errorf("failed to copy image: %w", err)
+		}
+		return nil
+	}
+
+	if _, err = file.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("failed to reset file position: %w", err)
+	}
+
+	img, _, err := image.Decode(file)
+	if err != nil {
+		return fmt.Errorf("failed to decode image: %w", err)
+	}
+
+	err = jpeg.Encode(outputFile, img, nil)
+	if err != nil {
+		os.Remove(outputPath)
+		return fmt.Errorf("failed to encode image: %w", err)
+	}
+
+	return nil
+}
+
+func DetectImageFormat(file io.ReadSeeker) (string, error) {
+	header := make([]byte, 12)
+
+	_, err := file.Read(header)
+	if err != nil {
+		return "", fmt.Errorf("failed to read file header: %w", err)
+	}
+	if _, err = file.Seek(0, io.SeekStart); err != nil {
+		return "", fmt.Errorf("failed to reset file position: %w", err)
+	}
+	if len(header) < 12 {
+		return "", ErrFileTooShort
+	}
+	if bytes.HasPrefix(header, jpegMagic) {
+		return "jpeg", nil
+	}
+
+	if bytes.HasPrefix(header, pngMagic) {
+		return "png", nil
+	}
+
+	if bytes.HasPrefix(header, gifMagic) {
+		return "gif", nil
+	}
+
+	if isHEIF(header) {
+		return "heif", nil
+	}
+
+	if bytes.HasPrefix(header, riffMagic) {
+		if bytes.Equal(header[8:12], webpMagic) {
+			return "webp", nil
+		}
+		return "", ErrUnknownRIFF
+	}
+
+	return "", ErrUnsupportedImageFormat
+}
+
+func isHEIF(header []byte) bool {
+	if len(header) < 12 {
+		return false
+	}
+	isHeifHeader := header[0] == 0x00 && header[1] == 0x00 &&
+		header[2] == 0x00 && (header[3] == 0x18 || header[3] == 0x1C) &&
+		bytes.Equal(header[4:8], []byte("ftyp"))
+	if !isHeifHeader {
+		return false
+	}
+	heifBrands := []string{"heic", "heix", "mif1", "msf1"}
+	brand := string(header[8:12])
+
+	for _, b := range heifBrands {
+		if brand == b {
+			return true
+		}
+	}
+	return false
+}

util/misc.go

@@ -0,0 +1,112 @@
+package util
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/PaulSonOfLars/gotgbot/v2"
+	"github.com/aki237/nscjar"
+)
+
+func GetLocationURL(
+	url string,
+	userAgent string,
+) (string, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %w", err)
+	}
+	if userAgent == "" {
+		userAgent = ChromeUA
+	}
+	req.Header.Set("User-Agent", ChromeUA)
+	session := GetHTTPSession()
+	resp, err := session.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+	return resp.Request.URL.String(), nil
+}
+
+func IsUserAdmin(
+	bot *gotgbot.Bot,
+	chatID int64,
+	userID int64,
+) bool {
+	chatMember, err := bot.GetChatMember(chatID, userID, nil)
+	if err != nil {
+		return false
+	}
+	if chatMember == nil {
+		return false
+	}
+	status := chatMember.GetStatus()
+	switch status {
+	case "creator":
+		return true
+	case "administrator":
+		if chatMember.MergeChatMember().CanChangeInfo {
+			return true
+		}
+		return false
+	}
+	return false
+}
+
+func EscapeCaption(str string) string {
+	// we wont use html.EscapeString
+	// cuz it will escape all the characters
+	// and we only need to escape < and >
+	chars := map[string]string{
+		"<": "&lt;",
+		">": "&gt;",
+	}
+	for k, v := range chars {
+		str = strings.ReplaceAll(str, k, v)
+	}
+	return str
+}
+
+func GetLastError(err error) error {
+	var lastErr error = err
+	for {
+		unwrapped := errors.Unwrap(lastErr)
+		if unwrapped == nil {
+			break
+		}
+		lastErr = unwrapped
+	}
+	return lastErr
+}
+
+func ParseCookieFile(fileName string) ([]*http.Cookie, error) {
+	cookiePath := filepath.Join("cookies", fileName)
+	cookieFile, err := os.Open(cookiePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open cookie file: %w", err)
+	}
+	defer cookieFile.Close()
+
+	var parser nscjar.Parser
+	cookies, err := parser.Unmarshal(cookieFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse cookie file: %w", err)
+	}
+	return cookies, nil
+}
+
+func FixURL(url string) string {
+	return strings.ReplaceAll(url, "&amp;", "&")
+}
+
+func CheckFFmpeg() bool {
+	_, err := exec.LookPath("ffmpeg")
+	return err == nil
+}

util/parser/m3u8.go

@@ -0,0 +1,182 @@
+package parser
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"govd/enums"
+	"govd/models"
+
+	"github.com/grafov/m3u8"
+)
+
+var httpClient = &http.Client{
+	Timeout: 30 * time.Second,
+}
+
+func ParseM3U8Content(
+	content []byte,
+	baseURL string,
+) ([]*models.MediaFormat, error) {
+	baseURLObj, err := url.Parse(baseURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid base url: %w", err)
+	}
+
+	buf := bytes.NewBuffer(content)
+	playlist, listType, err := m3u8.DecodeFrom(buf, true)
+	if err != nil {
+		return nil, fmt.Errorf("failed parsing m3u8: %w", err)
+	}
+
+	var formats []*models.MediaFormat
+
+	if listType == m3u8.MASTER {
+		masterpl := playlist.(*m3u8.MasterPlaylist)
+
+		for _, variant := range masterpl.Variants {
+			if variant == nil || variant.URI == "" {
+				continue
+			}
+
+			width, height := int64(0), int64(0)
+			if variant.Resolution != "" {
+				var w, h int
+				if _, err := fmt.Sscanf(variant.Resolution, "%dx%d", &w, &h); err == nil {
+					width, height = int64(w), int64(h)
+				}
+			}
+
+			format := &models.MediaFormat{
+				Type:       enums.MediaTypeVideo,
+				FormatID:   fmt.Sprintf("hls-%d", variant.Bandwidth/1000),
+				VideoCodec: getCodecFromCodecs(variant.Codecs),
+				AudioCodec: getAudioCodecFromCodecs(variant.Codecs),
+				Bitrate:    int64(variant.Bandwidth),
+				Width:      width,
+				Height:     height,
+			}
+
+			variantURL := resolveURL(baseURLObj, variant.URI)
+			format.URL = []string{variantURL}
+
+			variantContent, err := fetchContent(variantURL)
+			if err == nil {
+				variantFormats, err := ParseM3U8Content(variantContent, variantURL)
+				if err == nil && len(variantFormats) > 0 {
+					format.Segments = variantFormats[0].Segments
+					if variantFormats[0].Duration > 0 {
+						format.Duration = variantFormats[0].Duration
+					}
+				}
+			}
+
+			formats = append(formats, format)
+		}
+
+		return formats, nil
+	}
+
+	if listType == m3u8.MEDIA {
+		mediapl := playlist.(*m3u8.MediaPlaylist)
+
+		var segments []string
+		var totalDuration float64
+
+		for _, segment := range mediapl.Segments {
+			if segment != nil && segment.URI != "" {
+				segmentURL := segment.URI
+				if !strings.HasPrefix(segmentURL, "http://") && !strings.HasPrefix(segmentURL, "https://") {
+					segmentURL = resolveURL(baseURLObj, segmentURL)
+				}
+
+				segments = append(segments, segmentURL)
+				totalDuration += segment.Duration
+			}
+		}
+
+		format := &models.MediaFormat{
+			Type:       enums.MediaTypeVideo,
+			FormatID:   "hls",
+			VideoCodec: enums.MediaCodecAVC,
+			AudioCodec: enums.MediaCodecAAC,
+			Duration:   int64(totalDuration),
+			URL:        []string{baseURL},
+			Segments:   segments,
+		}
+
+		return []*models.MediaFormat{format}, nil
+	}
+
+	return nil, errors.New("unsupported m3u8 playlist type")
+}
+
+func ParseM3U8FromURL(url string) ([]*models.MediaFormat, error) {
+	body, err := fetchContent(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch m3u8 content: %w", err)
+	}
+	return ParseM3U8Content(body, url)
+}
+
+func fetchContent(url string) ([]byte, error) {
+	resp, err := httpClient.Get(url)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch content: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("server returned status code: %d", resp.StatusCode)
+	}
+
+	return io.ReadAll(resp.Body)
+}
+
+func getCodecFromCodecs(codecs string) enums.MediaCodec {
+	if strings.Contains(codecs, "avc") || strings.Contains(codecs, "h264") {
+		return enums.MediaCodecAVC
+	} else if strings.Contains(codecs, "hvc") || strings.Contains(codecs, "h265") {
+		return enums.MediaCodecHEVC
+	} else if strings.Contains(codecs, "av01") {
+		return enums.MediaCodecAV1
+	} else if strings.Contains(codecs, "vp9") {
+		return enums.MediaCodecVP9
+	} else if strings.Contains(codecs, "vp8") {
+		return enums.MediaCodecVP8
+	}
+	return enums.MediaCodecAVC
+}
+
+func getAudioCodecFromCodecs(codecs string) enums.MediaCodec {
+	if strings.Contains(codecs, "mp4a") {
+		return enums.MediaCodecAAC
+	} else if strings.Contains(codecs, "opus") {
+		return enums.MediaCodecOpus
+	} else if strings.Contains(codecs, "mp3") {
+		return enums.MediaCodecMP3
+	} else if strings.Contains(codecs, "flac") {
+		return enums.MediaCodecFLAC
+	} else if strings.Contains(codecs, "vorbis") {
+		return enums.MediaCodecVorbis
+	}
+	return enums.MediaCodecAAC
+}
+
+func resolveURL(base *url.URL, uri string) string {
+	if strings.HasPrefix(uri, "http://") || strings.HasPrefix(uri, "https://") {
+		return uri
+	}
+	ref, err := url.Parse(uri)
+	if err != nil {
+		return uri
+	}
+	return base.ResolveReference(ref).String()
+}

util/parser/mpd.go

@@ -0,0 +1 @@
+package parser