Merge branch 'main' of https://github.com/govdbot/govd

README.md

@@ -1,7 +1,7 @@
 # govd
 a telegram bot for downloading media from various platforms
 
-this project was born after the discontinuation of a highly popular bot known as uvd, and draws significant inspiration from [yt-dlp](https://github.com/yt-dlp/yt-dlp)
+this project draws significant inspiration from [yt-dlp](https://github.com/yt-dlp/yt-dlp)
 
 - official instance: [@govd_bot](https://t.me/govd_bot)
 - support group: [govdsupport](https://t.me/govdsupport)
@@ -28,7 +28,7 @@ this project was born after the discontinuation of a highly popular bot known as
 # installation
 ## build
 > [!NOTE]
-> there's no official support for windows yet. if you want to run the bot on it, please follow [docker installation](#docker-recommended)
+> there's no official support for windows yet. if you want to run the bot on it, please follow [docker installation](#docker-recommended).
 
 1. clone the repository
     ```bash
@@ -54,7 +54,7 @@ this project was born after the discontinuation of a highly popular bot known as
     ```
 
 2. update the `.env` file to ensure the database properties match the environment variables defined for the mariadb service in the `docker-compose.yml` file.  
-   for enhanced security, it is recommended to change the `MYSQL_PASSWORD` property in `docker-compose.yaml` and ensure `DB_PASSWORD` in `.env` matches it.
+   for enhanced security, it is recommended to change the `MARIADB_PASSWORD` property in `docker-compose.yaml` and ensure `DB_PASSWORD` in `.env` matches it.
 
     the following line in the `.env` file **must** be set as:
 

bot/core/inline.go

@@ -3,7 +3,6 @@ package core
 import (
 	"context"
 	"fmt"
-	"log"
 	"sync"
 	"time"
 
@@ -238,11 +237,7 @@ func StartInlineTask(
 			IsPersonal: true,
 		},
 	)
-	if err != nil {
+	if err != nil || !ok {
-		log.Println("failed to answer inline query:", err)
-	}
-	if !ok {
-		log.Println("failed to answer inline query")
 		return nil
 	}
 	SetTask(taskID, dlCtx)

bot/handlers/start.go

@@ -10,9 +10,7 @@ import (
 
 var startMessage = "govd is an open-source telegram bot " +
 	"that allows you to download medias from " +
-	"various platforms. the project born after " +
+	"various platforms."
-	"the discontinuation of an " +
-	"highly popular bot, known as UVD."
 
 func getStartKeyboard(bot *gotgbot.Bot) gotgbot.InlineKeyboardMarkup {
 	return gotgbot.InlineKeyboardMarkup{

ext/instagram/main.go

@@ -10,24 +10,7 @@ import (
 	"regexp"
 )
 
-// as a public service, we can't use the official API
+var instagramHost = []string{"instagram.com"}
-// so we use igram.world API, a third-party service
-// that provides a similar functionality
-// feel free to open PR, if you want to
-// add support for the official Instagram API
-
-const (
-	apiHostname  = "api.igram.world"
-	apiKey       = "aaeaf2805cea6abef3f9d2b6a666fce62fd9d612a43ab772bb50ce81455112e0"
-	apiTimestamp = "1742201548873"
-
-	// todo: Implement a proper way
-	// to get the API key and timestamp
-)
-
-var instagramHost = []string{
-	"instagram.com",
-}
 
 var Extractor = &models.Extractor{
 	Name:       "Instagram",
@@ -39,10 +22,28 @@ var Extractor = &models.Extractor{
 	IsRedirect: false,
 
 	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
-		mediaList, err := MediaListFromAPI(ctx, false)
+		// method 1: get media from GQL web API
+		mediaList, err := GetGQLMediaList(ctx)
+		if err == nil && len(mediaList) > 0 {
 			return &models.ExtractorResponse{
 				MediaList: mediaList,
-		}, err
+			}, nil
+		}
+		// method 2: get media from embed page
+		mediaList, err = GetEmbedMediaList(ctx)
+		if err == nil && len(mediaList) > 0 {
+			return &models.ExtractorResponse{
+				MediaList: mediaList,
+			}, nil
+		}
+		// method 3: get media from 3rd party service (unlikely)
+		mediaList, err = GetIGramMediaList(ctx)
+		if err == nil && len(mediaList) > 0 {
+			return &models.ExtractorResponse{
+				MediaList: mediaList,
+			}, nil
+		}
+		return nil, fmt.Errorf("failed to extract media: all methods failed")
 	},
 }
 
@@ -56,7 +57,7 @@ var StoriesExtractor = &models.Extractor{
 	IsRedirect: false,
 
 	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
-		mediaList, err := MediaListFromAPI(ctx, true)
+		mediaList, err := GetIGramMediaList(ctx)
 		return &models.ExtractorResponse{
 			MediaList: mediaList,
 		}, err
@@ -88,31 +89,63 @@ var ShareURLExtractor = &models.Extractor{
 	},
 }
 
-func MediaListFromAPI(
+func GetGQLMediaList(
 	ctx *models.DownloadContext,
-	stories bool,
 ) ([]*models.Media, error) {
-	client := util.GetHTTPClient(ctx.Extractor.CodeName)
+	graphData, err := GetGQLData(ctx, ctx.MatchedContentID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get graph data: %w", err)
+	}
+	return ParseGQLMedia(ctx, graphData.ShortcodeMedia)
+}
 
+func GetEmbedMediaList(
+	ctx *models.DownloadContext,
+) ([]*models.Media, error) {
+	session := util.GetHTTPClient(ctx.Extractor.CodeName)
+	embedURL := fmt.Sprintf("https://www.instagram.com/p/%s/embed/captioned", ctx.MatchedContentID)
+	req, err := http.NewRequest(
+		http.MethodGet,
+		embedURL,
+		nil,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	for key, value := range igHeaders {
+		req.Header.Set(key, value)
+	}
+	resp, err := session.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get embed page: %s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	graphData, err := ParseEmbedGQL(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse embed page: %w", err)
+	}
+	return ParseGQLMedia(ctx, graphData)
+}
+
+func GetIGramMediaList(ctx *models.DownloadContext) ([]*models.Media, error) {
 	var mediaList []*models.Media
 	postURL := ctx.MatchedContentURL
-	details, err := GetVideoAPI(client, postURL)
+	details, err := GetFromIGram(ctx, postURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get post: %w", err)
 	}
-	var caption string
-	if !stories {
-		caption, err = GetPostCaption(client, postURL)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get caption: %w", err)
-		}
-	}
 	for _, item := range details.Items {
 		media := ctx.Extractor.NewMedia(
 			ctx.MatchedContentID,
 			ctx.MatchedContentURL,
 		)
-		media.SetCaption(caption)
 		urlObj := item.URL[0]
 		contentURL, err := GetCDNURL(urlObj.URL)
 		if err != nil {
@@ -150,26 +183,27 @@ func MediaListFromAPI(
 	return mediaList, nil
 }
 
-func GetVideoAPI(
+func GetFromIGram(
-	client models.HTTPClient,
+	ctx *models.DownloadContext,
 	contentURL string,
 ) (*IGramResponse, error) {
+	session := util.GetHTTPClient(ctx.Extractor.CodeName)
 	apiURL := fmt.Sprintf(
 		"https://%s/api/convert",
-		apiHostname,
+		igramHostname,
 	)
-	payload, err := BuildSignedPayload(contentURL)
+	payload, err := BuildIGramPayload(contentURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build signed payload: %w", err)
 	}
-	req, err := http.NewRequest(http.MethodPost, apiURL, payload)
+	req, err := http.NewRequest("POST", apiURL, payload)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("User-Agent", util.ChromeUA)
 
-	resp, err := client.Do(req)
+	resp, err := session.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

ext/instagram/models.go

@@ -1,17 +1,107 @@
 package instagram
 
+type GraphQLResponse struct {
+	Data   *GraphQLData `json:"data"`
+	Status string       `json:"status"`
+}
+
+type GraphQLData struct {
+	ShortcodeMedia *Media `json:"xdt_shortcode_media"`
+}
+
+type ContextJSON struct {
+	Context *Context `json:"context"`
+	GqlData *GqlData `json:"gql_data"`
+}
+
+type GqlData struct {
+	ShortcodeMedia *Media `json:"shortcode_media"`
+}
+
+type EdgeMediaToCaption struct {
+	Edges []*Edges `json:"edges"`
+}
+
+type EdgeNode struct {
+	Node *Media `json:"node"`
+}
+
+type EdgeSidecarToChildren struct {
+	Edges []*EdgeNode `json:"edges"`
+}
+
+type Dimensions struct {
+	Height int `json:"height"`
+	Width  int `json:"width"`
+}
+
+type DisplayResources struct {
+	ConfigHeight int    `json:"config_height"`
+	ConfigWidth  int    `json:"config_width"`
+	Src          string `json:"src"`
+}
+
+type Node struct {
+	Text string `json:"text"`
+}
+
+type Edges struct {
+	Node *Node `json:"node"`
+}
+type Media struct {
+	Typename              string                 `json:"__typename"`
+	CommenterCount        int                    `json:"commenter_count"`
+	Dimensions            *Dimensions            `json:"dimensions"`
+	DisplayResources      []*DisplayResources    `json:"display_resources"`
+	EdgeMediaToCaption    *EdgeMediaToCaption    `json:"edge_media_to_caption"`
+	EdgeSidecarToChildren *EdgeSidecarToChildren `json:"edge_sidecar_to_children"`
+	DisplayURL            string                 `json:"display_url"`
+	ID                    string                 `json:"id"`
+	IsVideo               bool                   `json:"is_video"`
+	MediaPreview          string                 `json:"media_preview"`
+	Shortcode             string                 `json:"shortcode"`
+	TakenAtTimestamp      int                    `json:"taken_at_timestamp"`
+	Title                 string                 `json:"title"`
+	VideoURL              string                 `json:"video_url"`
+	VideoViewCount        int                    `json:"video_view_count"`
+}
+
+type Posts struct {
+	Src    string `json:"src"`
+	Srcset string `json:"srcset"`
+}
+
+type Context struct {
+	AltText               string `json:"alt_text"`
+	Caption               string `json:"caption"`
+	CaptionTitleLinkified string `json:"caption_title_linkified"`
+	DisplaySrc            string `json:"display_src"`
+	DisplaySrcset         string `json:"display_srcset"`
+	IsIgtv                bool   `json:"is_igtv"`
+	LikesCount            int    `json:"likes_count"`
+	Media                 *Media `json:"media"`
+	MediaPermalink        string `json:"media_permalink"`
+	RequestID             string `json:"request_id"`
+	Shortcode             string `json:"shortcode"`
+	Title                 string `json:"title"`
+	Type                  string `json:"type"`
+	Username              string `json:"username"`
+	Verified              bool   `json:"verified"`
+	VideoViews            int    `json:"video_views"`
+}
+
 type IGramResponse struct {
 	Items []*IGramMedia `json:"items"`
 }
 
 type IGramMedia struct {
-	URL       []*MediaURL `json:"url"`
+	URL       []*IGramMediaURL `json:"url"`
 	Thumb     string           `json:"thumb"`
 	Hosting   string           `json:"hosting"`
 	Timestamp int              `json:"timestamp"`
 }
 
-type MediaURL struct {
+type IGramMediaURL struct {
 	URL  string `json:"url"`
 	Name string `json:"name"`
 	Type string `json:"type"`

ext/instagram/util.go

@@ -1,13 +1,15 @@
 package instagram
 
 import (
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"govd/enums"
 	"govd/models"
 	"govd/util"
-	"html"
 	"io"
+	"math/big"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -16,11 +18,21 @@ import (
 	"time"
 
 	"github.com/bytedance/sonic"
+	"github.com/titanous/json5"
+)
+
+const (
+	graphQLEndpoint = "https://www.instagram.com/graphql/query/"
+	polarisAction   = "PolarisPostActionLoadPostQueryQuery"
+
+	igramHostname  = "api.igram.world"
+	igramKey       = "aaeaf2805cea6abef3f9d2b6a666fce62fd9d612a43ab772bb50ce81455112e0"
+	igramTimestamp = "1742201548873"
 )
 
 var (
-	captionPattern = regexp.MustCompile(
+	embedPattern = regexp.MustCompile(
-		`(?s)<meta property="og:title" content=".*?: &quot;(.*?)&quot;"`)
+		`new ServerJS\(\)\);s\.handle\(({.*})\);requireLazy`)
 
 	igHeaders = map[string]string{
 		"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -40,12 +52,129 @@ var (
 	}
 )
 
-func BuildSignedPayload(contentURL string) (io.Reader, error) {
+func ParseGQLMedia(
+	ctx *models.DownloadContext,
+	data *Media,
+) ([]*models.Media, error) {
+	var mediaList []*models.Media
+
+	var caption string
+	if data.EdgeMediaToCaption != nil && len(data.EdgeMediaToCaption.Edges) > 0 {
+		caption = data.EdgeMediaToCaption.Edges[0].Node.Text
+	}
+
+	mediaType := data.Typename
+	contentID := ctx.MatchedContentID
+	contentURL := ctx.MatchedContentURL
+
+	switch mediaType {
+	case "GraphVideo", "XDTGraphVideo":
+		media := ctx.Extractor.NewMedia(contentID, contentURL)
+		media.SetCaption(caption)
+
+		media.AddFormat(&models.MediaFormat{
+			FormatID:   "video",
+			Type:       enums.MediaTypeVideo,
+			VideoCodec: enums.MediaCodecAVC,
+			AudioCodec: enums.MediaCodecAAC,
+			URL:        []string{data.VideoURL},
+			Thumbnail:  []string{data.DisplayURL},
+			Width:      int64(data.Dimensions.Width),
+			Height:     int64(data.Dimensions.Height),
+		})
+
+		mediaList = append(mediaList, media)
+
+	case "GraphImage", "XDTGraphImage":
+		media := ctx.Extractor.NewMedia(contentID, contentURL)
+		media.SetCaption(caption)
+
+		media.AddFormat(&models.MediaFormat{
+			FormatID: "image",
+			Type:     enums.MediaTypePhoto,
+			URL:      []string{data.DisplayURL},
+		})
+
+		mediaList = append(mediaList, media)
+
+	case "GraphSidecar", "XDTGraphSidecar":
+		if data.EdgeSidecarToChildren != nil && len(data.EdgeSidecarToChildren.Edges) > 0 {
+			for _, edge := range data.EdgeSidecarToChildren.Edges {
+				node := edge.Node
+				media := ctx.Extractor.NewMedia(contentID, contentURL)
+				media.SetCaption(caption)
+
+				switch node.Typename {
+				case "GraphVideo", "XDTGraphVideo":
+					media.AddFormat(&models.MediaFormat{
+						FormatID:   "video",
+						Type:       enums.MediaTypeVideo,
+						VideoCodec: enums.MediaCodecAVC,
+						AudioCodec: enums.MediaCodecAAC,
+						URL:        []string{node.VideoURL},
+						Thumbnail:  []string{node.DisplayURL},
+						Width:      int64(node.Dimensions.Width),
+						Height:     int64(node.Dimensions.Height),
+					})
+
+				case "GraphImage", "XDTGraphImage":
+
+					media.AddFormat(&models.MediaFormat{
+						FormatID: "image",
+						Type:     enums.MediaTypePhoto,
+						URL:      []string{node.DisplayURL},
+					})
+				}
+
+				mediaList = append(mediaList, media)
+			}
+		}
+	}
+
+	return mediaList, nil
+}
+
+func ParseEmbedGQL(
+	body []byte,
+) (*Media, error) {
+	match := embedPattern.FindStringSubmatch(string(body))
+	if len(match) < 2 {
+		return nil, fmt.Errorf("failed to find JSON in response")
+	}
+	jsonData := match[1]
+
+	var data map[string]interface{}
+	if err := json5.Unmarshal([]byte(jsonData), &data); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+	}
+	igCtx := util.TraverseJSON(data, "contextJSON")
+	if igCtx == nil {
+		return nil, fmt.Errorf("contextJSON not found in data")
+	}
+	var ctxJSON ContextJSON
+	switch v := igCtx.(type) {
+	case string:
+		if err := json5.Unmarshal([]byte(v), &ctxJSON); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal contextJSON: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("contextJSON is not a string")
+	}
+	if ctxJSON.GqlData == nil {
+		return nil, fmt.Errorf("gql_data is nil")
+	}
+	if ctxJSON.GqlData.ShortcodeMedia == nil {
+		return nil, fmt.Errorf("media is nil")
+	}
+	return ctxJSON.GqlData.ShortcodeMedia, nil
+}
+
+func BuildIGramPayload(contentURL string) (io.Reader, error) {
 	timestamp := strconv.FormatInt(time.Now().UnixMilli(), 10)
 	hash := sha256.New()
 	_, err := io.WriteString(
 		hash,
-		contentURL+timestamp+apiKey,
+		contentURL+timestamp+igramKey,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("error writing to SHA256 hash: %w", err)
@@ -56,7 +185,7 @@ func BuildSignedPayload(contentURL string) (io.Reader, error) {
 	payload := map[string]string{
 		"url":  contentURL,
 		"ts":   timestamp,
-		"_ts":  apiTimestamp,
+		"_ts":  igramTimestamp,
 		"_tsc": "0", // ?
 		"_s":   secretString,
 	}
@@ -69,15 +198,14 @@ func BuildSignedPayload(contentURL string) (io.Reader, error) {
 }
 
 func ParseIGramResponse(body []byte) (*IGramResponse, error) {
-	var rawResponse interface{}
+	var rawResponse any
-	//move to the start of the body
+
-	// Use sonic's decoder to unmarshal the raw response
 	if err := sonic.ConfigFastest.Unmarshal(body, &rawResponse); err != nil {
 		return nil, fmt.Errorf("failed to decode response1: %w", err)
 	}
 
 	switch rawResponse.(type) {
-	case []interface{}:
+	case []any:
 		// array of IGramMedia
 		var media []*IGramMedia
 		if err := sonic.ConfigFastest.Unmarshal(body, &media); err != nil {
@@ -86,7 +214,7 @@ func ParseIGramResponse(body []byte) (*IGramResponse, error) {
 		return &IGramResponse{
 			Items: media,
 		}, nil
-	case map[string]interface{}:
+	case map[string]any:
 		// single IGramMedia
 		var media IGramMedia
 		if err := sonic.ConfigFastest.Unmarshal(body, &media); err != nil {
@@ -113,53 +241,142 @@ func GetCDNURL(contentURL string) (string, error) {
 	return cdnURL, nil
 }
 
-func GetPostCaption(
+func GetGQLData(
-	client models.HTTPClient,
+	ctx *models.DownloadContext,
-	postURL string,
+	shortcode string,
-) (string, error) {
+) (*GraphQLData, error) {
+	session := util.GetHTTPClient(ctx.Extractor.CodeName)
+	graphHeaders, body, err := BuildGQLData()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build GQL data: %w", err)
+	}
+	formData := url.Values{}
+	for key, value := range body {
+		formData.Set(key, value)
+	}
+	formData.Set("fb_api_caller_class", "RelayModern")
+	formData.Set("fb_api_req_friendly_name", polarisAction)
+	variables := map[string]any{
+		"shortcode":               shortcode,
+		"fetch_tagged_user_count": nil,
+		"hoisted_comment_id":      nil,
+		"hoisted_reply_id":        nil,
+	}
+	variablesJSON, err := sonic.ConfigFastest.Marshal(variables)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal variables: %w", err)
+	}
+	formData.Set("variables", string(variablesJSON))
+	formData.Set("server_timestamps", "true")
+	formData.Set("doc_id", "8845758582119845") // idk what this is
 	req, err := http.NewRequest(
-		http.MethodGet,
+		http.MethodPost,
-		postURL,
+		graphQLEndpoint,
-		nil,
+		strings.NewReader(formData.Encode()),
 	)
 	if err != nil {
-		return "", fmt.Errorf("failed to create request: %w", err)
+		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
-	req.Header.Set("User-Agent", util.ChromeUA)
+	for key, value := range igHeaders {
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set(key, value)
-	req.Header.Set("Accept-Language", "it-IT,it;q=0.8,en-US;q=0.5,en;q=0.3")
+	}
-	req.Header.Set("Referer", "https://www.instagram.com/accounts/onetap/?next=%2F")
+	for key, value := range graphHeaders {
-	req.Header.Set("Alt-Used", "www.instagram.com")
+		req.Header.Set(key, value)
-	req.Header.Set("Connection", "keep-alive")
+	}
-	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	resp, err := session.Do(req)
-	req.Header.Set("Sec-Fetch-Dest", "document")
-	req.Header.Set("Sec-Fetch-Mode", "navigate")
-	req.Header.Set("Sec-Fetch-Site", "same-origin")
-	req.Header.Set("Priority", "u=0, i")
-	req.Header.Set("Pragma", "no-cache")
-	req.Header.Set("Cache-Control", "no-cache")
-	req.Header.Set("TE", "trailers")
-
-	resp, err := client.Do(req)
 	if err != nil {
-		return "", fmt.Errorf("failed to send request: %w", err)
+		return nil, fmt.Errorf("failed to send request: %w", err)
 	}
 	defer resp.Body.Close()
-
 	if resp.StatusCode != http.StatusOK {
-		// return an empty caption
+		return nil, fmt.Errorf("invalid response code: %s", resp.Status)
-		// probably 429 error
-		return "", nil
 	}
-	body, err := io.ReadAll(resp.Body)
+	var response GraphQLResponse
-	if err != nil {
+	decoder := sonic.ConfigFastest.NewDecoder(resp.Body)
-		return "", fmt.Errorf("failed to read response body: %w", err)
+	if err := decoder.Decode(&response); err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+	if response.Data == nil {
+		return nil, fmt.Errorf("data is nil")
+	}
+	if response.Status != "ok" {
+		return nil, fmt.Errorf("status is not ok: %s", response.Status)
+	}
+	if response.Data.ShortcodeMedia == nil {
+		return nil, fmt.Errorf("media is nil")
+	}
+	return response.Data, nil
 }
 
-	matches := captionPattern.FindStringSubmatch(string(body))
+func BuildGQLData() (map[string]string, map[string]string, error) {
-	if len(matches) < 2 {
+	const (
-		// post has no caption most likely
+		domain                = "www"
-		return "", nil
+		requestID             = "b"
+		clientCapabilityGrade = "EXCELLENT"
+		sessionInternalID     = "7436540909012459023"
+		apiVersion            = "1"
+		rolloutHash           = "1019933358"
+		appID                 = "936619743392459"
+		bloksVersionID        = "6309c8d03d8a3f47a1658ba38b304a3f837142ef5f637ebf1f8f52d4b802951e"
+		asbdID                = "129477"
+		hiddenState           = "20126.HYP:instagram_web_pkg.2.1...0"
+		loggedIn              = "0"
+		cometRequestID        = "7"
+		appVersion            = "0"
+		pixelRatio            = "2"
+		buildType             = "trunk"
+	)
+	session := "::" + util.RandomAlphaString(6)
+	sessionData := util.RandomBase64(8)
+	csrfToken := util.RandomBase64(32)
+	deviceID := util.RandomBase64(24)
+	machineID := util.RandomBase64(24)
+	dynamicFlags := util.RandomBase64(154)
+	clientSessionRnd := util.RandomBase64(154)
+	jazoestBig, err := rand.Int(rand.Reader, big.NewInt(10000))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate jazoest: %w", err)
 	}
-	return html.UnescapeString(matches[1]), nil
+	jazoest := strconv.FormatInt(jazoestBig.Int64()+1, 10)
+	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+	cookies := []string{
+		"csrftoken=" + csrfToken,
+		"ig_did=" + deviceID,
+		"wd=1280x720",
+		"dpr=2",
+		"mid=" + machineID,
+		"ig_nrcb=1",
+	}
+	headers := map[string]string{
+		"x-ig-app-id":        appID,
+		"X-FB-LSD":           sessionData,
+		"X-CSRFToken":        csrfToken,
+		"X-Bloks-Version-Id": bloksVersionID,
+		"x-asbd-id":          asbdID,
+		"cookie":             strings.Join(cookies, "; "),
+		"Content-Type":       "application/x-www-form-urlencoded",
+		"X-FB-Friendly-Name": polarisAction,
+	}
+	body := map[string]string{
+		"__d":         domain,
+		"__a":         apiVersion,
+		"__s":         session,
+		"__hs":        hiddenState,
+		"__req":       requestID,
+		"__ccg":       clientCapabilityGrade,
+		"__rev":       rolloutHash,
+		"__hsi":       sessionInternalID,
+		"__dyn":       dynamicFlags,
+		"__csr":       clientSessionRnd,
+		"__user":      loggedIn,
+		"__comet_req": cometRequestID,
+		"av":          appVersion,
+		"dpr":         pixelRatio,
+		"lsd":         sessionData,
+		"jazoest":     jazoest,
+		"__spin_r":    rolloutHash,
+		"__spin_b":    buildType,
+		"__spin_t":    timestamp,
+	}
+	return headers, body, nil
 }

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/strukturag/libheif v1.19.7
 	github.com/u2takey/ffmpeg-go v0.5.0
 	golang.org/x/image v0.26.0
+	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/gorm v1.25.12
 )
 
@@ -26,7 +27,6 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
@@ -38,6 +38,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/titanous/json5 v1.0.0
 	github.com/u2takey/go-utils v0.3.1 // indirect
 	golang.org/x/text v0.24.0 // indirect
 	gorm.io/driver/mysql v1.5.7

go.sum

@@ -51,6 +51,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -61,6 +63,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0=
+github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -76,6 +80,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/strukturag/libheif v1.19.7 h1:XMfSJvmnucTbiS6CSxxZmpx5XSPjdqkpA3wiL6+I2Iw=
 github.com/strukturag/libheif v1.19.7/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks=
+github.com/titanous/json5 v1.0.0 h1:hJf8Su1d9NuI/ffpxgxQfxh/UiBFZX7bMPid0rIL/7s=
+github.com/titanous/json5 v1.0.0/go.mod h1:7JH1M8/LHKc6cyP5o5g3CSaRj+mBrIimTxzpvmckH8c=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU=
@@ -104,7 +110,10 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
+gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

util/http.go

@@ -2,7 +2,6 @@ package util
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"govd/config"
 	"govd/models"
@@ -14,6 +13,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/bytedance/sonic"
 )
 
 var (
@@ -235,7 +236,7 @@ func parseProxyResponse(proxyResp *http.Response, originalReq *http.Request) (*h
 	}
 
 	var response models.ProxyResponse
-	if err := json.Unmarshal(body, &response); err != nil {
+	if err := sonic.ConfigFastest.Unmarshal(body, &response); err != nil {
 		return nil, fmt.Errorf("error parsing proxy response: %w", err)
 	}
 

util/misc.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"crypto/rand"
 	"fmt"
 	"govd/models"
 	"net/http"
@@ -95,6 +96,44 @@ func GetLastError(err error) error {
 	return lastErr
 }
 
+func RandomBase64(length int) string {
+	const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+	const mask = 63 // 6 bits, since len(letters) == 64
+
+	result := make([]byte, length)
+	random := make([]byte, length)
+	_, err := rand.Read(random)
+	if err != nil {
+		return strings.Repeat("A", length)
+	}
+	for i, b := range random {
+		result[i] = letters[int(b)&mask]
+	}
+	return string(result)
+}
+
+func RandomAlphaString(length int) string {
+	const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	const lettersLen = byte(len(letters))
+	const maxByte = 255 - (255 % lettersLen) // 255 - (255 % 52) = 255 - 47 = 208
+
+	result := make([]byte, length)
+	i := 0
+	for i < length {
+		b := make([]byte, 1)
+		_, err := rand.Read(b)
+		if err != nil {
+			return strings.Repeat("a", length)
+		}
+		if b[0] > maxByte {
+			continue // avoid bias
+		}
+		result[i] = letters[b[0]%lettersLen]
+		i++
+	}
+	return string(result)
+}
+
 func ParseCookieFile(fileName string) ([]*http.Cookie, error) {
 	cachedCookies, ok := cookiesCache[fileName]
 	if ok {