instagram: new extraction method

new extraction method first tries to fetch content directly from instagram graphql API, fallback to html embed page. in case every method fail, rely on 3rd party

README.md

@@ -1,7 +1,7 @@
 # govd
 a telegram bot for downloading media from various platforms
 
-this project was born after the discontinuation of a highly popular bot known as uvd, and draws significant inspiration from [yt-dlp](https://github.com/yt-dlp/yt-dlp)
+this project draws significant inspiration from [yt-dlp](https://github.com/yt-dlp/yt-dlp)
 
 - official instance: [@govd_bot](https://t.me/govd_bot)
 - support group: [govdsupport](https://t.me/govdsupport)
@@ -28,7 +28,7 @@ this project was born after the discontinuation of a highly popular bot known as
 # installation
 ## build
 > [!NOTE]
-> there's no official support for windows yet. if you want to run the bot on it, please follow [docker installation](#docker-recommended)
+> there's no official support for windows yet. if you want to run the bot on it, please follow [docker installation](#docker-recommended).
 
 1. clone the repository
     ```bash
@@ -54,7 +54,7 @@ this project was born after the discontinuation of a highly popular bot known as
     ```
 
 2. update the `.env` file to ensure the database properties match the environment variables defined for the mariadb service in the `docker-compose.yml` file.  
-   for enhanced security, it is recommended to change the `MYSQL_PASSWORD` property in `docker-compose.yaml` and ensure `DB_PASSWORD` in `.env` matches it.
+   for enhanced security, it is recommended to change the `MARIADB_PASSWORD` property in `docker-compose.yaml` and ensure `DB_PASSWORD` in `.env` matches it.
 
     the following line in the `.env` file **must** be set as:
 

bot/core/inline.go

@@ -3,7 +3,6 @@ package core
 import (
 	"context"
 	"fmt"
-	"log"
 	"sync"
 	"time"
 
@@ -238,11 +237,7 @@ func StartInlineTask(
 			IsPersonal: true,
 		},
 	)
-	if err != nil {
+	if err != nil || !ok {
-		log.Println("failed to answer inline query:", err)
-	}
-	if !ok {
-		log.Println("failed to answer inline query")
 		return nil
 	}
 	SetTask(taskID, dlCtx)

bot/handlers/start.go

@@ -10,9 +10,7 @@ import (
 
 var startMessage = "govd is an open-source telegram bot " +
 	"that allows you to download medias from " +
-	"various platforms. the project born after " +
+	"various platforms."
-	"the discontinuation of an " +
-	"highly popular bot, known as UVD."
 
 func getStartKeyboard(bot *gotgbot.Bot) gotgbot.InlineKeyboardMarkup {
 	return gotgbot.InlineKeyboardMarkup{

ext/instagram/main.go

@@ -10,24 +10,7 @@ import (
 	"regexp"
 )
 
-// as a public service, we can't use the official API
+var instagramHost = []string{"instagram.com"}
-// so we use igram.world API, a third-party service
-// that provides a similar functionality
-// feel free to open PR, if you want to
-// add support for the official Instagram API
-
-const (
-	apiHostname  = "api.igram.world"
-	apiKey       = "aaeaf2805cea6abef3f9d2b6a666fce62fd9d612a43ab772bb50ce81455112e0"
-	apiTimestamp = "1742201548873"
-
-	// todo: Implement a proper way
-	// to get the API key and timestamp
-)
-
-var instagramHost = []string{
-	"instagram.com",
-}
 
 var Extractor = &models.Extractor{
 	Name:       "Instagram",
@@ -39,10 +22,28 @@ var Extractor = &models.Extractor{
 	IsRedirect: false,
 
 	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
-		mediaList, err := MediaListFromAPI(ctx, false)
+		// method 1: get media from GQL web API
+		mediaList, err := GetGQLMediaList(ctx)
+		if err == nil && len(mediaList) > 0 {
 			return &models.ExtractorResponse{
 				MediaList: mediaList,
-		}, err
+			}, nil
+		}
+		// method 2: get media from embed page
+		mediaList, err = GetEmbedMediaList(ctx)
+		if err == nil && len(mediaList) > 0 {
+			return &models.ExtractorResponse{
+				MediaList: mediaList,
+			}, nil
+		}
+		// method 3: get media from 3rd party service (unlikely)
+		mediaList, err = GetIGramMediaList(ctx)
+		if err == nil && len(mediaList) > 0 {
+			return &models.ExtractorResponse{
+				MediaList: mediaList,
+			}, nil
+		}
+		return nil, fmt.Errorf("failed to extract media: all methods failed")
 	},
 }
 
@@ -56,7 +57,7 @@ var StoriesExtractor = &models.Extractor{
 	IsRedirect: false,
 
 	Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
-		mediaList, err := MediaListFromAPI(ctx, true)
+		mediaList, err := GetIGramMediaList(ctx)
 		return &models.ExtractorResponse{
 			MediaList: mediaList,
 		}, err
@@ -88,31 +89,63 @@ var ShareURLExtractor = &models.Extractor{
 	},
 }
 
-func MediaListFromAPI(
+func GetGQLMediaList(
 	ctx *models.DownloadContext,
-	stories bool,
 ) ([]*models.Media, error) {
-	client := util.GetHTTPClient(ctx.Extractor.CodeName)
+	graphData, err := GetGQLData(ctx, ctx.MatchedContentID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get graph data: %w", err)
+	}
+	return ParseGQLMedia(ctx, graphData.ShortcodeMedia)
+}
 
+func GetEmbedMediaList(
+	ctx *models.DownloadContext,
+) ([]*models.Media, error) {
+	session := util.GetHTTPClient(ctx.Extractor.CodeName)
+	embedURL := fmt.Sprintf("https://www.instagram.com/p/%s/embed/captioned", ctx.MatchedContentID)
+	req, err := http.NewRequest(
+		http.MethodGet,
+		embedURL,
+		nil,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	for key, value := range igHeaders {
+		req.Header.Set(key, value)
+	}
+	resp, err := session.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to get embed page: %s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+	graphData, err := ParseEmbedGQL(body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse embed page: %w", err)
+	}
+	return ParseGQLMedia(ctx, graphData)
+}
+
+func GetIGramMediaList(ctx *models.DownloadContext) ([]*models.Media, error) {
 	var mediaList []*models.Media
 	postURL := ctx.MatchedContentURL
-	details, err := GetVideoAPI(client, postURL)
+	details, err := GetFromIGram(ctx, postURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get post: %w", err)
 	}
-	var caption string
-	if !stories {
-		caption, err = GetPostCaption(client, postURL)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get caption: %w", err)
-		}
-	}
 	for _, item := range details.Items {
 		media := ctx.Extractor.NewMedia(
 			ctx.MatchedContentID,
 			ctx.MatchedContentURL,
 		)
-		media.SetCaption(caption)
 		urlObj := item.URL[0]
 		contentURL, err := GetCDNURL(urlObj.URL)
 		if err != nil {
@@ -150,26 +183,27 @@ func MediaListFromAPI(
 	return mediaList, nil
 }
 
-func GetVideoAPI(
+func GetFromIGram(
-	client models.HTTPClient,
+	ctx *models.DownloadContext,
 	contentURL string,
 ) (*IGramResponse, error) {
+	session := util.GetHTTPClient(ctx.Extractor.CodeName)
 	apiURL := fmt.Sprintf(
 		"https://%s/api/convert",
-		apiHostname,
+		igramHostname,
 	)
-	payload, err := BuildSignedPayload(contentURL)
+	payload, err := BuildIGramPayload(contentURL)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build signed payload: %w", err)
 	}
-	req, err := http.NewRequest(http.MethodPost, apiURL, payload)
+	req, err := http.NewRequest("POST", apiURL, payload)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("User-Agent", util.ChromeUA)
 
-	resp, err := client.Do(req)
+	resp, err := session.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("failed to send request: %w", err)
 	}

ext/instagram/models.go

@@ -1,17 +1,107 @@
 package instagram
 
+type GraphQLResponse struct {
+	Data   *GraphQLData `json:"data"`
+	Status string       `json:"status"`
+}
+
+type GraphQLData struct {
+	ShortcodeMedia *Media `json:"xdt_shortcode_media"`
+}
+
+type ContextJSON struct {
+	Context *Context `json:"context"`
+	GqlData *GqlData `json:"gql_data"`
+}
+
+type GqlData struct {
+	ShortcodeMedia *Media `json:"shortcode_media"`
+}
+
+type EdgeMediaToCaption struct {
+	Edges []*Edges `json:"edges"`
+}
+
+type EdgeNode struct {
+	Node *Media `json:"node"`
+}
+
+type EdgeSidecarToChildren struct {
+	Edges []*EdgeNode `json:"edges"`
+}
+
+type Dimensions struct {
+	Height int `json:"height"`
+	Width  int `json:"width"`
+}
+
+type DisplayResources struct {
+	ConfigHeight int    `json:"config_height"`
+	ConfigWidth  int    `json:"config_width"`
+	Src          string `json:"src"`
+}
+
+type Node struct {
+	Text string `json:"text"`
+}
+
+type Edges struct {
+	Node *Node `json:"node"`
+}
+type Media struct {
+	Typename              string                 `json:"__typename"`
+	CommenterCount        int                    `json:"commenter_count"`
+	Dimensions            *Dimensions            `json:"dimensions"`
+	DisplayResources      []*DisplayResources    `json:"display_resources"`
+	EdgeMediaToCaption    *EdgeMediaToCaption    `json:"edge_media_to_caption"`
+	EdgeSidecarToChildren *EdgeSidecarToChildren `json:"edge_sidecar_to_children"`
+	DisplayURL            string                 `json:"display_url"`
+	ID                    string                 `json:"id"`
+	IsVideo               bool                   `json:"is_video"`
+	MediaPreview          string                 `json:"media_preview"`
+	Shortcode             string                 `json:"shortcode"`
+	TakenAtTimestamp      int                    `json:"taken_at_timestamp"`
+	Title                 string                 `json:"title"`
+	VideoURL              string                 `json:"video_url"`
+	VideoViewCount        int                    `json:"video_view_count"`
+}
+
+type Posts struct {
+	Src    string `json:"src"`
+	Srcset string `json:"srcset"`
+}
+
+type Context struct {
+	AltText               string `json:"alt_text"`
+	Caption               string `json:"caption"`
+	CaptionTitleLinkified string `json:"caption_title_linkified"`
+	DisplaySrc            string `json:"display_src"`
+	DisplaySrcset         string `json:"display_srcset"`
+	IsIgtv                bool   `json:"is_igtv"`
+	LikesCount            int    `json:"likes_count"`
+	Media                 *Media `json:"media"`
+	MediaPermalink        string `json:"media_permalink"`
+	RequestID             string `json:"request_id"`
+	Shortcode             string `json:"shortcode"`
+	Title                 string `json:"title"`
+	Type                  string `json:"type"`
+	Username              string `json:"username"`
+	Verified              bool   `json:"verified"`
+	VideoViews            int    `json:"video_views"`
+}
+
 type IGramResponse struct {
 	Items []*IGramMedia `json:"items"`
 }
 
 type IGramMedia struct {
-	URL       []*MediaURL `json:"url"`
+	URL       []*IGramMediaURL `json:"url"`
 	Thumb     string           `json:"thumb"`
 	Hosting   string           `json:"hosting"`
 	Timestamp int              `json:"timestamp"`
 }
 
-type MediaURL struct {
+type IGramMediaURL struct {
 	URL  string `json:"url"`
 	Name string `json:"name"`
 	Type string `json:"type"`

ext/instagram/util.go

@@ -1,13 +1,15 @@
 package instagram
 
 import (
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"govd/enums"
 	"govd/models"
 	"govd/util"
-	"html"
 	"io"
+	"math/big"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -16,11 +18,21 @@ import (
 	"time"
 
 	"github.com/bytedance/sonic"
+	"github.com/titanous/json5"
+)
+
+const (
+	graphQLEndpoint = "https://www.instagram.com/graphql/query/"
+	polarisAction   = "PolarisPostActionLoadPostQueryQuery"
+
+	igramHostname  = "api.igram.world"
+	igramKey       = "aaeaf2805cea6abef3f9d2b6a666fce62fd9d612a43ab772bb50ce81455112e0"
+	igramTimestamp = "1742201548873"
 )
 
 var (
-	captionPattern = regexp.MustCompile(
+	embedPattern = regexp.MustCompile(
-		`(?s)<meta property="og:title" content=".*?: &quot;(.*?)&quot;"`)
+		`new ServerJS\(\)\);s\.handle\(({.*})\);requireLazy`)
 
 	igHeaders = map[string]string{
 		"Accept":                    "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
@@ -40,12 +52,129 @@ var (
 	}
 )
 
-func BuildSignedPayload(contentURL string) (io.Reader, error) {
+func ParseGQLMedia(
+	ctx *models.DownloadContext,
+	data *Media,
+) ([]*models.Media, error) {
+	var mediaList []*models.Media
+
+	var caption string
+	if data.EdgeMediaToCaption != nil && len(data.EdgeMediaToCaption.Edges) > 0 {
+		caption = data.EdgeMediaToCaption.Edges[0].Node.Text
+	}
+
+	mediaType := data.Typename
+	contentID := ctx.MatchedContentID
+	contentURL := ctx.MatchedContentURL
+
+	switch mediaType {
+	case "GraphVideo", "XDTGraphVideo":
+		media := ctx.Extractor.NewMedia(contentID, contentURL)
+		media.SetCaption(caption)
+
+		media.AddFormat(&models.MediaFormat{
+			FormatID:   "video",
+			Type:       enums.MediaTypeVideo,
+			VideoCodec: enums.MediaCodecAVC,
+			AudioCodec: enums.MediaCodecAAC,
+			URL:        []string{data.VideoURL},
+			Thumbnail:  []string{data.DisplayURL},
+			Width:      int64(data.Dimensions.Width),
+			Height:     int64(data.Dimensions.Height),
+		})
+
+		mediaList = append(mediaList, media)
+
+	case "GraphImage", "XDTGraphImage":
+		media := ctx.Extractor.NewMedia(contentID, contentURL)
+		media.SetCaption(caption)
+
+		media.AddFormat(&models.MediaFormat{
+			FormatID: "image",
+			Type:     enums.MediaTypePhoto,
+			URL:      []string{data.DisplayURL},
+		})
+
+		mediaList = append(mediaList, media)
+
+	case "GraphSidecar", "XDTGraphSidecar":
+		if data.EdgeSidecarToChildren != nil && len(data.EdgeSidecarToChildren.Edges) > 0 {
+			for _, edge := range data.EdgeSidecarToChildren.Edges {
+				node := edge.Node
+				media := ctx.Extractor.NewMedia(contentID, contentURL)
+				media.SetCaption(caption)
+
+				switch node.Typename {
+				case "GraphVideo", "XDTGraphVideo":
+					media.AddFormat(&models.MediaFormat{
+						FormatID:   "video",
+						Type:       enums.MediaTypeVideo,
+						VideoCodec: enums.MediaCodecAVC,
+						AudioCodec: enums.MediaCodecAAC,
+						URL:        []string{node.VideoURL},
+						Thumbnail:  []string{node.DisplayURL},
+						Width:      int64(node.Dimensions.Width),
+						Height:     int64(node.Dimensions.Height),
+					})
+
+				case "GraphImage", "XDTGraphImage":
+
+					media.AddFormat(&models.MediaFormat{
+						FormatID: "image",
+						Type:     enums.MediaTypePhoto,
+						URL:      []string{node.DisplayURL},
+					})
+				}
+
+				mediaList = append(mediaList, media)
+			}
+		}
+	}
+
+	return mediaList, nil
+}
+
+func ParseEmbedGQL(
+	body []byte,
+) (*Media, error) {
+	match := embedPattern.FindStringSubmatch(string(body))
+	if len(match) < 2 {
+		return nil, fmt.Errorf("failed to find JSON in response")
+	}
+	jsonData := match[1]
+
+	var data map[string]interface{}
+	if err := json5.Unmarshal([]byte(jsonData), &data); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+	}
+	igCtx := util.TraverseJSON(data, "contextJSON")
+	if igCtx == nil {
+		return nil, fmt.Errorf("contextJSON not found in data")
+	}
+	var ctxJSON ContextJSON
+	switch v := igCtx.(type) {
+	case string:
+		if err := json5.Unmarshal([]byte(v), &ctxJSON); err != nil {
+			return nil, fmt.Errorf("failed to unmarshal contextJSON: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("contextJSON is not a string")
+	}
+	if ctxJSON.GqlData == nil {
+		return nil, fmt.Errorf("gql_data is nil")
+	}
+	if ctxJSON.GqlData.ShortcodeMedia == nil {
+		return nil, fmt.Errorf("media is nil")
+	}
+	return ctxJSON.GqlData.ShortcodeMedia, nil
+}
+
+func BuildIGramPayload(contentURL string) (io.Reader, error) {
 	timestamp := strconv.FormatInt(time.Now().UnixMilli(), 10)
 	hash := sha256.New()
 	_, err := io.WriteString(
 		hash,
-		contentURL+timestamp+apiKey,
+		contentURL+timestamp+igramKey,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("error writing to SHA256 hash: %w", err)
@@ -56,7 +185,7 @@ func BuildSignedPayload(contentURL string) (io.Reader, error) {
 	payload := map[string]string{
 		"url":  contentURL,
 		"ts":   timestamp,
-		"_ts":  apiTimestamp,
+		"_ts":  igramTimestamp,
 		"_tsc": "0", // ?
 		"_s":   secretString,
 	}
@@ -69,15 +198,14 @@ func BuildSignedPayload(contentURL string) (io.Reader, error) {
 }
 
 func ParseIGramResponse(body []byte) (*IGramResponse, error) {
-	var rawResponse interface{}
+	var rawResponse any
-	//move to the start of the body
+
-	// Use sonic's decoder to unmarshal the raw response
 	if err := sonic.ConfigFastest.Unmarshal(body, &rawResponse); err != nil {
 		return nil, fmt.Errorf("failed to decode response1: %w", err)
 	}
 
 	switch rawResponse.(type) {
-	case []interface{}:
+	case []any:
 		// array of IGramMedia
 		var media []*IGramMedia
 		if err := sonic.ConfigFastest.Unmarshal(body, &media); err != nil {
@@ -86,7 +214,7 @@ func ParseIGramResponse(body []byte) (*IGramResponse, error) {
 		return &IGramResponse{
 			Items: media,
 		}, nil
-	case map[string]interface{}:
+	case map[string]any:
 		// single IGramMedia
 		var media IGramMedia
 		if err := sonic.ConfigFastest.Unmarshal(body, &media); err != nil {
@@ -113,53 +241,142 @@ func GetCDNURL(contentURL string) (string, error) {
 	return cdnURL, nil
 }
 
-func GetPostCaption(
+func GetGQLData(
-	client models.HTTPClient,
+	ctx *models.DownloadContext,
-	postURL string,
+	shortcode string,
-) (string, error) {
+) (*GraphQLData, error) {
+	session := util.GetHTTPClient(ctx.Extractor.CodeName)
+	graphHeaders, body, err := BuildGQLData()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build GQL data: %w", err)
+	}
+	formData := url.Values{}
+	for key, value := range body {
+		formData.Set(key, value)
+	}
+	formData.Set("fb_api_caller_class", "RelayModern")
+	formData.Set("fb_api_req_friendly_name", polarisAction)
+	variables := map[string]any{
+		"shortcode":               shortcode,
+		"fetch_tagged_user_count": nil,
+		"hoisted_comment_id":      nil,
+		"hoisted_reply_id":        nil,
+	}
+	variablesJSON, err := sonic.ConfigFastest.Marshal(variables)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal variables: %w", err)
+	}
+	formData.Set("variables", string(variablesJSON))
+	formData.Set("server_timestamps", "true")
+	formData.Set("doc_id", "8845758582119845") // idk what this is
 	req, err := http.NewRequest(
-		http.MethodGet,
+		http.MethodPost,
-		postURL,
+		graphQLEndpoint,
-		nil,
+		strings.NewReader(formData.Encode()),
 	)
 	if err != nil {
-		return "", fmt.Errorf("failed to create request: %w", err)
+		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
-	req.Header.Set("User-Agent", util.ChromeUA)
+	for key, value := range igHeaders {
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set(key, value)
-	req.Header.Set("Accept-Language", "it-IT,it;q=0.8,en-US;q=0.5,en;q=0.3")
+	}
-	req.Header.Set("Referer", "https://www.instagram.com/accounts/onetap/?next=%2F")
+	for key, value := range graphHeaders {
-	req.Header.Set("Alt-Used", "www.instagram.com")
+		req.Header.Set(key, value)
-	req.Header.Set("Connection", "keep-alive")
+	}
-	req.Header.Set("Upgrade-Insecure-Requests", "1")
+	resp, err := session.Do(req)
-	req.Header.Set("Sec-Fetch-Dest", "document")
-	req.Header.Set("Sec-Fetch-Mode", "navigate")
-	req.Header.Set("Sec-Fetch-Site", "same-origin")
-	req.Header.Set("Priority", "u=0, i")
-	req.Header.Set("Pragma", "no-cache")
-	req.Header.Set("Cache-Control", "no-cache")
-	req.Header.Set("TE", "trailers")
-
-	resp, err := client.Do(req)
 	if err != nil {
-		return "", fmt.Errorf("failed to send request: %w", err)
+		return nil, fmt.Errorf("failed to send request: %w", err)
 	}
 	defer resp.Body.Close()
-
 	if resp.StatusCode != http.StatusOK {
-		// return an empty caption
+		return nil, fmt.Errorf("invalid response code: %s", resp.Status)
-		// probably 429 error
-		return "", nil
 	}
-	body, err := io.ReadAll(resp.Body)
+	var response GraphQLResponse
-	if err != nil {
+	decoder := sonic.ConfigFastest.NewDecoder(resp.Body)
-		return "", fmt.Errorf("failed to read response body: %w", err)
+	if err := decoder.Decode(&response); err != nil {
+		return nil, fmt.Errorf("failed to parse response: %w", err)
+	}
+	if response.Data == nil {
+		return nil, fmt.Errorf("data is nil")
+	}
+	if response.Status != "ok" {
+		return nil, fmt.Errorf("status is not ok: %s", response.Status)
+	}
+	if response.Data.ShortcodeMedia == nil {
+		return nil, fmt.Errorf("media is nil")
+	}
+	return response.Data, nil
 }
 
-	matches := captionPattern.FindStringSubmatch(string(body))
+func BuildGQLData() (map[string]string, map[string]string, error) {
-	if len(matches) < 2 {
+	const (
-		// post has no caption most likely
+		domain                = "www"
-		return "", nil
+		requestID             = "b"
+		clientCapabilityGrade = "EXCELLENT"
+		sessionInternalID     = "7436540909012459023"
+		apiVersion            = "1"
+		rolloutHash           = "1019933358"
+		appID                 = "936619743392459"
+		bloksVersionID        = "6309c8d03d8a3f47a1658ba38b304a3f837142ef5f637ebf1f8f52d4b802951e"
+		asbdID                = "129477"
+		hiddenState           = "20126.HYP:instagram_web_pkg.2.1...0"
+		loggedIn              = "0"
+		cometRequestID        = "7"
+		appVersion            = "0"
+		pixelRatio            = "2"
+		buildType             = "trunk"
+	)
+	session := "::" + util.RandomAlphaString(6)
+	sessionData := util.RandomBase64(8)
+	csrfToken := util.RandomBase64(32)
+	deviceID := util.RandomBase64(24)
+	machineID := util.RandomBase64(24)
+	dynamicFlags := util.RandomBase64(154)
+	clientSessionRnd := util.RandomBase64(154)
+	jazoestBig, err := rand.Int(rand.Reader, big.NewInt(10000))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to generate jazoest: %w", err)
 	}
-	return html.UnescapeString(matches[1]), nil
+	jazoest := strconv.FormatInt(jazoestBig.Int64()+1, 10)
+	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+	cookies := []string{
+		"csrftoken=" + csrfToken,
+		"ig_did=" + deviceID,
+		"wd=1280x720",
+		"dpr=2",
+		"mid=" + machineID,
+		"ig_nrcb=1",
+	}
+	headers := map[string]string{
+		"x-ig-app-id":        appID,
+		"X-FB-LSD":           sessionData,
+		"X-CSRFToken":        csrfToken,
+		"X-Bloks-Version-Id": bloksVersionID,
+		"x-asbd-id":          asbdID,
+		"cookie":             strings.Join(cookies, "; "),
+		"Content-Type":       "application/x-www-form-urlencoded",
+		"X-FB-Friendly-Name": polarisAction,
+	}
+	body := map[string]string{
+		"__d":         domain,
+		"__a":         apiVersion,
+		"__s":         session,
+		"__hs":        hiddenState,
+		"__req":       requestID,
+		"__ccg":       clientCapabilityGrade,
+		"__rev":       rolloutHash,
+		"__hsi":       sessionInternalID,
+		"__dyn":       dynamicFlags,
+		"__csr":       clientSessionRnd,
+		"__user":      loggedIn,
+		"__comet_req": cometRequestID,
+		"av":          appVersion,
+		"dpr":         pixelRatio,
+		"lsd":         sessionData,
+		"jazoest":     jazoest,
+		"__spin_r":    rolloutHash,
+		"__spin_b":    buildType,
+		"__spin_t":    timestamp,
+	}
+	return headers, body, nil
 }

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/strukturag/libheif v1.19.7
 	github.com/u2takey/ffmpeg-go v0.5.0
 	golang.org/x/image v0.26.0
+	gopkg.in/yaml.v3 v3.0.1
 	gorm.io/gorm v1.25.12
 )
 
@@ -26,7 +27,6 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
@@ -38,6 +38,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/titanous/json5 v1.0.0
 	github.com/u2takey/go-utils v0.3.1 // indirect
 	golang.org/x/text v0.24.0 // indirect
 	gorm.io/driver/mysql v1.5.7

go.sum

@@ -51,6 +51,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
 github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -61,6 +63,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/robertkrimen/otto v0.2.1 h1:FVP0PJ0AHIjC+N4pKCG9yCDz6LHNPCwi/GKID5pGGF0=
+github.com/robertkrimen/otto v0.2.1/go.mod h1:UPwtJ1Xu7JrLcZjNWN8orJaM5n5YEtqL//farB5FlRY=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -76,6 +80,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/strukturag/libheif v1.19.7 h1:XMfSJvmnucTbiS6CSxxZmpx5XSPjdqkpA3wiL6+I2Iw=
 github.com/strukturag/libheif v1.19.7/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks=
+github.com/titanous/json5 v1.0.0 h1:hJf8Su1d9NuI/ffpxgxQfxh/UiBFZX7bMPid0rIL/7s=
+github.com/titanous/json5 v1.0.0/go.mod h1:7JH1M8/LHKc6cyP5o5g3CSaRj+mBrIimTxzpvmckH8c=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU=
@@ -104,7 +110,10 @@ golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
+gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

util/http.go

@@ -2,7 +2,6 @@ package util
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"govd/config"
 	"govd/models"
@@ -14,6 +13,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+
+	"github.com/bytedance/sonic"
 )
 
 var (
@@ -235,7 +236,7 @@ func parseProxyResponse(proxyResp *http.Response, originalReq *http.Request) (*h
 	}
 
 	var response models.ProxyResponse
-	if err := json.Unmarshal(body, &response); err != nil {
+	if err := sonic.ConfigFastest.Unmarshal(body, &response); err != nil {
 		return nil, fmt.Errorf("error parsing proxy response: %w", err)
 	}
 

util/misc.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"crypto/rand"
 	"fmt"
 	"govd/models"
 	"net/http"
@@ -95,6 +96,44 @@ func GetLastError(err error) error {
 	return lastErr
 }
 
+func RandomBase64(length int) string {
+	const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+	const mask = 63 // 6 bits, since len(letters) == 64
+
+	result := make([]byte, length)
+	random := make([]byte, length)
+	_, err := rand.Read(random)
+	if err != nil {
+		return strings.Repeat("A", length)
+	}
+	for i, b := range random {
+		result[i] = letters[int(b)&mask]
+	}
+	return string(result)
+}
+
+func RandomAlphaString(length int) string {
+	const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+	const lettersLen = byte(len(letters))
+	const maxByte = 255 - (255 % lettersLen) // 255 - (255 % 52) = 255 - 47 = 208
+
+	result := make([]byte, length)
+	i := 0
+	for i < length {
+		b := make([]byte, 1)
+		_, err := rand.Read(b)
+		if err != nil {
+			return strings.Repeat("a", length)
+		}
+		if b[0] > maxByte {
+			continue // avoid bias
+		}
+		result[i] = letters[b[0]%lettersLen]
+		i++
+	}
+	return string(result)
+}
+
 func ParseCookieFile(fileName string) ([]*http.Cookie, error) {
 	cachedCookies, ok := cookiesCache[fileName]
 	if ok {