Merge branch 'main' of https://github.com/govdbot/govd

ext/twitter/main.go

@@ -15,9 +15,9 @@ import (
 )
 
 const (
-	apiHostname   = "x.com"
+	apiHostname = "x.com"
-	apiEndpoint   = "https://x.com/i/api/graphql/zZXycP0V6H7m-2r0mOnFcA/TweetDetail"
+	apiBase     = "https://" + apiHostname + "/i/api/graphql/"
-	transactionID = "H/HJB3naILIqzncBBvY50XFL36IYeol67HU4ZlUe8wYvWdn9q7KJf7k2UBKOMwliRmCnohzCodsUCuvWOl9t0Z/wVY3QHA"
+	apiEndpoint = apiBase + "2ICDjqPd81tulZcYrtpTuQ/TweetResultByRestId"
 )
 
 var ShortExtractor = &models.Extractor{
@@ -180,11 +180,17 @@ func GetTweetAPI(
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse response: %w", err)
 	}
-
+	result := apiResponse.Data.TweetResult.Result
-	tweet, err := FindTweetData(&apiResponse, tweetID)
+	if result == nil {
-	if err != nil {
+		return nil, errors.New("failed to get tweet result")
-		return nil, fmt.Errorf("failed to get tweet data: %w", err)
+	}
+	var tweet *Tweet
+	if result.Tweet != nil {
+		tweet = result.Tweet
+	} else if result.Legacy != nil {
+		tweet = result.Legacy
+	} else {
+		return nil, errors.New("failed to get tweet data")
 	}
-
 	return tweet, nil
 }

ext/twitter/models.go

@@ -2,49 +2,67 @@ package twitter
 
 type APIResponse struct {
 	Data struct {
-		ThreadedConversationWithInjectionsV2 struct {
+		TweetResult struct {
-			Instructions []struct {
+			Result *TweetResult `json:"result,omitempty"`
-				Entries []struct {
+		} `json:"tweetResult"`
-					EntryID string `json:"entryId"`
-					Content struct {
-						ItemContent struct {
-							TweetResults struct {
-								Result TweetResult `json:"result"`
-							} `json:"tweet_results"`
-						} `json:"itemContent"`
-					} `json:"content"`
-				} `json:"entries"`
-			} `json:"instructions"`
-		} `json:"threaded_conversation_with_injections_v2"`
 	} `json:"data"`
 }
 
 type TweetResult struct {
-	Tweet  *Tweet `json:"tweet,omitempty"`
+	Tweet       *Tweet     `json:"tweet,omitempty"`
-	Legacy *Tweet `json:"legacy,omitempty"`
+	Legacy      *Tweet     `json:"legacy,omitempty"`
-	RestID string `json:"rest_id,omitempty"`
+	RestID      string     `json:"rest_id,omitempty"`
-	Core   *Core  `json:"core,omitempty"`
+	Core        *Core      `json:"core,omitempty"`
+	Views       *ViewsInfo `json:"views,omitempty"`
+	Source      string     `json:"source,omitempty"`
+	EditControl *EditInfo  `json:"edit_control,omitempty"`
+	TypeName    string     `json:"__typename,omitempty"`
+}
+
+type EditInfo struct {
+	EditTweetIDs   []string `json:"edit_tweet_ids,omitempty"`
+	EditableUntil  string   `json:"editable_until_msecs,omitempty"`
+	IsEditEligible bool     `json:"is_edit_eligible,omitempty"`
+	EditsRemaining string   `json:"edits_remaining,omitempty"`
+}
+
+type ViewsInfo struct {
+	Count string `json:"count,omitempty"`
+	State string `json:"state,omitempty"`
 }
 
 type Core struct {
 	UserResults struct {
 		Result struct {
-			Legacy *UserLegacy `json:"legacy,omitempty"`
+			TypeName string      `json:"__typename,omitempty"`
+			RestID   string      `json:"rest_id,omitempty"`
+			Legacy   *UserLegacy `json:"legacy,omitempty"`
 		} `json:"result"`
 	} `json:"user_results"`
 }
 
 type UserLegacy struct {
-	ScreenName string `json:"screen_name"`
+	ScreenName           string `json:"screen_name"`
-	Name       string `json:"name"`
+	Name                 string `json:"name"`
+	ProfileImageURLHTTPS string `json:"profile_image_url_https,omitempty"`
+	CreatedAt            string `json:"created_at,omitempty"`
 }
 
 type Tweet struct {
-	FullText         string            `json:"full_text"`
+	FullText          string            `json:"full_text"`
-	ExtendedEntities *ExtendedEntities `json:"extended_entities,omitempty"`
+	ExtendedEntities  *ExtendedEntities `json:"extended_entities,omitempty"`
-	Entities         *ExtendedEntities `json:"entities,omitempty"`
+	Entities          *ExtendedEntities `json:"entities,omitempty"`
-	CreatedAt        string            `json:"created_at"`
+	CreatedAt         string            `json:"created_at"`
-	ID               string            `json:"id_str"`
+	ID                string            `json:"id_str"`
+	BookmarkCount     int               `json:"bookmark_count,omitempty"`
+	FavoriteCount     int               `json:"favorite_count,omitempty"`
+	ReplyCount        int               `json:"reply_count,omitempty"`
+	RetweetCount      int               `json:"retweet_count,omitempty"`
+	QuoteCount        int               `json:"quote_count,omitempty"`
+	PossiblySensitive bool              `json:"possibly_sensitive,omitempty"`
+	ConversationID    string            `json:"conversation_id_str,omitempty"`
+	Lang              string            `json:"lang,omitempty"`
+	UserIDStr         string            `json:"user_id_str,omitempty"`
 }
 
 type ExtendedEntities struct {
@@ -52,11 +70,39 @@ type ExtendedEntities struct {
 }
 
 type MediaEntity struct {
-	Type          string     `json:"type"`
+	Type              string             `json:"type"`
-	MediaURLHTTPS string     `json:"media_url_https"`
+	MediaURLHTTPS     string             `json:"media_url_https"`
-	ExpandedURL   string     `json:"expanded_url"`
+	ExpandedURL       string             `json:"expanded_url"`
-	URL           string     `json:"url"`
+	URL               string             `json:"url"`
-	VideoInfo     *VideoInfo `json:"video_info,omitempty"`
+	DisplayURL        string             `json:"display_url,omitempty"`
+	IDStr             string             `json:"id_str,omitempty"`
+	MediaKey          string             `json:"media_key,omitempty"`
+	VideoInfo         *VideoInfo         `json:"video_info,omitempty"`
+	Sizes             *MediaSizes        `json:"sizes,omitempty"`
+	OriginalInfo      *OriginalInfo      `json:"original_info,omitempty"`
+	MediaAvailability *MediaAvailability `json:"ext_media_availability,omitempty"`
+}
+
+type MediaAvailability struct {
+	Status string `json:"status"`
+}
+
+type MediaSizes struct {
+	Large  SizeInfo `json:"large,omitempty"`
+	Medium SizeInfo `json:"medium,omitempty"`
+	Small  SizeInfo `json:"small,omitempty"`
+	Thumb  SizeInfo `json:"thumb,omitempty"`
+}
+
+type SizeInfo struct {
+	H      int    `json:"h"`
+	W      int    `json:"w"`
+	Resize string `json:"resize,omitempty"`
+}
+
+type OriginalInfo struct {
+	Height int `json:"height"`
+	Width  int `json:"width"`
 }
 
 type VideoInfo struct {

ext/twitter/util.go

@@ -12,7 +12,6 @@ import (
 	"govd/util"
 
 	"github.com/bytedance/sonic"
-	"github.com/pkg/errors"
 )
 
 const authToken = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
@@ -33,7 +32,6 @@ func BuildAPIHeaders(cookies []*http.Cookie) map[string]string {
 	headers := map[string]string{
 		"authorization":             "Bearer " + authToken,
 		"user-agent":                util.ChromeUA,
-		"x-client-transaction-id":   transactionID,
 		"x-twitter-auth-type":       "OAuth2Session",
 		"x-twitter-client-language": "en",
 		"x-twitter-active-user":     "yes",
@@ -47,44 +45,47 @@ func BuildAPIHeaders(cookies []*http.Cookie) map[string]string {
 }
 
 func BuildAPIQuery(tweetID string) map[string]string {
-	variables := map[string]interface{}{
+	variables := map[string]any{
-		"focalTweetId":                           tweetID,
+		"tweetId":                tweetID,
-		"includePromotedContent":                 true,
+		"withCommunity":          false,
-		"with_rux_injections":                    false,
+		"includePromotedContent": false,
-		"withBirdwatchNotes":                     true,
+		"withVoice":              false,
-		"withCommunity":                          true,
-		"withDownvotePerspective":                false,
-		"withQuickPromoteEligibilityTweetFields": true,
-		"withReactionsMetadata":                  false,
-		"withReactionsPerspective":               false,
-		"withSuperFollowsTweetFields":            true,
-		"withSuperFollowsUserFields":             true,
-		"withV2Timeline":                         true,
-		"withVoice":                              true,
 	}
 
-	features := map[string]interface{}{
+	features := map[string]any{
-		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              false,
+		"creator_subscriptions_tweet_preview_api_enabled":                         true,
-		"interactive_text_enabled":                                                true,
-		"responsive_web_edit_tweet_api_enabled":                                   true,
-		"responsive_web_enhance_cards_enabled":                                    true,
-		"responsive_web_graphql_timeline_navigation_enabled":                      false,
-		"responsive_web_text_conversations_enabled":                               false,
-		"responsive_web_uc_gql_enabled":                                           true,
-		"standardized_nudges_misinfo":                                             true,
-		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
 		"tweetypie_unmention_optimization_enabled":                                true,
-		"unified_cards_ad_metadata_container_dynamic_card_content_query_enabled":  true,
+		"responsive_web_edit_tweet_api_enabled":                                   true,
+		"graphql_is_translatable_rweb_tweet_is_translatable_enabled":              true,
+		"view_counts_everywhere_api_enabled":                                      true,
+		"longform_notetweets_consumption_enabled":                                 true,
+		"responsive_web_twitter_article_tweet_consumption_enabled":                false,
+		"tweet_awards_web_tipping_enabled":                                        false,
+		"freedom_of_speech_not_reach_fetch_enabled":                               true,
+		"standardized_nudges_misinfo":                                             true,
+		"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
+		"longform_notetweets_rich_text_read_enabled":                              true,
+		"longform_notetweets_inline_media_enabled":                                true,
+		"responsive_web_graphql_exclude_directive_enabled":                        true,
 		"verified_phone_label_enabled":                                            false,
-		"vibe_api_enabled":                                                        true,
+		"responsive_web_media_download_video_enabled":                             false,
+		"responsive_web_graphql_skip_user_profile_image_extensions_enabled":       false,
+		"responsive_web_graphql_timeline_navigation_enabled":                      true,
+		"responsive_web_enhance_cards_enabled":                                    false,
+	}
+
+	fieldToggles := map[string]any{
+		"withArticleRichContentState": false,
 	}
 
 	variablesJSON, _ := sonic.ConfigFastest.Marshal(variables)
 	featuresJSON, _ := sonic.ConfigFastest.Marshal(features)
+	fieldTogglesJSON, _ := sonic.ConfigFastest.Marshal(fieldToggles)
 
 	return map[string]string{
-		"variables": string(variablesJSON),
+		"variables":    string(variablesJSON),
-		"features":  string(featuresJSON),
+		"features":     string(featuresJSON),
+		"fieldToggles": string(fieldTogglesJSON),
 	}
 }
 
@@ -136,31 +137,3 @@ func extractResolution(url string) (int64, int64) {
 	}
 	return 0, 0
 }
-
-func FindTweetData(resp *APIResponse, tweetID string) (*Tweet, error) {
-	instructions := resp.Data.ThreadedConversationWithInjectionsV2.Instructions
-	if len(instructions) == 0 {
-		return nil, errors.New("tweet data missing")
-	}
-
-	entries := instructions[0].Entries
-	entryID := "tweet-" + tweetID
-
-	for _, entry := range entries {
-		if entry.EntryID == entryID {
-			result := entry.Content.ItemContent.TweetResults.Result
-
-			if result.Tweet != nil {
-				return result.Tweet, nil
-			}
-
-			if result.Legacy != nil {
-				return result.Legacy, nil
-			}
-
-			return nil, errors.New("invalid tweet data")
-		}
-	}
-
-	return nil, errors.New("tweet not found")
-}