Init
This commit is contained in:
stefanodvx
parent
264c97183e
commit
3faede7b1c
74 changed files with 6228 additions and 1 deletions
184
ext/tiktok/main.go
Normal file
184
ext/tiktok/main.go
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
package tiktok
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/quic-go/quic-go"
|
||||
"github.com/quic-go/quic-go/http3"
|
||||
|
||||
"govd/enums"
|
||||
"govd/models"
|
||||
"govd/util"
|
||||
)
|
||||
|
||||
const (
|
||||
apiHostname = "api16-normal-c-useast1a.tiktokv.com"
|
||||
installationID = "7127307272354596614"
|
||||
appName = "musical_ly"
|
||||
appID = "1233"
|
||||
appVersion = "37.1.4"
|
||||
manifestAppVersion = "2023508030"
|
||||
packageID = "com.zhiliaoapp.musically/" + manifestAppVersion
|
||||
appUserAgent = packageID + " (Linux; U; Android 13; en_US; Pixel 7; Build/TD1A.220804.031; Cronet/58.0.2991.0)"
|
||||
)
|
||||
|
||||
var HTTPClient = &http.Client{
|
||||
Transport: &http3.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
QUICConfig: &quic.Config{
|
||||
MaxIncomingStreams: -1,
|
||||
EnableDatagrams: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var VMExtractor = &models.Extractor{
|
||||
Name: "TikTok VM",
|
||||
CodeName: "tiktokvm",
|
||||
Type: enums.ExtractorTypeSingle,
|
||||
Category: enums.ExtractorCategorySocial,
|
||||
URLPattern: regexp.MustCompile(`https:\/\/((?:vm|vt|www)\.)?(vx)?tiktok\.com\/(?:t\/)?(?P<id>[a-zA-Z0-9]+)`),
|
||||
IsRedirect: true,
|
||||
|
||||
Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
|
||||
location, err := util.GetLocationURL(ctx.MatchedContentURL, "")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get url location: %w", err)
|
||||
}
|
||||
return &models.ExtractorResponse{
|
||||
URL: location,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
var Extractor = &models.Extractor{
|
||||
Name: "TikTok",
|
||||
CodeName: "tiktok",
|
||||
Type: enums.ExtractorTypeSingle,
|
||||
Category: enums.ExtractorCategorySocial,
|
||||
URLPattern: regexp.MustCompile(`https?:\/\/((www|m)\.)?(vx)?tiktok\.com\/((?:embed|@[\w\.-]+)\/)?(v(ideo)?|p(hoto)?)\/(?P<id>[0-9]+)`),
|
||||
|
||||
Run: func(ctx *models.DownloadContext) (*models.ExtractorResponse, error) {
|
||||
mediaList, err := MediaListFromAPI(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get media: %w", err)
|
||||
}
|
||||
return &models.ExtractorResponse{
|
||||
MediaList: mediaList,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
func MediaListFromAPI(ctx *models.DownloadContext) ([]*models.Media, error) {
|
||||
var mediaList []*models.Media
|
||||
|
||||
details, err := GetVideoAPI(ctx.MatchedContentID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get from api: %w", err)
|
||||
}
|
||||
caption := details.Desc
|
||||
isImageSlide := details.ImagePostInfo != nil
|
||||
if !isImageSlide {
|
||||
media := ctx.Extractor.NewMedia(
|
||||
ctx.MatchedContentID,
|
||||
ctx.MatchedContentURL,
|
||||
)
|
||||
media.SetCaption(caption)
|
||||
video := details.Video
|
||||
|
||||
// generic PlayAddr
|
||||
if video.PlayAddr != nil {
|
||||
format, err := ParsePlayAddr(video, video.PlayAddr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse playaddr: %w", err)
|
||||
}
|
||||
media.AddFormat(format)
|
||||
}
|
||||
// hevc PlayAddr
|
||||
if video.PlayAddrBytevc1 != nil {
|
||||
format, err := ParsePlayAddr(video, video.PlayAddrBytevc1)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse playaddr: %w", err)
|
||||
}
|
||||
media.AddFormat(format)
|
||||
}
|
||||
// h264 PlayAddr
|
||||
if video.PlayAddrH264 != nil {
|
||||
format, err := ParsePlayAddr(video, video.PlayAddrH264)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse playaddr: %w", err)
|
||||
}
|
||||
media.AddFormat(format)
|
||||
}
|
||||
mediaList = append(mediaList, media)
|
||||
} else {
|
||||
images := details.ImagePostInfo.Images
|
||||
for _, image := range images {
|
||||
media := ctx.Extractor.NewMedia(
|
||||
ctx.MatchedContentID,
|
||||
ctx.MatchedContentURL,
|
||||
)
|
||||
media.SetCaption(caption)
|
||||
media.AddFormat(&models.MediaFormat{
|
||||
FormatID: "image",
|
||||
Type: enums.MediaTypePhoto,
|
||||
URL: image.DisplayImage.URLList,
|
||||
})
|
||||
mediaList = append(mediaList, media)
|
||||
}
|
||||
}
|
||||
return mediaList, nil
|
||||
}
|
||||
|
||||
func GetVideoAPI(awemeID string) (*AwemeDetails, error) {
|
||||
apiURL := fmt.Sprintf(
|
||||
"https://%s/aweme/v1/multi/aweme/detail/",
|
||||
apiHostname,
|
||||
)
|
||||
queryParams, err := BuildAPIQuery()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to build api query: %w", err)
|
||||
}
|
||||
postData := BuildPostData(awemeID)
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
apiURL,
|
||||
postData,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
req.URL.RawQuery = queryParams.Encode()
|
||||
req.Header.Set("User-Agent", appUserAgent)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("X-Argus", "")
|
||||
|
||||
resp, err := HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body: %w", err)
|
||||
}
|
||||
|
||||
var data *Response
|
||||
err = json.Unmarshal(body, &data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
videoData, err := FindVideoData(data, awemeID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find video data: %w", err)
|
||||
}
|
||||
return videoData, nil
|
||||
}
|
||||
65
ext/tiktok/models.go
Normal file
65
ext/tiktok/models.go
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
package tiktok
|
||||
|
||||
type Response struct {
|
||||
AwemeDetails []AwemeDetails `json:"aweme_details"`
|
||||
StatusCode int `json:"status_code"`
|
||||
StatusMsg string `json:"status_msg"`
|
||||
}
|
||||
|
||||
type Cover struct {
|
||||
Height int64 `json:"height"`
|
||||
URI string `json:"uri"`
|
||||
URLList []string `json:"url_list"`
|
||||
URLPrefix any `json:"url_prefix"`
|
||||
Width int64 `json:"width"`
|
||||
}
|
||||
|
||||
type PlayAddr struct {
|
||||
DataSize int64 `json:"data_size"`
|
||||
FileCs string `json:"file_cs"`
|
||||
FileHash string `json:"file_hash"`
|
||||
Height int64 `json:"height"`
|
||||
URI string `json:"uri"`
|
||||
URLKey string `json:"url_key"`
|
||||
URLList []string `json:"url_list"`
|
||||
Width int64 `json:"width"`
|
||||
}
|
||||
|
||||
type Image struct {
|
||||
DisplayImage *DisplayImage `json:"display_image"`
|
||||
}
|
||||
|
||||
type DisplayImage struct {
|
||||
Height int `json:"height"`
|
||||
URI string `json:"uri"`
|
||||
URLList []string `json:"url_list"`
|
||||
URLPrefix any `json:"url_prefix"`
|
||||
Width int `json:"width"`
|
||||
}
|
||||
|
||||
type ImagePostInfo struct {
|
||||
Images []Image `json:"images"`
|
||||
MusicVolume float64 `json:"music_volume"`
|
||||
PostExtra string `json:"post_extra"`
|
||||
Title string `json:"title"`
|
||||
}
|
||||
|
||||
type Video struct {
|
||||
CdnURLExpired int64 `json:"cdn_url_expired"`
|
||||
Cover Cover `json:"cover"`
|
||||
Duration int64 `json:"duration"`
|
||||
HasWatermark bool `json:"has_watermark"`
|
||||
Height int64 `json:"height"`
|
||||
PlayAddr *PlayAddr `json:"play_addr"`
|
||||
PlayAddrBytevc1 *PlayAddr `json:"play_addr_bytevc1"`
|
||||
PlayAddrH264 *PlayAddr `json:"play_addr_h264"`
|
||||
Width int64 `json:"width"`
|
||||
}
|
||||
|
||||
type AwemeDetails struct {
|
||||
AwemeID string `json:"aweme_id"`
|
||||
AwemeType int `json:"aweme_type"`
|
||||
Desc string `json:"desc"`
|
||||
Video *Video `json:"video"`
|
||||
ImagePostInfo *ImagePostInfo `json:"image_post_info"`
|
||||
}
|
||||
177
ext/tiktok/util.go
Normal file
177
ext/tiktok/util.go
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
package tiktok
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"govd/enums"
|
||||
"govd/models"
|
||||
"govd/util"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func BuildAPIQuery() (url.Values, error) {
|
||||
requestTicket := strconv.Itoa(int(time.Now().Unix()) * 1000)
|
||||
clientDeviceID := uuid.New().String()
|
||||
versionCode, err := GetAppVersionCode(appVersion)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get app version code: %w", err)
|
||||
}
|
||||
return url.Values{
|
||||
"device_platform": []string{"android"},
|
||||
"os": []string{"android"},
|
||||
"ssmix": []string{"0"}, // what is this?
|
||||
"_rticket": []string{requestTicket},
|
||||
"cdid": []string{clientDeviceID},
|
||||
"channel": []string{"googleplay"},
|
||||
"aid": []string{appID},
|
||||
"app_name": []string{appName},
|
||||
"version_code": []string{versionCode},
|
||||
"version_name": []string{appVersion},
|
||||
"manifest_version_code": []string{manifestAppVersion},
|
||||
"update_version_code": []string{manifestAppVersion},
|
||||
"ab_version": []string{appVersion},
|
||||
"resolution": []string{"1080*2400"},
|
||||
"dpi": []string{"420"},
|
||||
"device_type": []string{"Pixel 7"},
|
||||
"device_brand": []string{"Google"},
|
||||
"language": []string{"en"},
|
||||
"os_api": []string{"29"},
|
||||
"os_version": []string{"13"},
|
||||
"ac": []string{"wifi"},
|
||||
"is_pad": []string{"0"},
|
||||
"current_region": []string{"US"},
|
||||
"app_type": []string{"normal"},
|
||||
"last_install_time": []string{GetRandomInstallTime()},
|
||||
"timezone_name": []string{"America/New_York"},
|
||||
"residence": []string{"US"},
|
||||
"app_language": []string{"en"},
|
||||
"timezone_offset": []string{"-14400"},
|
||||
"host_abi": []string{"armeabi-v7a"},
|
||||
"locale": []string{"en"},
|
||||
"ac2": []string{"wifi5g"},
|
||||
"uoo": []string{"1"}, // what is this?
|
||||
"carrier_region": []string{"US"},
|
||||
"build_number": []string{appVersion},
|
||||
"region": []string{"US"},
|
||||
"ts": []string{strconv.Itoa(int(time.Now().Unix()))},
|
||||
"iid": []string{installationID},
|
||||
"device_id": []string{GetRandomDeviceID()},
|
||||
"openudid": []string{GetRandomUdid()},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ParsePlayAddr(
|
||||
video *Video,
|
||||
playAddr *PlayAddr,
|
||||
) (*models.MediaFormat, error) {
|
||||
formatID := playAddr.URLKey
|
||||
if formatID == "" {
|
||||
return nil, errors.New("url_key not found")
|
||||
}
|
||||
videoCodec := enums.MediaCodecHEVC
|
||||
if strings.Contains(formatID, "h264") {
|
||||
videoCodec = enums.MediaCodecAVC
|
||||
}
|
||||
videoURL := playAddr.URLList
|
||||
videoDuration := video.Duration / 1000
|
||||
videoWidth := playAddr.Width
|
||||
videoHeight := playAddr.Height
|
||||
videoCover := &video.Cover
|
||||
videoThumbnailURLs := videoCover.URLList
|
||||
|
||||
return &models.MediaFormat{
|
||||
Type: enums.MediaTypeVideo,
|
||||
FormatID: formatID,
|
||||
URL: videoURL,
|
||||
VideoCodec: videoCodec,
|
||||
AudioCodec: enums.MediaCodecAAC,
|
||||
Duration: videoDuration,
|
||||
Thumbnail: videoThumbnailURLs,
|
||||
Width: videoWidth,
|
||||
Height: videoHeight,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func GetRandomInstallTime() string {
|
||||
currentTime := int(time.Now().Unix())
|
||||
minOffset := big.NewInt(86400)
|
||||
maxOffset := big.NewInt(1123200)
|
||||
diff := new(big.Int).Sub(maxOffset, minOffset)
|
||||
randomOffset, _ := rand.Int(rand.Reader, diff)
|
||||
randomOffset.Add(randomOffset, minOffset)
|
||||
result := currentTime - int(randomOffset.Int64())
|
||||
return strconv.Itoa(result)
|
||||
}
|
||||
|
||||
func GetRandomUdid() string {
|
||||
const charset = "0123456789abcdef"
|
||||
result := make([]byte, 16)
|
||||
|
||||
for i := range result {
|
||||
index, _ := rand.Int(rand.Reader, big.NewInt(int64(len(charset))))
|
||||
result[i] = charset[index.Int64()]
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func GetRandomDeviceID() string {
|
||||
minNum := big.NewInt(7250000000000000000)
|
||||
maxNum := big.NewInt(7351147085025500000)
|
||||
diff := new(big.Int).Sub(maxNum, minNum)
|
||||
randNum, _ := rand.Int(rand.Reader, diff)
|
||||
result := new(big.Int).Add(randNum, minNum)
|
||||
return result.String()
|
||||
}
|
||||
|
||||
func BuildPostData(awemeID string) *strings.Reader {
|
||||
data := url.Values{
|
||||
"aweme_ids": []string{fmt.Sprintf("[%s]", awemeID)},
|
||||
"request_source": []string{"0"},
|
||||
}
|
||||
return strings.NewReader(data.Encode())
|
||||
|
||||
}
|
||||
|
||||
func GetAppVersionCode(version string) (string, error) {
|
||||
parts := strings.Split(version, ".")
|
||||
|
||||
var result strings.Builder
|
||||
for _, part := range parts {
|
||||
num, err := strconv.Atoi(part)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse version part: %w", err)
|
||||
}
|
||||
_, err = fmt.Fprintf(&result, "%02d", num)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to format version part: %w", err)
|
||||
}
|
||||
}
|
||||
return result.String(), nil
|
||||
}
|
||||
|
||||
func FindVideoData(
|
||||
resp *Response,
|
||||
expectedAwemeID string,
|
||||
) (*AwemeDetails, error) {
|
||||
if resp.StatusCode == 2053 {
|
||||
return nil, util.ErrUnavailable
|
||||
}
|
||||
if resp.AwemeDetails == nil {
|
||||
return nil, errors.New("aweme_details is nil")
|
||||
}
|
||||
for _, item := range resp.AwemeDetails {
|
||||
if item.AwemeID == expectedAwemeID {
|
||||
return &item, nil
|
||||
}
|
||||
}
|
||||
return nil, errors.New("matching aweme_id not found")
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue