From e35bd2fe5715656628471f4c72a03a2b396ca5d8 Mon Sep 17 00:00:00 2001 From: dogghi no <75123663+doggyhaha@users.noreply.github.com> Date: Thu, 17 Apr 2025 14:16:30 +0200 Subject: [PATCH] implement sonic for json + remove some TODOs from readme + fix go version requirement (#5) --- README.md | 4 +--- ext/instagram/util.go | 22 ++++++++++++++-------- ext/pinterest/main.go | 12 ++++-------- ext/pinterest/util.go | 5 +++-- ext/reddit/main.go | 12 ++++-------- ext/tiktok/main.go | 11 ++++------- ext/twitter/main.go | 11 ++++------- ext/twitter/util.go | 7 ++++--- go.mod | 10 +++++++++- go.sum | 23 +++++++++++++++++++++++ 10 files changed, 70 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 9d0a2ac..ce86e22 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ this project was born after the discontinuation of a highly popular bot known as - ffmpeg >= 6.1.1 - libheif >= 1.19.7 - pkg-config -- mysql db +- mysql or mariadb ## installation @@ -84,10 +84,8 @@ some extractors require cookies for download. to add your cookies, just insert a ## todo - [ ] add more extractors -- [ ] switch to sonic json parser - [ ] switch to native libav - [ ] add tests -- [ ] add dockerfile and compose - [ ] improve error handling - [ ] add support for telegram wehbhooks - [ ] switch to pgsql (?) diff --git a/ext/instagram/util.go b/ext/instagram/util.go index bd6bb68..120c79b 100644 --- a/ext/instagram/util.go +++ b/ext/instagram/util.go @@ -3,7 +3,6 @@ package instagram import ( "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "govd/util" "html" @@ -14,6 +13,8 @@ import ( "strconv" "strings" "time" + + "github.com/bytedance/sonic" ) var captionPattern = regexp.MustCompile( @@ -40,7 +41,7 @@ func BuildSignedPayload(contentURL string) (io.Reader, error) { "_tsc": "0", // ? "_s": secretString, } - parsedPayload, err := json.Marshal(payload) + parsedPayload, err := sonic.ConfigFastest.Marshal(payload) if err != nil { return nil, fmt.Errorf("error marshalling payload: %w", err) } @@ -50,15 +51,20 @@ func BuildSignedPayload(contentURL string) (io.Reader, error) { func ParseIGramResponse(body []byte) (*IGramResponse, error) { var rawResponse interface{} - if err := json.Unmarshal(body, &rawResponse); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) + //move to the start of the body + // Use sonic's decoder to unmarshal the raw response + if err := sonic.ConfigFastest.Unmarshal(body, &rawResponse); err != nil { + return nil, fmt.Errorf("failed to decode response1: %w", err) } + + + switch rawResponse.(type) { case []interface{}: // array of IGramMedia var media []*IGramMedia - if err := json.Unmarshal(body, &media); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) + if err := sonic.ConfigFastest.Unmarshal(body, &media); err != nil { + return nil, fmt.Errorf("failed to decode response2: %w", err) } return &IGramResponse{ Items: media, @@ -66,8 +72,8 @@ func ParseIGramResponse(body []byte) (*IGramResponse, error) { case map[string]interface{}: // single IGramMedia var media IGramMedia - if err := json.Unmarshal(body, &media); err != nil { - return nil, fmt.Errorf("failed to decode response: %w", err) + if err := sonic.ConfigFastest.Unmarshal(body, &media); err != nil { + return nil, fmt.Errorf("failed to decode response3: %w", err) } return &IGramResponse{ Items: []*IGramMedia{&media}, diff --git a/ext/pinterest/main.go b/ext/pinterest/main.go index 1a263fd..d50dfe9 100644 --- a/ext/pinterest/main.go +++ b/ext/pinterest/main.go @@ -1,9 +1,7 @@ package pinterest import ( - "encoding/json" "fmt" - "io" "net/http" "regexp" "strings" @@ -11,6 +9,8 @@ import ( "govd/enums" "govd/models" "govd/util" + + "github.com/bytedance/sonic" ) const ( @@ -185,13 +185,9 @@ func GetPinData(pinID string) (*PinData, error) { return nil, fmt.Errorf("bad response: %s", resp.Status) } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var pinResponse PinResponse - err = json.Unmarshal(body, &pinResponse) + decoder := sonic.ConfigFastest.NewDecoder(resp.Body) + err = decoder.Decode(&pinResponse) if err != nil { return nil, fmt.Errorf("failed to parse response: %w", err) } diff --git a/ext/pinterest/util.go b/ext/pinterest/util.go index cff5aae..53bdf12 100644 --- a/ext/pinterest/util.go +++ b/ext/pinterest/util.go @@ -1,11 +1,12 @@ package pinterest import ( - "encoding/json" "fmt" "govd/enums" "govd/models" "govd/util/parser" + + "github.com/bytedance/sonic" ) func ParseVideoObject(videoObj *Videos) ([]*models.MediaFormat, error) { @@ -48,7 +49,7 @@ func BuildPinRequestParams(pinID string) map[string]string { }, } - jsonData, _ := json.Marshal(options) + jsonData, _ := sonic.ConfigFastest.Marshal(options) return map[string]string{ "data": string(jsonData), } diff --git a/ext/reddit/main.go b/ext/reddit/main.go index fb5cb38..1cf111f 100644 --- a/ext/reddit/main.go +++ b/ext/reddit/main.go @@ -1,15 +1,15 @@ package reddit import ( - "encoding/json" "fmt" - "io" "net/http" "regexp" "govd/enums" "govd/models" "govd/util" + + "github.com/bytedance/sonic" ) var ( @@ -255,13 +255,9 @@ func GetRedditData(host string, slug string) (RedditResponse, error) { return GetRedditData(altHost, slug) } - body, err := io.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - var response RedditResponse - err = json.Unmarshal(body, &response) + decoder := sonic.ConfigFastest.NewDecoder(res.Body) + err = decoder.Decode(&response) if err != nil { return nil, fmt.Errorf("failed to parse response: %w", err) } diff --git a/ext/tiktok/main.go b/ext/tiktok/main.go index 4126ab2..03049af 100644 --- a/ext/tiktok/main.go +++ b/ext/tiktok/main.go @@ -1,15 +1,15 @@ package tiktok import ( - "encoding/json" "fmt" - "io" "net/http" "regexp" "govd/enums" "govd/models" "govd/util" + + "github.com/bytedance/sonic" ) const ( @@ -166,13 +166,10 @@ func GetVideoAPI(awemeID string) (*AwemeDetails, error) { return nil, fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } var data *Response - err = json.Unmarshal(body, &data) + decoder := sonic.ConfigFastest.NewDecoder(resp.Body) + err = decoder.Decode(&data) if err != nil { return nil, fmt.Errorf("failed to unmarshal response: %w", err) } diff --git a/ext/twitter/main.go b/ext/twitter/main.go index 7d37c16..70c871b 100644 --- a/ext/twitter/main.go +++ b/ext/twitter/main.go @@ -1,7 +1,6 @@ package twitter import ( - "encoding/json" "fmt" "io" "net/http" @@ -10,6 +9,8 @@ import ( "govd/enums" "govd/models" "govd/util" + + "github.com/bytedance/sonic" ) const ( @@ -168,13 +169,9 @@ func GetTweetAPI(tweetID string) (*Tweet, error) { return nil, fmt.Errorf("invalid response code: %s", resp.Status) } - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read body: %w", err) - } - var apiResponse APIResponse - err = json.Unmarshal(body, &apiResponse) + decoder := sonic.ConfigFastest.NewDecoder(resp.Body) + err = decoder.Decode(&apiResponse) if err != nil { return nil, fmt.Errorf("failed to parse response: %w", err) } diff --git a/ext/twitter/util.go b/ext/twitter/util.go index e2afad0..c7b342b 100644 --- a/ext/twitter/util.go +++ b/ext/twitter/util.go @@ -1,7 +1,6 @@ package twitter import ( - "encoding/json" "fmt" "govd/enums" "govd/models" @@ -10,6 +9,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/bytedance/sonic" ) const authToken = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" @@ -75,8 +76,8 @@ func BuildAPIQuery(tweetID string) map[string]string { "vibe_api_enabled": true, } - variablesJSON, _ := json.Marshal(variables) - featuresJSON, _ := json.Marshal(features) + variablesJSON, _ := sonic.ConfigFastest.Marshal(variables) + featuresJSON, _ := sonic.ConfigFastest.Marshal(features) return map[string]string{ "variables": string(variablesJSON), diff --git a/go.mod b/go.mod index 9394834..184510c 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,12 @@ module govd -go 1.24.0 +go 1.23.0 + +toolchain go1.24.2 require ( github.com/PaulSonOfLars/gotgbot/v2 v2.0.0-rc.31 + github.com/bytedance/sonic v1.13.2 github.com/google/uuid v1.6.0 github.com/guregu/null/v6 v6.0.0 github.com/joho/godotenv v1.5.1 @@ -15,8 +18,13 @@ require ( ) require ( + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/stretchr/testify v1.10.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect ) require ( diff --git a/go.sum b/go.sum index 74164a5..598966b 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,14 @@ github.com/aki237/nscjar v0.0.0-20210417074043-bbb606196143/go.mod h1:l0r3UsMujH github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,6 +44,9 @@ github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwA github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -48,9 +59,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/strukturag/libheif v1.19.7 h1:XMfSJvmnucTbiS6CSxxZmpx5XSPjdqkpA3wiL6+I2Iw= @@ -62,11 +79,15 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -91,6 +112,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= @@ -98,4 +120,5 @@ gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkD gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=