instagram: new extraction method
new extraction method first tries to fetch content directly from instagram graphql API, fallback to html embed page. in case every method fail, rely on 3rd party
This commit is contained in:
parent
1b3c426808
commit
93e964a28b
10 changed files with 494 additions and 110 deletions
39
util/misc.go
39
util/misc.go
|
@ -1,6 +1,7 @@
|
|||
package util
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"govd/models"
|
||||
"net/http"
|
||||
|
@ -95,6 +96,44 @@ func GetLastError(err error) error {
|
|||
return lastErr
|
||||
}
|
||||
|
||||
func RandomBase64(length int) string {
|
||||
const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
|
||||
const mask = 63 // 6 bits, since len(letters) == 64
|
||||
|
||||
result := make([]byte, length)
|
||||
random := make([]byte, length)
|
||||
_, err := rand.Read(random)
|
||||
if err != nil {
|
||||
return strings.Repeat("A", length)
|
||||
}
|
||||
for i, b := range random {
|
||||
result[i] = letters[int(b)&mask]
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func RandomAlphaString(length int) string {
|
||||
const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
||||
const lettersLen = byte(len(letters))
|
||||
const maxByte = 255 - (255 % lettersLen) // 255 - (255 % 52) = 255 - 47 = 208
|
||||
|
||||
result := make([]byte, length)
|
||||
i := 0
|
||||
for i < length {
|
||||
b := make([]byte, 1)
|
||||
_, err := rand.Read(b)
|
||||
if err != nil {
|
||||
return strings.Repeat("a", length)
|
||||
}
|
||||
if b[0] > maxByte {
|
||||
continue // avoid bias
|
||||
}
|
||||
result[i] = letters[b[0]%lettersLen]
|
||||
i++
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func ParseCookieFile(fileName string) ([]*http.Cookie, error) {
|
||||
cachedCookies, ok := cookiesCache[fileName]
|
||||
if ok {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue