remove simulate request methods
This commit is contained in:
105
flaresolverr.go
105
flaresolverr.go
@@ -4,11 +4,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Danny-Dasilva/CycleTLS/cycletls"
|
||||
)
|
||||
|
||||
var instances map[string]*Flaresolverr = make(map[string]*Flaresolverr)
|
||||
@@ -17,7 +14,6 @@ type Flaresolverr struct {
|
||||
url string
|
||||
v1Url string
|
||||
sessionID string
|
||||
cycletlsC *cycletlsClient
|
||||
proxy string
|
||||
}
|
||||
|
||||
@@ -131,104 +127,3 @@ func (f *Flaresolverr) ListSessionsV1(sessionID string) (*V1ResponseBase, error)
|
||||
}
|
||||
return f.requestV1(req)
|
||||
}
|
||||
|
||||
func (f *Flaresolverr) GetJa3AndUserAgent() (ua string, ja3 string, err error) {
|
||||
resp, err := f.GetV1("view-source:https://tools.scrapfly.io/api/tls", nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
j := regexp.MustCompile(`{(.*)}`).FindStringSubmatch(resp.Solution.RawResponse.(string))
|
||||
if len(j) < 2 {
|
||||
return "", "", fmt.Errorf("failed to get ja3")
|
||||
}
|
||||
ja3Obj := Ja3{}
|
||||
err = json.Unmarshal([]byte(j[0]), &ja3Obj)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return resp.Solution.UserAgent, ja3Obj.Ja3, nil
|
||||
}
|
||||
|
||||
func (f *Flaresolverr) preSimulateRequest(opts *SimulateOptions) error {
|
||||
if f.cycletlsC == nil {
|
||||
ua, ja3, err := f.GetJa3AndUserAgent()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.cycletlsC = NewCycletlsClient(ja3, ua)
|
||||
}
|
||||
if opts == nil {
|
||||
opts = &SimulateOptions{}
|
||||
}
|
||||
if opts.HttpCookies == nil {
|
||||
opts.HttpCookies = []*http.Cookie{}
|
||||
}
|
||||
opts.Cookies = ConvertToCycletlsCookies(opts.HttpCookies)
|
||||
if f.proxy != "" {
|
||||
opts.Proxy = f.proxy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Flaresolverr) SimulateGet(reqURL string, opts *SimulateOptions) (cycletls.Response, error) {
|
||||
if opts == nil {
|
||||
opts = &SimulateOptions{}
|
||||
}
|
||||
err := f.preSimulateRequest(opts)
|
||||
if err != nil {
|
||||
return cycletls.Response{}, err
|
||||
}
|
||||
return f.cycletlsC.Get(reqURL, &opts.Options)
|
||||
}
|
||||
|
||||
func (f *Flaresolverr) SimulatePost(reqURL string, opts *SimulateOptions) (cycletls.Response, error) {
|
||||
if opts == nil {
|
||||
opts = &SimulateOptions{}
|
||||
}
|
||||
err := f.preSimulateRequest(opts)
|
||||
if err != nil {
|
||||
return cycletls.Response{}, err
|
||||
}
|
||||
return f.cycletlsC.Post(reqURL, &opts.Options)
|
||||
}
|
||||
|
||||
type SimulateOptions struct {
|
||||
cycletls.Options
|
||||
HttpCookies []*http.Cookie
|
||||
}
|
||||
|
||||
func ConvertToCycletlsCookies(cookies []*http.Cookie) []cycletls.Cookie {
|
||||
var convertedCookies []cycletls.Cookie
|
||||
for _, cookie := range cookies {
|
||||
convertedCookies = append(convertedCookies, cycletls.Cookie{
|
||||
Name: cookie.Name,
|
||||
Value: cookie.Value,
|
||||
Path: cookie.Path,
|
||||
Domain: cookie.Domain,
|
||||
Expires: cookie.Expires,
|
||||
MaxAge: cookie.MaxAge,
|
||||
Secure: cookie.Secure,
|
||||
HTTPOnly: cookie.HttpOnly,
|
||||
SameSite: cookie.SameSite,
|
||||
})
|
||||
}
|
||||
return convertedCookies
|
||||
}
|
||||
|
||||
func ConvertToCookies(cookies []cycletls.Cookie) []*http.Cookie {
|
||||
var convertedCookies []*http.Cookie
|
||||
for _, cookie := range cookies {
|
||||
convertedCookies = append(convertedCookies, &http.Cookie{
|
||||
Name: cookie.Name,
|
||||
Value: cookie.Value,
|
||||
Path: cookie.Path,
|
||||
Domain: cookie.Domain,
|
||||
Expires: cookie.Expires,
|
||||
MaxAge: cookie.MaxAge,
|
||||
Secure: cookie.Secure,
|
||||
HttpOnly: cookie.HTTPOnly,
|
||||
SameSite: cookie.SameSite,
|
||||
})
|
||||
}
|
||||
return convertedCookies
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user