ccs/cf-clearance-scraper.go

package ccs

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/Danny-Dasilva/CycleTLS/cycletls"
	"github.com/go-resty/resty/v2"
)

type Request struct {
	Url     string `json:"url"`
	Mode    string `json:"mode"`
	SiteKey string `json:"siteKey"`
}

type Cookie struct {
	Name         string  `json:"name"`
	Value        string  `json:"value"`
	Domain       string  `json:"domain"`
	Path         string  `json:"path"`
	Expires      float64 `json:"expires"`
	Size         int     `json:"size"`
	HTTPOnly     bool    `json:"httpOnly"`
	Secure       bool    `json:"secure"`
	Session      bool    `json:"session"`
	SameSite     string  `json:"sameSite"`
	Priority     string  `json:"priority"`
	SameParty    bool    `json:"sameParty"`
	SourceScheme string  `json:"sourceScheme"`
	PartitionKey string  `json:"partitionKey"`
}

type Session struct {
	Cookies []Cookie          `json:"cookies"`
	Headers map[string]string `json:"headers"`
	Code    int               `json:"code"`
}

var (
	httpClient     *resty.Client
	cycletlsClient cycletls.CycleTLS
)

func init() {
	httpClient = resty.New()
	httpClient.SetRetryCount(3).SetTimeout(10 * time.Second).SetRetryWaitTime(3 * time.Second)
	cycletlsClient = cycletls.Init()
}

func WAFSession(ccsUrl string, requestUrl string) (Session, error) {
	data := Request{
		Url:  requestUrl,
		Mode: "waf-session",
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return Session{}, err
	}
	var response Session
	err = json.Unmarshal(resp.Body(), &response)
	if err != nil {
		return Session{}, err
	}
	if response.Code != 200 {
		return Session{}, errors.New("Failed to get WAF session")
	}
	return response, nil
}

func Source(ccsUrl string, requestUrl string) ([]byte, error) {
	data := Request{
		Url:  requestUrl,
		Mode: "source",
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return nil, err
	}
	type response struct {
		Source string `json:"source"`
		Code   int    `json:"code"`
	}
	var ccsResp response
	err = json.Unmarshal(resp.Body(), &ccsResp)
	if err != nil {
		return nil, err
	}
	if ccsResp.Code != 200 {
		return nil, errors.New("Failed to get source")
	}
	return []byte(ccsResp.Source), nil
}

func TurnstileToken(ccsUrl string, requestUrl string, siteKey string) (string, error) {
	data := Request{
		Url:     requestUrl,
		Mode:    "turnstile-min",
		SiteKey: siteKey,
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return "", err
	}
	var ccsResp struct {
		Token string `json:"token"`
		Code  int    `json:"code"`
	}
	err = json.Unmarshal(resp.Body(), &ccsResp)
	if err != nil {
		return "", err
	}
	if ccsResp.Code != 200 {
		return "", errors.New("Failed to get source")
	}
	return ccsResp.Token, nil
}

func TurnstileMaxToken(ccsUrl string, requestUrl string) (string, error) {
	data := Request{
		Url:  requestUrl,
		Mode: "turnstile-max",
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return "", err
	}
	var ccsResp struct {
		Token string `json:"token"`
		Code  int    `json:"code"`
	}
	err = json.Unmarshal(resp.Body(), &ccsResp)
	if err != nil {
		return "", err
	}
	if ccsResp.Code != 200 {
		return "", errors.New("Failed to get source")
	}
	return ccsResp.Token, nil
}

func RequestWithWAFSession(method string, URL string, wafSession Session, options *cycletls.Options) (cycletls.Response, error) {
	if options == nil {
		options = &cycletls.Options{}
	}
	headers := map[string]string{}
	for key, value := range wafSession.Headers {
		headers[key] = value
	}

	options.Ja3 = "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,23-27-65037-43-51-45-16-11-13-17513-5-18-65281-0-10-35,25497-29-23-24,0"
	options.UserAgent = headers["user-agent"]

	if options.Headers == nil {
		options.Headers = map[string]string{}
	}
	for key, value := range headers {
		options.Headers[key] = value
	}

	if options.Cookies == nil {
		options.Cookies = []cycletls.Cookie{}
	}
	for _, cookie := range wafSession.Cookies {
		options.Cookies = append(options.Cookies, cycletls.Cookie{
			Name:  cookie.Name,
			Value: cookie.Value,
		})
	}

	resp, err := cycletlsClient.Do(URL, *options, method)
	if err != nil {
		return resp, err
	}
	if resp.Status == 403 {
		return resp, errors.New("Cloudflare blocked request")
	}
	return resp, nil
}