ccs/cf-clearance-scraper.go

package ccs

import (
	"encoding/json"
	"errors"
	"strings"
	"time"

	"github.com/Danny-Dasilva/CycleTLS/cycletls"
	"github.com/go-resty/resty/v2"
)

type Request struct {
	Url     string `json:"url"`
	Mode    string `json:"mode"`
	SiteKey string `json:"siteKey"`
}

type Session struct {
	Cookies []struct {
		Name         string  `json:"name"`
		Value        string  `json:"value"`
		Domain       string  `json:"domain"`
		Path         string  `json:"path"`
		Expires      float64 `json:"expires"`
		Size         int     `json:"size"`
		HTTPOnly     bool    `json:"httpOnly"`
		Secure       bool    `json:"secure"`
		Session      bool    `json:"session"`
		SameSite     string  `json:"sameSite"`
		Priority     string  `json:"priority"`
		SameParty    bool    `json:"sameParty"`
		SourceScheme string  `json:"sourceScheme"`
		PartitionKey string  `json:"partitionKey"`
	} `json:"cookies"`
	Headers map[string]string `json:"headers"`
	Code    int               `json:"code"`
}

var (
	httpClient     *resty.Client
	cycletlsClient cycletls.CycleTLS
)

func init() {
	httpClient = resty.New()
	httpClient.SetRetryCount(3).SetRetryWaitTime(3 * time.Second)
	cycletlsClient = cycletls.Init()
}

func WAFSession(ccsUrl string, requestUrl string) (Session, error) {
	data := Request{
		Url:  requestUrl,
		Mode: "waf-session",
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return Session{}, err
	}
	var response Session
	err = json.Unmarshal(resp.Body(), &response)
	if err != nil {
		return Session{}, err
	}
	if response.Code != 200 {
		return Session{}, errors.New("Failed to get WAF session")
	}
	return response, nil
}

func Source(ccsUrl string, requestUrl string) ([]byte, error) {
	data := Request{
		Url:  requestUrl,
		Mode: "source",
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return nil, err
	}
	type response struct {
		Source string `json:"source"`
		Code   int    `json:"code"`
	}
	var ccsResp response
	err = json.Unmarshal(resp.Body(), &ccsResp)
	if err != nil {
		return nil, err
	}
	if ccsResp.Code != 200 {
		return nil, errors.New("Failed to get source")
	}
	return []byte(ccsResp.Source), nil
}

func TurnstileToken(ccsUrl string, requestUrl string, siteKey string) (string, error) {
	data := Request{
		Url:     requestUrl,
		Mode:    "turnstile-min",
		SiteKey: siteKey,
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return "", err
	}
	var ccsResp struct {
		Token string `json:"token"`
		Code  int    `json:"code"`
	}
	err = json.Unmarshal(resp.Body(), &ccsResp)
	if err != nil {
		return "", err
	}
	if ccsResp.Code != 200 {
		return "", errors.New("Failed to get source")
	}
	return ccsResp.Token, nil
}

func TurnstileMaxToken(ccsUrl string, requestUrl string) (string, error) {
	data := Request{
		Url:  requestUrl,
		Mode: "turnstile-max",
	}
	resp, err := httpClient.SetTimeout(60 * time.Second).R().SetBody(data).Post(ccsUrl)
	if err != nil {
		return "", err
	}
	var ccsResp struct {
		Token string `json:"token"`
		Code  int    `json:"code"`
	}
	err = json.Unmarshal(resp.Body(), &ccsResp)
	if err != nil {
		return "", err
	}
	if ccsResp.Code != 200 {
		return "", errors.New("Failed to get source")
	}
	return ccsResp.Token, nil
}

func RequestWithWAFSession(method string, URL string, wafSession Session, body interface{}) (cycletls.Response, error) {
	headers := map[string]string{}
	cookies := []string{}
	for _, cookie := range wafSession.Cookies {
		cookies = append(cookies, cookie.Name+"="+cookie.Value)
	}
	for key, value := range wafSession.Headers {
		headers[key] = value
	}
	headers["Cookie"] = strings.Join(cookies, "; ")

	options := cycletls.Options{
		Body:      "",
		Ja3:       "772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,23-27-65037-43-51-45-16-11-13-17513-5-18-65281-0-10-35,25497-29-23-24,0",
		UserAgent: headers["user-agent"],
		Headers:   headers,
	}

	return cycletlsClient.Do(URL, options, method)
}