add cf-clearance-scraper

fix greegogcrawler
fix goggamescrawler

config/config.go

@@ -12,19 +12,24 @@ import (
 )
 
 type config struct {
-	LogLevel           string    `env:"LOG_LEVEL" json:"log_level"`
+	LogLevel           string             `env:"LOG_LEVEL" json:"log_level"`
-	Server             server    `json:"server"`
+	Server             server             `json:"server"`
-	Database           database  `json:"database"`
+	Database           database           `json:"database"`
-	Redis              redis     `json:"redis"`
+	Redis              redis              `json:"redis"`
-	OnlineFix          onlinefix `json:"online_fix"`
+	OnlineFix          onlinefix          `json:"online_fix"`
-	Twitch             twitch    `json:"twitch"`
+	Twitch             twitch             `json:"twitch"`
-	Webhooks           webhooks  `json:"webhooks"`
+	Webhooks           webhooks           `json:"webhooks"`
+	CFClearanceScraper cfClearanceScraper `json:"cf_clearance_scraper"`
 	DatabaseAvaliable  bool
 	OnlineFixAvaliable bool
 	MegaAvaliable      bool
 	RedisAvaliable     bool
 }
 
+type cfClearanceScraper struct {
+	Url string `env:"CF_CLEARANCE_SCRAPER_URL" json:"url"`
+}
+
 type webhooks struct {
 	CrawlTask []string `env:"WEBHOOKS_CRAWL_TASK" json:"crawl_task"`
 }

crawler/crawler.go

@@ -24,10 +24,10 @@ type PagedCrawler interface {
 
 func BuildCrawlerMap(logger *zap.Logger) map[string]Crawler {
 	ret := map[string]Crawler{
-		"fitgirl":  NewFitGirlCrawler(logger),
+		"fitgirl":   NewFitGirlCrawler(logger),
-		"dodi":     NewDODICrawler(logger),
+		"dodi":      NewDODICrawler(logger),
-		"kaoskrew": NewKaOsKrewCrawler(logger),
+		"kaoskrew":  NewKaOsKrewCrawler(logger),
-		// "freegog":   NewFreeGOGCrawler(logger),
+		"freegog":   NewFreeGOGCrawler(logger),
 		"xatab":     NewXatabCrawler(logger),
 		"onlinefix": NewOnlineFixCrawler(logger),
 		"steamrip":  NewSteamRIPCrawler(logger),

crawler/freegog.go

@@ -3,10 +3,12 @@ package crawler
 import (
 	"bytes"
 	"encoding/base64"
+	"errors"
 	"html"
 	"regexp"
 	"strings"
 
+	"github.com/nitezs/pcgamedb/config"
 	"github.com/nitezs/pcgamedb/constant"
 	"github.com/nitezs/pcgamedb/db"
 	"github.com/nitezs/pcgamedb/model"
@@ -17,21 +19,33 @@ import (
 )
 
 type FreeGOGCrawler struct {
-	logger *zap.Logger
+	logger  *zap.Logger
+	session *utils.WAFSession
 }
 
-// Deprecated: Unable to get through cloudflare
 func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
 	return &FreeGOGCrawler{
 		logger: logger,
 	}
 }
 
+func (c *FreeGOGCrawler) Name() string {
+	return "FreeGOG"
+}
+
 func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
 	count := 0
-	resp, err := utils.Fetch(utils.FetchConfig{
+	var err error
+	if c.session == nil {
+		c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
+	}
+	if err != nil {
+		c.logger.Error("Failed to create session", zap.Error(err))
+		return nil, err
+	}
+	resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
 		Url: constant.FreeGOGListURL,
-	})
+	}, c.session)
 	if err != nil {
 		c.logger.Error("Failed to fetch", zap.Error(err))
 		return nil, err
@@ -86,9 +100,16 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
 }
 
 func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
-	resp, err := utils.Fetch(utils.FetchConfig{
+	var err error
+	if c.session == nil {
+		c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
+	}
+	if err != nil {
+		return nil, errors.New("Failed to create session")
+	}
+	resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
 		Url: url,
-	})
+	}, c.session)
 	if err != nil {
 		return nil, err
 	}
@@ -121,7 +142,7 @@ func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 		}
 		item.Download = string(magnet)
 	} else {
-		return nil, err
+		return nil, errors.New("Failed to find magnet link")
 	}
 	item.Author = "FreeGOG"
 	return item, nil

crawler/goggames.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/nitezs/pcgamedb/config"
 	"github.com/nitezs/pcgamedb/constant"
 	"github.com/nitezs/pcgamedb/db"
 	"github.com/nitezs/pcgamedb/model"
@@ -28,9 +29,17 @@ func (c *GOGGamesCrawler) Name() string {
 	return "GOGGamesCrawler"
 }
 
-func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
+// URL is api url, like https://www.gog-games.to/api/v1/games/%s
+func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
+	token, err := utils.CCSTurnstileToken(config.Config.CFClearanceScraper.Url, URL, "0x4AAAAAAAfOlgvCKbOdW1zc")
+	if err != nil {
+		return nil, err
+	}
 	resp, err := utils.Fetch(utils.FetchConfig{
-		Url: url,
+		Url: URL,
+		Headers: map[string]string{
+			"cf-turnstile-response": token,
+		},
 	})
 	if err != nil {
 		return nil, err
@@ -57,14 +66,14 @@ func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 		size += s
 	}
 
-	item, err := db.GetGameItemByUrl(url)
+	item, err := db.GetGameItemByUrl(URL)
 	if err != nil {
 		return nil, err
 	}
 	item.Name = name
 	item.RawName = name
 	item.Download = strings.Join(links, ",")
-	item.Url = url
+	item.Url = URL
 	item.Size = utils.BytesToSize(size)
 	item.Author = "GOGGames"
 	return item, nil

go.mod

@@ -49,8 +49,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.23.0 // indirect
-	github.com/go-rod/rod v0.116.2 // indirect
-	github.com/go-rod/stealth v0.4.9 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -82,11 +80,6 @@ require (
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
-	github.com/ysmood/fetchup v0.2.4 // indirect
-	github.com/ysmood/goob v0.4.0 // indirect
-	github.com/ysmood/got v0.40.0 // indirect
-	github.com/ysmood/gson v0.7.3 // indirect
-	github.com/ysmood/leakless v0.9.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/arch v0.12.0 // indirect
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect

go.sum

@@ -142,11 +142,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
 github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
-github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
-github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
-github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
-github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
-github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
 github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
@@ -347,21 +342,6 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
-github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
-github.com/ysmood/fetchup v0.2.4 h1:2kfWr/UrdiHg4KYRrxL2Jcrqx4DZYD+OtWu7WPBZl5o=
-github.com/ysmood/fetchup v0.2.4/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A=
-github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
-github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
-github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
-github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
-github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
-github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
-github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
-github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
-github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
-github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
-github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
-github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
 go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=

utils/cf-clearance-scraper.go

@@ -0,0 +1,147 @@
+package utils
+
+import (
+	"encoding/json"
+	"errors"
+	"time"
+)
+
+// https://github.com/ZFC-Digital/cf-clearance-scraper
+
+type ccsRequest struct {
+	Url     string `json:"url"`
+	Mode    string `json:"mode"`
+	SiteKey string `json:"siteKey"`
+}
+
+type WAFSession struct {
+	Cookies []struct {
+		Name         string  `json:"name"`
+		Value        string  `json:"value"`
+		Domain       string  `json:"domain"`
+		Path         string  `json:"path"`
+		Expires      float64 `json:"expires"`
+		Size         int     `json:"size"`
+		HTTPOnly     bool    `json:"httpOnly"`
+		Secure       bool    `json:"secure"`
+		Session      bool    `json:"session"`
+		SameSite     string  `json:"sameSite"`
+		Priority     string  `json:"priority"`
+		SameParty    bool    `json:"sameParty"`
+		SourceScheme string  `json:"sourceScheme"`
+		PartitionKey string  `json:"partitionKey"`
+	} `json:"cookies"`
+	Headers map[string]string `json:"headers"`
+	Code    int               `json:"code"`
+}
+
+func CCSWAFSession(ccsUrl string, requestUrl string) (*WAFSession, error) {
+	data := ccsRequest{
+		Url:  requestUrl,
+		Mode: "waf-session",
+	}
+	resp, err := Fetch(FetchConfig{
+		Url:     ccsUrl,
+		Method:  "POST",
+		Data:    data,
+		Timeout: 60 * time.Second,
+	})
+	if err != nil {
+		return nil, err
+	}
+	var response WAFSession
+	err = json.Unmarshal(resp.Data, &response)
+	if err != nil {
+		return nil, err
+	}
+	if response.Code != 200 {
+		return nil, errors.New("Failed to get WAF session")
+	}
+	return &response, nil
+}
+
+func CCSSource(ccsUrl string, requestUrl string) (string, error) {
+	data := ccsRequest{
+		Url:  requestUrl,
+		Mode: "source",
+	}
+	resp, err := Fetch(FetchConfig{
+		Url:     ccsUrl,
+		Method:  "POST",
+		Data:    data,
+		Timeout: 60 * time.Second,
+	})
+	if err != nil {
+		return "", err
+	}
+	type response struct {
+		Source string `json:"source"`
+		Code   int    `json:"code"`
+	}
+	var ccsResp response
+	err = json.Unmarshal(resp.Data, &ccsResp)
+	if err != nil {
+		return "", err
+	}
+	if ccsResp.Code != 200 {
+		return "", errors.New("Failed to get source")
+	}
+	return ccsResp.Source, nil
+}
+
+func CCSTurnstileToken(ccsUrl string, requestUrl string, siteKey string) (string, error) {
+	data := ccsRequest{
+		Url:     requestUrl,
+		Mode:    "turnstile-min",
+		SiteKey: siteKey,
+	}
+	resp, err := Fetch(FetchConfig{
+		Url:     ccsUrl,
+		Method:  "POST",
+		Data:    data,
+		Timeout: 60 * time.Second,
+	})
+	if err != nil {
+		return "", err
+	}
+	var ccsResp struct {
+		Token string `json:"token"`
+		Code  int    `json:"code"`
+	}
+	err = json.Unmarshal(resp.Data, &ccsResp)
+	if err != nil {
+		return "", err
+	}
+	if ccsResp.Code != 200 {
+		return "", errors.New("Failed to get source")
+	}
+	return ccsResp.Token, nil
+}
+
+func CCSTurnstileMaxToken(ccsUrl string, requestUrl string) (string, error) {
+	data := ccsRequest{
+		Url:  requestUrl,
+		Mode: "turnstile-max",
+	}
+	resp, err := Fetch(FetchConfig{
+		Url:     ccsUrl,
+		Method:  "POST",
+		Data:    data,
+		Timeout: 60 * time.Second,
+	})
+	if err != nil {
+		return "", err
+	}
+	var ccsResp struct {
+		Token string `json:"token"`
+		Code  int    `json:"code"`
+	}
+	err = json.Unmarshal(resp.Data, &ccsResp)
+	if err != nil {
+		return "", err
+	}
+	if ccsResp.Code != 200 {
+		return "", errors.New("Failed to get source")
+	}
+	return ccsResp.Token, nil
+}

utils/fetch.go

@@ -24,6 +24,7 @@ type FetchConfig struct {
 	RetryTimes int
 	Headers    map[string]string
 	Cookies    map[string]string
+	Timeout    time.Duration
 }
 
 type FetchResponse struct {
@@ -46,15 +47,23 @@ func Fetch(cfg FetchConfig) (*FetchResponse, error) {
 	if cfg.Method == "" {
 		cfg.Method = "GET"
 	}
+	if cfg.Timeout == 0 {
+		cfg.Timeout = 10 * time.Second
+	}
 
 	if cfg.Data != nil && (cfg.Method == "POST" || cfg.Method == "PUT") {
 		if cfg.Headers == nil {
 			cfg.Headers = map[string]string{}
 		}
-		if _, exist := cfg.Headers["Content-Type"]; !exist {
+		newHeaders := make(map[string]string)
-			cfg.Headers["Content-Type"] = "application/json"
+		for k, v := range cfg.Headers {
+			newHeaders[strings.ToLower(k)] = v
 		}
-		v := cfg.Headers["Content-Type"]
+		cfg.Headers = newHeaders
+		if _, exist := cfg.Headers["content-type"]; !exist {
+			cfg.Headers["content-type"] = "application/json"
+		}
+		v := cfg.Headers["content-type"]
 		if v == "application/x-www-form-urlencoded" {
 			switch data := cfg.Data.(type) {
 			case map[string]string:
@@ -71,34 +80,53 @@ func Fetch(cfg FetchConfig) (*FetchResponse, error) {
 				return nil, errors.New("unsupported data type")
 			}
 		} else if v == "application/json" {
-			var jsonData []byte
+			switch data := cfg.Data.(type) {
-			jsonData, err = json.Marshal(cfg.Data)
+			case []byte:
-			if err != nil {
+				reqBody = bytes.NewReader(data)
-				return nil, err
+			case string:
+				reqBody = strings.NewReader(data)
+			case interface{}:
+				jsonData, err := json.Marshal(cfg.Data)
+				if err != nil {
+					return nil, err
+				}
+				reqBody = bytes.NewReader(jsonData)
+			default:
+				return nil, errors.New("unsupported data type")
 			}
-			reqBody = bytes.NewReader(jsonData)
 		} else {
 			reqBody = strings.NewReader(cfg.Data.(string))
 		}
 	}
 
-	for retryTime := 0; retryTime <= cfg.RetryTimes; retryTime++ {
+	var bodyBuffer *bytes.Buffer
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	if reqBody != nil {
-		defer cancel()
+		bodyBuffer = new(bytes.Buffer)
-
+		_, err = io.Copy(bodyBuffer, reqBody)
-		req, err = http.NewRequestWithContext(ctx, cfg.Method, cfg.Url, reqBody)
 		if err != nil {
 			return nil, err
 		}
-		if cfg.Method == "POST" || cfg.Method == "PUT" {
+	}
-			req.Header.Set("Content-Type", "application/json")
+
+	for retryTime := 0; retryTime <= cfg.RetryTimes; retryTime++ {
+		ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
+		defer cancel()
+
+		var currentReqBody io.Reader
+		if bodyBuffer != nil {
+			currentReqBody = bytes.NewReader(bodyBuffer.Bytes())
 		}
-		if v, exist := cfg.Headers["User-Agent"]; exist {
+
+		req, err = http.NewRequestWithContext(ctx, cfg.Method, cfg.Url, currentReqBody)
+		if err != nil {
+			return nil, err
+		}
+		if v, exist := cfg.Headers["user-agent"]; exist {
 			if v != "" {
-				req.Header.Set("User-Agent", v)
+				req.Header.Set("user-agent", v)
 			}
 		} else {
-			req.Header.Set("User-Agent", userAgent)
+			req.Header.Set("user-agent", userAgent)
 		}
 		for k, v := range cfg.Headers {
 			req.Header.Set(k, v)
@@ -127,7 +155,7 @@ func Fetch(cfg FetchConfig) (*FetchResponse, error) {
 			continue
 		}
 
-		contentType := resp.Header.Get("Content-Type")
+		contentType := resp.Header.Get("content-type")
 		var reader io.Reader
 		if strings.Contains(contentType, "charset=") {
 			reader, err = charset.NewReader(resp.Body, contentType)
@@ -176,3 +204,19 @@ func isRetryableError(err error) bool {
 	}
 	return false
 }
+
+func FetchWithWAFSession(cfg FetchConfig, session *WAFSession) (*FetchResponse, error) {
+	if cfg.Cookies == nil {
+		cfg.Cookies = map[string]string{}
+	}
+	for _, cookie := range session.Cookies {
+		cfg.Cookies[cookie.Name] = cookie.Value
+	}
+	if cfg.Headers == nil {
+		cfg.Headers = map[string]string{}
+	}
+	for k, v := range session.Headers {
+		cfg.Headers[k] = v
+	}
+	return Fetch(cfg)
+}

utils/magnet.go

@@ -2,8 +2,6 @@ package utils
 
 import (
 	"bytes"
-	"strconv"
-	"strings"
 
 	"github.com/anacrolix/torrent/metainfo"
 )
@@ -29,38 +27,3 @@ func ConvertTorrentToMagnet(torrent []byte) (string, string, error) {
 	}
 	return magnet.String(), BytesToSize(size), nil
 }
-
-func SubSizeStrings(sizes []string) (string, error) {
-	size := uint64(0)
-	for _, sizeStr := range sizes {
-		sizeStr := strings.ToLower(sizeStr)
-		if strings.Contains(sizeStr, "gb") {
-			sizeStr = strings.ReplaceAll(sizeStr, "gb", "")
-			sizeStr = strings.TrimSpace(sizeStr)
-			addSize, err := strconv.ParseFloat(sizeStr, 64)
-			if err != nil {
-				return "", err
-			}
-			size += uint64(addSize * 1024 * 1024 * 1024)
-		}
-		if strings.Contains(sizeStr, "mb") {
-			sizeStr = strings.ReplaceAll(sizeStr, "mb", "")
-			sizeStr = strings.TrimSpace(sizeStr)
-			addSize, err := strconv.ParseFloat(sizeStr, 64)
-			if err != nil {
-				return "", err
-			}
-			size += uint64(addSize * 1024 * 1024)
-		}
-		if strings.Contains(sizeStr, "kb") {
-			sizeStr = strings.ReplaceAll(sizeStr, "kb", "")
-			sizeStr = strings.TrimSpace(sizeStr)
-			addSize, err := strconv.ParseFloat(sizeStr, 64)
-			if err != nil {
-				return "", err
-			}
-			size += uint64(addSize * 1024)
-		}
-	}
-	return BytesToSize(size), nil
-}

utils/rod.go

@@ -1,99 +0,0 @@
-package utils
-
-import (
-	"net/url"
-	"runtime"
-	"time"
-
-	"github.com/go-rod/rod"
-	"github.com/go-rod/rod/lib/launcher"
-	"github.com/go-rod/rod/lib/proto"
-	"github.com/go-rod/stealth"
-)
-
-var userAgentMap = map[string]string{
-	"windows": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100",
-	"darwin":  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
-	"linux":   "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
-}
-
-func buildBrowserAndPage(headless bool) (*rod.Browser, *rod.Page) {
-	if headless {
-		browser := rod.New().MustConnect()
-		page := stealth.MustPage(browser)
-		return browser, page
-	} else {
-		ws := launcher.New().Headless(false).MustLaunch()
-		browser := rod.New().ControlURL(ws).MustConnect()
-		page := stealth.MustPage(browser)
-		return browser, page
-	}
-}
-
-func Rod(url string, timeout time.Duration, headless bool) (string, error) {
-	_, page := buildBrowserAndPage(headless)
-	userAgent := userAgentMap[runtime.GOOS]
-	err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
-		UserAgent: userAgent,
-	}).Navigate(url)
-	if err != nil {
-		return "", err
-	}
-	err = page.WaitLoad()
-	if err != nil {
-		return "", err
-	}
-	return page.HTML()
-}
-
-func RodWaitElement(url string, selector string, timeout time.Duration, headless bool) (string, error) {
-	_, page := buildBrowserAndPage(headless)
-	userAgent := userAgentMap[runtime.GOOS]
-	err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
-		UserAgent: userAgent,
-	}).Navigate(url)
-	if err != nil {
-		return "", err
-	}
-	_, err = page.Element(selector)
-	if err != nil {
-		return "", err
-	}
-	return page.HTML()
-}
-
-func RodWaitRequestFinish(URL string, waitUrl string, timeout time.Duration, headless bool) (string, string, error) {
-	_, page := buildBrowserAndPage(headless)
-
-	router := page.HijackRequests()
-	defer router.MustStop()
-
-	var err error
-	waitURL, err := url.Parse(waitUrl)
-	if err != nil {
-		return "", "", err
-	}
-	hijackBody := ""
-	done := make(chan bool)
-	router.MustAdd("*", func(ctx *rod.Hijack) {
-		ctx.ContinueRequest(&proto.FetchContinueRequest{})
-
-		if ctx.Request.URL() == waitURL {
-			hijackBody = string(ctx.Response.Payload().Body)
-			done <- true
-		}
-	})
-
-	go router.Run()
-
-	userAgent := userAgentMap[runtime.GOOS]
-	err = page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
-		UserAgent: userAgent,
-	}).Navigate(URL)
-	if err != nil {
-		return "", "", err
-	}
-	<-done
-	body, err := page.HTML()
-	return body, hijackBody, err
-}