nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

add cf-clearance-scraper

Browse Source 
fix greegogcrawler
fix goggamescrawler
This commit is contained in:
Nite07 2024-11-20 10:40:33 +08:00
parent 21af3e5b3f
commit b7aadf7f88
12 changed files with 269 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
config/config.go
View File

@ -12,19 +12,24 @@ import (
)
type config struct {
LogLevel string `env:"LOG_LEVEL" json:"log_level"`
Server server `json:"server"`
Database database `json:"database"`
Redis redis `json:"redis"`
OnlineFix onlinefix `json:"online_fix"`
Twitch twitch `json:"twitch"`
Webhooks webhooks `json:"webhooks"`
LogLevel string `env:"LOG_LEVEL" json:"log_level"`
Server server `json:"server"`
Database database `json:"database"`
Redis redis `json:"redis"`
OnlineFix onlinefix `json:"online_fix"`
Twitch twitch `json:"twitch"`
Webhooks webhooks `json:"webhooks"`
CFClearanceScraper cfClearanceScraper `json:"cf_clearance_scraper"`
DatabaseAvaliable bool
OnlineFixAvaliable bool
MegaAvaliable bool
RedisAvaliable bool
}
type cfClearanceScraper struct {
Url string `env:"CF_CLEARANCE_SCRAPER_URL" json:"url"`
}
type webhooks struct {
CrawlTask []string `env:"WEBHOOKS_CRAWL_TASK" json:"crawl_task"`
}

8
crawler/crawler.go
View File

@ -24,10 +24,10 @@ type PagedCrawler interface {
func BuildCrawlerMap(logger *zap.Logger) map[string]Crawler {
ret := map[string]Crawler{
"fitgirl": NewFitGirlCrawler(logger),
"dodi": NewDODICrawler(logger),
"kaoskrew": NewKaOsKrewCrawler(logger),
// "freegog": NewFreeGOGCrawler(logger),
"fitgirl": NewFitGirlCrawler(logger),
"dodi": NewDODICrawler(logger),
"kaoskrew": NewKaOsKrewCrawler(logger),
"freegog": NewFreeGOGCrawler(logger),
"xatab": NewXatabCrawler(logger),
"onlinefix": NewOnlineFixCrawler(logger),
"steamrip": NewSteamRIPCrawler(logger),

35
crawler/freegog.go
View File

@ -3,10 +3,12 @@ package crawler
import (
"bytes"
"encoding/base64"
"errors"
"html"
"regexp"
"strings"
"github.com/nitezs/pcgamedb/config"
"github.com/nitezs/pcgamedb/constant"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
@ -17,21 +19,33 @@ import (
)
type FreeGOGCrawler struct {
logger *zap.Logger
logger *zap.Logger
session *utils.WAFSession
}
// Deprecated: Unable to get through cloudflare
func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
return &FreeGOGCrawler{
logger: logger,
}
}
func (c *FreeGOGCrawler) Name() string {
return "FreeGOG"
}
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
count := 0
resp, err := utils.Fetch(utils.FetchConfig{
var err error
if c.session == nil {
c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
}
if err != nil {
c.logger.Error("Failed to create session", zap.Error(err))
return nil, err
}
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
Url: constant.FreeGOGListURL,
})
}, c.session)
if err != nil {
c.logger.Error("Failed to fetch", zap.Error(err))
return nil, err
@ -86,9 +100,16 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
}
func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
var err error
if c.session == nil {
c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
}
if err != nil {
return nil, errors.New("Failed to create session")
}
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
Url: url,
})
}, c.session)
if err != nil {
return nil, err
}
@ -121,7 +142,7 @@ func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
}
item.Download = string(magnet)
} else {
return nil, err
return nil, errors.New("Failed to find magnet link")
}
item.Author = "FreeGOG"
return item, nil

17
crawler/goggames.go
View File

@ -6,6 +6,7 @@ import (
"strings"
"time"
"github.com/nitezs/pcgamedb/config"
"github.com/nitezs/pcgamedb/constant"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
@ -28,9 +29,17 @@ func (c *GOGGamesCrawler) Name() string {
return "GOGGamesCrawler"
}
func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
// URL is api url, like https://www.gog-games.to/api/v1/games/%s
func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
token, err := utils.CCSTurnstileToken(config.Config.CFClearanceScraper.Url, URL, "0x4AAAAAAAfOlgvCKbOdW1zc")
if err != nil {
return nil, err
}
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
Url: URL,
Headers: map[string]string{
"cf-turnstile-response": token,
},
})
if err != nil {
return nil, err
@ -57,14 +66,14 @@ func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
size += s
}
item, err := db.GetGameItemByUrl(url)
item, err := db.GetGameItemByUrl(URL)
if err != nil {
return nil, err
}
item.Name = name
item.RawName = name
item.Download = strings.Join(links, ",")
item.Url = url
item.Url = URL
item.Size = utils.BytesToSize(size)
item.Author = "GOGGames"
return item, nil

2
game_infos.json
View File

File diff suppressed because one or more lines are too long

2
games.json
View File

File diff suppressed because one or more lines are too long

7
go.mod
View File

@ -49,8 +49,6 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.23.0 // indirect
github.com/go-rod/rod v0.116.2 // indirect
github.com/go-rod/stealth v0.4.9 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@ -82,11 +80,6 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
github.com/ysmood/fetchup v0.2.4 // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/got v0.40.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/arch v0.12.0 // indirect
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect

20
go.sum
View File

@ -142,11 +142,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
@ -347,21 +342,6 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
github.com/ysmood/fetchup v0.2.4 h1:2kfWr/UrdiHg4KYRrxL2Jcrqx4DZYD+OtWu7WPBZl5o=
github.com/ysmood/fetchup v0.2.4/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A=
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
github.com/ysmood/got v0.40.0 h1:ZQk1B55zIvS7zflRrkGfPDrPG3d7+JOza1ZkNxcc74Q=
github.com/ysmood/got v0.40.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=

147
utils/cf-clearance-scraper.go Normal file
View File

@ -0,0 +1,147 @@
package utils
import (
"encoding/json"
"errors"
"time"
)
// https://github.com/ZFC-Digital/cf-clearance-scraper
type ccsRequest struct {
Url string `json:"url"`
Mode string `json:"mode"`
SiteKey string `json:"siteKey"`
}
type WAFSession struct {
Cookies []struct {
Name string `json:"name"`
Value string `json:"value"`
Domain string `json:"domain"`
Path string `json:"path"`
Expires float64 `json:"expires"`
Size int `json:"size"`
HTTPOnly bool `json:"httpOnly"`
Secure bool `json:"secure"`
Session bool `json:"session"`
SameSite string `json:"sameSite"`
Priority string `json:"priority"`
SameParty bool `json:"sameParty"`
SourceScheme string `json:"sourceScheme"`
PartitionKey string `json:"partitionKey"`
} `json:"cookies"`
Headers map[string]string `json:"headers"`
Code int `json:"code"`
}
func CCSWAFSession(ccsUrl string, requestUrl string) (*WAFSession, error) {
data := ccsRequest{
Url: requestUrl,
Mode: "waf-session",
}
resp, err := Fetch(FetchConfig{
Url: ccsUrl,
Method: "POST",
Data: data,
Timeout: 60 * time.Second,
})
if err != nil {
return nil, err
}
var response WAFSession
err = json.Unmarshal(resp.Data, &response)
if err != nil {
return nil, err
}
if response.Code != 200 {
return nil, errors.New("Failed to get WAF session")
}
return &response, nil
}
func CCSSource(ccsUrl string, requestUrl string) (string, error) {
data := ccsRequest{
Url: requestUrl,
Mode: "source",
}
resp, err := Fetch(FetchConfig{
Url: ccsUrl,
Method: "POST",
Data: data,
Timeout: 60 * time.Second,
})
if err != nil {
return "", err
}
type response struct {
Source string `json:"source"`
Code int `json:"code"`
}
var ccsResp response
err = json.Unmarshal(resp.Data, &ccsResp)
if err != nil {
return "", err
}
if ccsResp.Code != 200 {
return "", errors.New("Failed to get source")
}
return ccsResp.Source, nil
}
func CCSTurnstileToken(ccsUrl string, requestUrl string, siteKey string) (string, error) {
data := ccsRequest{
Url: requestUrl,
Mode: "turnstile-min",
SiteKey: siteKey,
}
resp, err := Fetch(FetchConfig{
Url: ccsUrl,
Method: "POST",
Data: data,
Timeout: 60 * time.Second,
})
if err != nil {
return "", err
}
var ccsResp struct {
Token string `json:"token"`
Code int `json:"code"`
}
err = json.Unmarshal(resp.Data, &ccsResp)
if err != nil {
return "", err
}
if ccsResp.Code != 200 {
return "", errors.New("Failed to get source")
}
return ccsResp.Token, nil
}
func CCSTurnstileMaxToken(ccsUrl string, requestUrl string) (string, error) {
data := ccsRequest{
Url: requestUrl,
Mode: "turnstile-max",
}
resp, err := Fetch(FetchConfig{
Url: ccsUrl,
Method: "POST",
Data: data,
Timeout: 60 * time.Second,
})
if err != nil {
return "", err
}
var ccsResp struct {
Token string `json:"token"`
Code int `json:"code"`
}
err = json.Unmarshal(resp.Data, &ccsResp)
if err != nil {
return "", err
}
if ccsResp.Code != 200 {
return "", errors.New("Failed to get source")
}
return ccsResp.Token, nil
}

82
utils/fetch.go
View File

@ -24,6 +24,7 @@ type FetchConfig struct {
RetryTimes int
Headers map[string]string
Cookies map[string]string
Timeout time.Duration
}
type FetchResponse struct {
@ -46,15 +47,23 @@ func Fetch(cfg FetchConfig) (*FetchResponse, error) {
if cfg.Method == "" {
cfg.Method = "GET"
}
if cfg.Timeout == 0 {
cfg.Timeout = 10 * time.Second
}
if cfg.Data != nil && (cfg.Method == "POST" || cfg.Method == "PUT") {
if cfg.Headers == nil {
cfg.Headers = map[string]string{}
}
if _, exist := cfg.Headers["Content-Type"]; !exist {
cfg.Headers["Content-Type"] = "application/json"
newHeaders := make(map[string]string)
for k, v := range cfg.Headers {
newHeaders[strings.ToLower(k)] = v
}
v := cfg.Headers["Content-Type"]
cfg.Headers = newHeaders
if _, exist := cfg.Headers["content-type"]; !exist {
cfg.Headers["content-type"] = "application/json"
}
v := cfg.Headers["content-type"]
if v == "application/x-www-form-urlencoded" {
switch data := cfg.Data.(type) {
case map[string]string:
@ -71,34 +80,53 @@ func Fetch(cfg FetchConfig) (*FetchResponse, error) {
return nil, errors.New("unsupported data type")
}
} else if v == "application/json" {
var jsonData []byte
jsonData, err = json.Marshal(cfg.Data)
if err != nil {
return nil, err
switch data := cfg.Data.(type) {
case []byte:
reqBody = bytes.NewReader(data)
case string:
reqBody = strings.NewReader(data)
case interface{}:
jsonData, err := json.Marshal(cfg.Data)
if err != nil {
return nil, err
}
reqBody = bytes.NewReader(jsonData)
default:
return nil, errors.New("unsupported data type")
}
reqBody = bytes.NewReader(jsonData)
} else {
reqBody = strings.NewReader(cfg.Data.(string))
}
}
for retryTime := 0; retryTime <= cfg.RetryTimes; retryTime++ {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
req, err = http.NewRequestWithContext(ctx, cfg.Method, cfg.Url, reqBody)
var bodyBuffer *bytes.Buffer
if reqBody != nil {
bodyBuffer = new(bytes.Buffer)
_, err = io.Copy(bodyBuffer, reqBody)
if err != nil {
return nil, err
}
if cfg.Method == "POST" || cfg.Method == "PUT" {
req.Header.Set("Content-Type", "application/json")
}
for retryTime := 0; retryTime <= cfg.RetryTimes; retryTime++ {
ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
defer cancel()
var currentReqBody io.Reader
if bodyBuffer != nil {
currentReqBody = bytes.NewReader(bodyBuffer.Bytes())
}
if v, exist := cfg.Headers["User-Agent"]; exist {
req, err = http.NewRequestWithContext(ctx, cfg.Method, cfg.Url, currentReqBody)
if err != nil {
return nil, err
}
if v, exist := cfg.Headers["user-agent"]; exist {
if v != "" {
req.Header.Set("User-Agent", v)
req.Header.Set("user-agent", v)
}
} else {
req.Header.Set("User-Agent", userAgent)
req.Header.Set("user-agent", userAgent)
}
for k, v := range cfg.Headers {
req.Header.Set(k, v)
@ -127,7 +155,7 @@ func Fetch(cfg FetchConfig) (*FetchResponse, error) {
continue
}
contentType := resp.Header.Get("Content-Type")
contentType := resp.Header.Get("content-type")
var reader io.Reader
if strings.Contains(contentType, "charset=") {
reader, err = charset.NewReader(resp.Body, contentType)
@ -176,3 +204,19 @@ func isRetryableError(err error) bool {
}
return false
}
func FetchWithWAFSession(cfg FetchConfig, session *WAFSession) (*FetchResponse, error) {
if cfg.Cookies == nil {
cfg.Cookies = map[string]string{}
}
for _, cookie := range session.Cookies {
cfg.Cookies[cookie.Name] = cookie.Value
}
if cfg.Headers == nil {
cfg.Headers = map[string]string{}
}
for k, v := range session.Headers {
cfg.Headers[k] = v
}
return Fetch(cfg)
}

37
utils/magnet.go
View File

@ -2,8 +2,6 @@ package utils
import (
"bytes"
"strconv"
"strings"
"github.com/anacrolix/torrent/metainfo"
)
@ -29,38 +27,3 @@ func ConvertTorrentToMagnet(torrent []byte) (string, string, error) {
}
return magnet.String(), BytesToSize(size), nil
}
func SubSizeStrings(sizes []string) (string, error) {
size := uint64(0)
for _, sizeStr := range sizes {
sizeStr := strings.ToLower(sizeStr)
if strings.Contains(sizeStr, "gb") {
sizeStr = strings.ReplaceAll(sizeStr, "gb", "")
sizeStr = strings.TrimSpace(sizeStr)
addSize, err := strconv.ParseFloat(sizeStr, 64)
if err != nil {
return "", err
}
size += uint64(addSize * 1024 * 1024 * 1024)
}
if strings.Contains(sizeStr, "mb") {
sizeStr = strings.ReplaceAll(sizeStr, "mb", "")
sizeStr = strings.TrimSpace(sizeStr)
addSize, err := strconv.ParseFloat(sizeStr, 64)
if err != nil {
return "", err
}
size += uint64(addSize * 1024 * 1024)
}
if strings.Contains(sizeStr, "kb") {
sizeStr = strings.ReplaceAll(sizeStr, "kb", "")
sizeStr = strings.TrimSpace(sizeStr)
addSize, err := strconv.ParseFloat(sizeStr, 64)
if err != nil {
return "", err
}
size += uint64(addSize * 1024)
}
}
return BytesToSize(size), nil
}

99
utils/rod.go
View File

@ -1,99 +0,0 @@
package utils
import (
"net/url"
"runtime"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/proto"
"github.com/go-rod/stealth"
)
var userAgentMap = map[string]string{
"windows": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100",
"darwin": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
"linux": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
}
func buildBrowserAndPage(headless bool) (*rod.Browser, *rod.Page) {
if headless {
browser := rod.New().MustConnect()
page := stealth.MustPage(browser)
return browser, page
} else {
ws := launcher.New().Headless(false).MustLaunch()
browser := rod.New().ControlURL(ws).MustConnect()
page := stealth.MustPage(browser)
return browser, page
}
}
func Rod(url string, timeout time.Duration, headless bool) (string, error) {
_, page := buildBrowserAndPage(headless)
userAgent := userAgentMap[runtime.GOOS]
err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
UserAgent: userAgent,
}).Navigate(url)
if err != nil {
return "", err
}
err = page.WaitLoad()
if err != nil {
return "", err
}
return page.HTML()
}
func RodWaitElement(url string, selector string, timeout time.Duration, headless bool) (string, error) {
_, page := buildBrowserAndPage(headless)
userAgent := userAgentMap[runtime.GOOS]
err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
UserAgent: userAgent,
}).Navigate(url)
if err != nil {
return "", err
}
_, err = page.Element(selector)
if err != nil {
return "", err
}
return page.HTML()
}
func RodWaitRequestFinish(URL string, waitUrl string, timeout time.Duration, headless bool) (string, string, error) {
_, page := buildBrowserAndPage(headless)
router := page.HijackRequests()
defer router.MustStop()
var err error
waitURL, err := url.Parse(waitUrl)
if err != nil {
return "", "", err
}
hijackBody := ""
done := make(chan bool)
router.MustAdd("*", func(ctx *rod.Hijack) {
ctx.ContinueRequest(&proto.FetchContinueRequest{})
if ctx.Request.URL() == waitURL {
hijackBody = string(ctx.Response.Payload().Body)
done <- true
}
})
go router.Run()
userAgent := userAgentMap[runtime.GOOS]
err = page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
UserAgent: userAgent,
}).Navigate(URL)
if err != nil {
return "", "", err
}
<-done
body, err := page.HTML()
return body, hijackBody, err
}