100 lines
2.6 KiB
Go
100 lines
2.6 KiB
Go
|
package utils
|
||
|
|
|
||
|
|
import (
|
||
|
|
"net/url"
|
||
|
|
"runtime"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/go-rod/rod"
|
||
|
|
"github.com/go-rod/rod/lib/launcher"
|
||
|
|
"github.com/go-rod/rod/lib/proto"
|
||
|
|
"github.com/go-rod/stealth"
|
||
|
|
)
|
||
|
|
|
||
|
|
var userAgentMap = map[string]string{
|
||
|
|
"windows": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100",
|
||
|
|
"darwin": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36",
|
||
|
|
"linux": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36",
|
||
|
|
}
|
||
|
|
|
||
|
|
func buildBrowserAndPage(headless bool) (*rod.Browser, *rod.Page) {
|
||
|
|
if headless {
|
||
|
|
browser := rod.New().MustConnect()
|
||
|
|
page := stealth.MustPage(browser)
|
||
|
|
return browser, page
|
||
|
|
} else {
|
||
|
|
ws := launcher.New().Headless(false).MustLaunch()
|
||
|
|
browser := rod.New().ControlURL(ws).MustConnect()
|
||
|
|
page := stealth.MustPage(browser)
|
||
|
|
return browser, page
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func Rod(url string, timeout time.Duration, headless bool) (string, error) {
|
||
|
|
_, page := buildBrowserAndPage(headless)
|
||
|
|
userAgent := userAgentMap[runtime.GOOS]
|
||
|
|
err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
|
||
|
|
UserAgent: userAgent,
|
||
|
|
}).Navigate(url)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
err = page.WaitLoad()
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
return page.HTML()
|
||
|
|
}
|
||
|
|
|
||
|
|
func RodWaitElement(url string, selector string, timeout time.Duration, headless bool) (string, error) {
|
||
|
|
_, page := buildBrowserAndPage(headless)
|
||
|
|
userAgent := userAgentMap[runtime.GOOS]
|
||
|
|
err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
|
||
|
|
UserAgent: userAgent,
|
||
|
|
}).Navigate(url)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
_, err = page.Element(selector)
|
||
|
|
if err != nil {
|
||
|
|
return "", err
|
||
|
|
}
|
||
|
|
return page.HTML()
|
||
|
|
}
|
||
|
|
|
||
|
|
func RodWaitRequestFinish(URL string, waitUrl string, timeout time.Duration, headless bool) (string, string, error) {
|
||
|
|
_, page := buildBrowserAndPage(headless)
|
||
|
|
|
||
|
|
router := page.HijackRequests()
|
||
|
|
defer router.MustStop()
|
||
|
|
|
||
|
|
var err error
|
||
|
|
waitURL, err := url.Parse(waitUrl)
|
||
|
|
if err != nil {
|
||
|
|
return "", "", err
|
||
|
|
}
|
||
|
|
hijackBody := ""
|
||
|
|
done := make(chan bool)
|
||
|
|
router.MustAdd("*", func(ctx *rod.Hijack) {
|
||
|
|
ctx.ContinueRequest(&proto.FetchContinueRequest{})
|
||
|
|
|
||
|
|
if ctx.Request.URL() == waitURL {
|
||
|
|
hijackBody = string(ctx.Response.Payload().Body)
|
||
|
|
done <- true
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
go router.Run()
|
||
|
|
|
||
|
|
userAgent := userAgentMap[runtime.GOOS]
|
||
|
|
err = page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{
|
||
|
|
UserAgent: userAgent,
|
||
|
|
}).Navigate(URL)
|
||
|
|
if err != nil {
|
||
|
|
return "", "", err
|
||
|
|
}
|
||
|
|
<-done
|
||
|
|
body, err := page.HTML()
|
||
|
|
return body, hijackBody, err
|
||
|
|
}
|