package utils import ( "net/url" "runtime" "time" "github.com/go-rod/rod" "github.com/go-rod/rod/lib/launcher" "github.com/go-rod/rod/lib/proto" "github.com/go-rod/stealth" ) var userAgentMap = map[string]string{ "windows": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 GLS/100.10.9939.100", "darwin": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36", "linux": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", } func buildBrowserAndPage(headless bool) (*rod.Browser, *rod.Page) { if headless { browser := rod.New().MustConnect() page := stealth.MustPage(browser) return browser, page } else { ws := launcher.New().Headless(false).MustLaunch() browser := rod.New().ControlURL(ws).MustConnect() page := stealth.MustPage(browser) return browser, page } } func Rod(url string, timeout time.Duration, headless bool) (string, error) { _, page := buildBrowserAndPage(headless) userAgent := userAgentMap[runtime.GOOS] err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{ UserAgent: userAgent, }).Navigate(url) if err != nil { return "", err } err = page.WaitLoad() if err != nil { return "", err } return page.HTML() } func RodWaitElement(url string, selector string, timeout time.Duration, headless bool) (string, error) { _, page := buildBrowserAndPage(headless) userAgent := userAgentMap[runtime.GOOS] err := page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{ UserAgent: userAgent, }).Navigate(url) if err != nil { return "", err } _, err = page.Element(selector) if err != nil { return "", err } return page.HTML() } func RodWaitRequestFinish(URL string, waitUrl string, timeout time.Duration, headless bool) (string, string, error) { _, page := buildBrowserAndPage(headless) router := page.HijackRequests() defer router.MustStop() var err error waitURL, err := url.Parse(waitUrl) if err != nil { return "", "", err } hijackBody := "" done := make(chan bool) router.MustAdd("*", func(ctx *rod.Hijack) { ctx.ContinueRequest(&proto.FetchContinueRequest{}) if ctx.Request.URL() == waitURL { hijackBody = string(ctx.Response.Payload().Body) done <- true } }) go router.Run() userAgent := userAgentMap[runtime.GOOS] err = page.Timeout(timeout).MustSetUserAgent(&proto.NetworkSetUserAgentOverride{ UserAgent: userAgent, }).Navigate(URL) if err != nil { return "", "", err } <-done body, err := page.HTML() return body, hijackBody, err }