nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

try fix freegog crawler
All checks were successful
docker / prepare-and-build (push) Successful in 3m1s
Details
release / goreleaser (push) Successful in 4m30s
Details

Browse Source
This commit is contained in:
Nite07 2024-11-21 00:03:39 +08:00
parent ea656745ec
commit b138550d9b
1 changed files with 6 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crawler/freegog.go
View File

@ -19,8 +19,7 @@ import (
) )
type FreeGOGCrawler struct { type FreeGOGCrawler struct {
logger *zap.Logger logger *zap.Logger
session *utils.WAFSession
} }
func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler { func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
@ -35,17 +34,14 @@ func (c *FreeGOGCrawler) Name() string {
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) { func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
count := 0 count := 0
var err error session, err := utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
if c.session == nil {
c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
}
if err != nil { if err != nil {
c.logger.Error("Failed to create session", zap.Error(err)) c.logger.Error("Failed to create session", zap.Error(err))
return nil, err return nil, err
} }
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{ resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
Url: constant.FreeGOGListURL, Url: constant.FreeGOGListURL,
}, c.session) }, session)
if err != nil { if err != nil {
c.logger.Error("Failed to fetch", zap.Error(err)) c.logger.Error("Failed to fetch", zap.Error(err))
return nil, err return nil, err
@ -72,7 +68,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
continue continue
} }
c.logger.Info("Crawling", zap.String("URL", u)) c.logger.Info("Crawling", zap.String("URL", u))
item, err := c.CrawlByUrl(u) item, err := c.CrawlByUrl(u, session)
if err != nil { if err != nil {
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u)) c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
continue continue
@ -99,17 +95,14 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
return res, nil return res, nil
} }
func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) { func (c *FreeGOGCrawler) CrawlByUrl(url string, session *utils.WAFSession) (*model.GameItem, error) {
var err error var err error
if c.session == nil {
c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
}
if err != nil { if err != nil {
return nil, errors.New("Failed to create session") return nil, errors.New("Failed to create session")
} }
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{ resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
Url: url, Url: url,
}, c.session) }, session)
if err != nil { if err != nil {
return nil, err return nil, err
} }