nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

try fix freegog crawler
All checks were successful
docker / prepare-and-build (push) Successful in 3m1s
Details
release / goreleaser (push) Successful in 4m30s
Details

Browse Source
This commit is contained in:
Nite07 2024-11-21 00:03:39 +08:00
parent ea656745ec
commit b138550d9b
1 changed files with 6 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
crawler/freegog.go
View File

@ -19,8 +19,7 @@ import (
)
type FreeGOGCrawler struct {
logger *zap.Logger
session *utils.WAFSession
logger *zap.Logger
}
func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
@ -35,17 +34,14 @@ func (c *FreeGOGCrawler) Name() string {
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
count := 0
var err error
if c.session == nil {
c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
}
session, err := utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
if err != nil {
c.logger.Error("Failed to create session", zap.Error(err))
return nil, err
}
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
Url: constant.FreeGOGListURL,
}, c.session)
}, session)
if err != nil {
c.logger.Error("Failed to fetch", zap.Error(err))
return nil, err
@ -72,7 +68,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
continue
}
c.logger.Info("Crawling", zap.String("URL", u))
item, err := c.CrawlByUrl(u)
item, err := c.CrawlByUrl(u, session)
if err != nil {
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
continue
@ -99,17 +95,14 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
return res, nil
}
func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
func (c *FreeGOGCrawler) CrawlByUrl(url string, session *utils.WAFSession) (*model.GameItem, error) {
var err error
if c.session == nil {
c.session, err = utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
}
if err != nil {
return nil, errors.New("Failed to create session")
}
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
Url: url,
}, c.session)
}, session)
if err != nil {
return nil, err
}