package crawler import ( "encoding/json" "fmt" "path" "strings" "time" "pcgamedb/config" "pcgamedb/constant" "pcgamedb/db" "pcgamedb/model" "pcgamedb/utils" "git.nite07.com/nite/ccs" "go.uber.org/zap" ) type GOGGamesCrawler struct { logger *zap.Logger } func NewGOGGamesCrawler(logger *zap.Logger) *GOGGamesCrawler { return &GOGGamesCrawler{ logger: logger, } } func (c *GOGGamesCrawler) Name() string { return "GOGGamesCrawler" } func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { if !strings.HasPrefix(URL, "https://www.gog-games.to/game/") { return nil, fmt.Errorf("invalid url") } _, slug := path.Split(URL) apiUrl := fmt.Sprintf(constant.GOGGamesGameAPIURL, slug) token, err := ccs.TurnstileToken(config.Config.CFClearanceScraper.Url, apiUrl, "0x4AAAAAAAfOlgvCKbOdW1zc") if err != nil { return nil, err } resp, err := utils.Request().SetHeader("cf-turnstile-response", token).Get(apiUrl) if err != nil { return nil, err } data := gameResult{} err = json.Unmarshal(resp.Body(), &data) if err != nil { return nil, err } name := data.Title // find download links fileHosters := []string{ "gofile", "fileditch", "qiwi", "filesfm", "pixeldrain", "1fichier", } links := make([]string, 0) for _, h := range fileHosters { if value, exist := data.Links.Game[h]; exist { for _, link := range value.Links { links = append(links, link.Link) } } if value, exist := data.Links.Patch[h]; exist { for _, link := range value.Links { links = append(links, link.Link) } } } if len(links) == 0 { return nil, fmt.Errorf("no download link found") } size := uint64(0) for _, file := range data.Files.Game { s, _ := utils.SizeToBytes(file.Size) size += s } item, err := db.GetGameItemByUrl(URL) if err != nil { return nil, err } item.Name = name item.RawName = name item.Download = strings.Join(links, ",") item.Url = URL item.Size = utils.BytesToSize(size) item.Author = "GOGGames" item.Platform = "windows" return item, nil } func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) { resp, err := utils.Request().Get(fmt.Sprintf(constant.GOGGamesURL, page)) if err != nil { return nil, err } data := searchResult{} err = json.Unmarshal(resp.Body(), &data) if err != nil { return nil, err } urls := make([]string, 0) var updateFlags []string //link+date for _, item := range data.Data { urls = append(urls, fmt.Sprintf(constant.GOGGamesPageURL, item.Slug)) updateFlags = append(updateFlags, fmt.Sprintf("%s%s", item.GogURL, item.LastUpdate)) } res := make([]*model.GameItem, 0) for i, u := range urls { if db.IsGameCrawled(updateFlags[i], "GOGGames") { continue } c.logger.Info("Crawling", zap.String("URL", u)) item, err := c.CrawlByUrl(u) if err != nil { c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u)) continue } item.UpdateFlag = updateFlags[i] if err := db.SaveGameItem(item); err != nil { c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u)) continue } res = append(res, item) if err := OrganizeGameItem(item); err != nil { c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u)) continue } } return res, nil } func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) { res := make([]*model.GameItem, 0) for _, page := range pages { items, err := c.Crawl(page) if err != nil { return nil, err } res = append(res, items...) } return res, nil } func (c *GOGGamesCrawler) CrawlAll() ([]*model.GameItem, error) { totalPageNum, err := c.GetTotalPageNum() if err != nil { return nil, err } var res []*model.GameItem for i := 1; i <= totalPageNum; i++ { items, err := c.Crawl(i) if err != nil { return nil, err } res = append(res, items...) } return res, nil } func (c *GOGGamesCrawler) GetTotalPageNum() (int, error) { resp, err := utils.Request().Get(fmt.Sprintf(constant.GOGGamesURL, 1)) if err != nil { return 0, err } data := searchResult{} err = json.Unmarshal(resp.Body(), &data) if err != nil { return 0, err } return data.Meta.LastPage, nil } type searchResult struct { Data []struct { ID string `json:"id"` Slug string `json:"slug"` Title string `json:"title"` Image string `json:"image"` Background string `json:"background"` GogURL string `json:"gog_url"` IsIndev bool `json:"is_indev"` IsNew bool `json:"is_new"` IsUpdated bool `json:"is_updated"` IsQueued bool `json:"is_queued"` IsUploading bool `json:"is_uploading"` VotedOn bool `json:"voted_on"` LastUpdate time.Time `json:"last_update"` Md5Filename string `json:"md5_filename"` Infohash string `json:"infohash"` IsVotable bool `json:"is_votable"` } `json:"data"` Links struct { First string `json:"first"` Last string `json:"last"` Prev any `json:"prev"` Next string `json:"next"` } `json:"links"` Meta struct { CurrentPage int `json:"current_page"` From int `json:"from"` LastPage int `json:"last_page"` Links []struct { URL any `json:"url"` Label string `json:"label"` Active bool `json:"active"` } `json:"links"` Path string `json:"path"` PerPage int `json:"per_page"` To int `json:"to"` Total int `json:"total"` } `json:"meta"` } type gameResult struct { ID string `json:"id"` Slug string `json:"slug"` Title string `json:"title"` Image string `json:"image"` Background string `json:"background"` GogURL string `json:"gog_url"` IsIndev bool `json:"is_indev"` IsNew bool `json:"is_new"` IsUpdated bool `json:"is_updated"` IsQueued bool `json:"is_queued"` IsUploading bool `json:"is_uploading"` VotedOn bool `json:"voted_on"` LastUpdate time.Time `json:"last_update"` Md5Filename string `json:"md5_filename"` Infohash string `json:"infohash"` Links struct { Goodie struct { OneFichier struct { ID string `json:"id"` Name string `json:"name"` Links []struct { Label string `json:"label"` Link string `json:"link"` } `json:"links"` } `json:"1fichier"` Vikingfile struct { ID string `json:"id"` Name string `json:"name"` Links []struct { Label string `json:"label"` Link string `json:"link"` } `json:"links"` } `json:"vikingfile"` Pixeldrain struct { ID string `json:"id"` Name string `json:"name"` Links []struct { Label string `json:"label"` Link string `json:"link"` } `json:"links"` } `json:"pixeldrain"` Gofile struct { ID string `json:"id"` Name string `json:"name"` Links []struct { Label string `json:"label"` Link string `json:"link"` } `json:"links"` } `json:"gofile"` } `json:"goodie"` Game map[string]struct { ID string `json:"id"` Name string `json:"name"` Links []struct { Label string `json:"label"` Link string `json:"link"` } `json:"links"` } `json:"game"` Patch map[string]struct { ID string `json:"id"` Name string `json:"name"` Links []struct { Label string `json:"label"` Link string `json:"link"` } `json:"links"` } `json:"patch"` } `json:"links"` Files struct { Game []struct { Name string `json:"name"` Type string `json:"type"` Size string `json:"size"` } `json:"game"` Goodie []struct { Name string `json:"name"` Type string `json:"type"` Size string `json:"size"` } `json:"goodie"` Patch []struct { Name string `json:"name"` Type string `json:"type"` Size string `json:"size"` } `json:"patch"` } `json:"files"` IsVotable bool `json:"is_votable"` }