package crawler import ( "bytes" "encoding/json" "errors" "fmt" "net/url" "regexp" "runtime/debug" "strconv" "strings" "sync" "time" "pcgamedb/cache" "pcgamedb/config" "pcgamedb/constant" "pcgamedb/db" "pcgamedb/model" "pcgamedb/utils" "github.com/PuerkitoBio/goquery" ) type twitchToken struct { Token string `json:"token"` Expires time.Time `json:"expires"` once sync.Once } var token = twitchToken{} func (t *twitchToken) getToken() (string, error) { t.once.Do(func() { if config.Config.RedisAvaliable { if dataBytes, exist := cache.Get("twitch_token"); exist { _ = json.Unmarshal([]byte(dataBytes), &token) } } }) if t.Token == "" || time.Now().After(t.Expires) { token, expires, err := loginTwitch() if err != nil { return "", fmt.Errorf("failed to login twitch: %w", err) } t.Token = token t.Expires = expires j, err := json.Marshal(t) if err == nil { _ = cache.Add("twitch_token", j) } } return t.Token, nil } func loginTwitch() (string, time.Time, error) { baseURL, _ := url.Parse(constant.TwitchAuthURL) params := url.Values{} params.Add("client_id", config.Config.Twitch.ClientID) params.Add("client_secret", config.Config.Twitch.ClientSecret) params.Add("grant_type", "client_credentials") baseURL.RawQuery = params.Encode() resp, err := utils.Fetch(utils.FetchConfig{ Url: baseURL.String(), Method: "POST", Headers: map[string]string{ "User-Agent": "", }, }) if err != nil { return "", time.Time{}, err } data := struct { AccessToken string `json:"access_token"` ExpiresIn int64 `json:"expires_in"` TokenType string `json:"token_type"` }{} err = json.Unmarshal(resp.Data, &data) if err != nil { return "", time.Time{}, err } return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil } func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) { t, err := token.getToken() if err != nil { return nil, err } resp, err := utils.Fetch(utils.FetchConfig{ Url: URL, Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + t, "User-Agent": "", "Content-Type": "text/plain", }, Data: dataBody, Method: "POST", }) if err != nil { return nil, err } return resp, nil } func getIGDBID(name string) (int, error) { var err error resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name)) if err != nil { return 0, err } if string(resp.Data) == "[]" { resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name)) if err != nil { return 0, err } } var data model.IGDBSearches if err = json.Unmarshal(resp.Data, &data); err != nil { return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) } if len(data) == 1 { return GetIGDBAppParentCache(data[0].Game) } maxSimilairty := 0.0 maxSimilairtyIndex := 0 for i, item := range data { if strings.EqualFold(item.Name, name) { return item.Game, nil } if sim := utils.Similarity(name, item.Name); sim >= 0.8 { if sim > maxSimilairty { maxSimilairty = sim maxSimilairtyIndex = i } } detail, err := GetIGDBAppDetailCache(item.Game) if err != nil { return 0, err } for _, alternativeNames := range detail.AlternativeNames { if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 { if sim > maxSimilairty { maxSimilairty = sim maxSimilairtyIndex = i } } } } if maxSimilairty >= 0.8 { return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game) } return 0, fmt.Errorf("IGDB ID not found: %s", name) } func getIGDBIDBySteamSearch(name string) (int, error) { baseURL, _ := url.Parse(constant.SteamSearchURL) params := url.Values{} params.Add("term", name) baseURL.RawQuery = params.Encode() resp, err := utils.Fetch(utils.FetchConfig{ Url: baseURL.String(), }) if err != nil { return 0, err } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data)) if err != nil { return 0, err } type searchResult struct { ID int Type string Name string } var items []searchResult doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) { if itemKey, exist := s.Attr("data-ds-itemkey"); exist { if strings.HasPrefix(itemKey, "App_") { id, err := strconv.Atoi(itemKey[4:]) if err != nil { return } name := s.Find(".title").Text() items = append(items, searchResult{ ID: id, Type: "App", Name: name, }) } if strings.HasPrefix(itemKey, "Bundle_") { id, err := strconv.Atoi(itemKey[7:]) if err != nil { return } name := s.Find(".title").Text() items = append(items, searchResult{ ID: id, Type: "Bundle", Name: name, }) } } }) maxSim := 0.0 var maxSimItem searchResult for _, item := range items { if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) { maxSimItem = item break } else { sim := utils.Similarity(item.Name, name) if sim >= 0.8 && sim > maxSim { maxSim = sim maxSimItem = item } } } if maxSim != 0 { if maxSimItem.Type == "App" { return GetIGDBIDBySteamAppIDCache(maxSimItem.ID) } if maxSimItem.Type == "Bundle" { return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID) } } return 0, fmt.Errorf("steam ID not found: %s", name) } // GetIGDBAppParent returns the parent of the game, if no parent return itself func GetIGDBAppParent(id int) (int, error) { detail, err := GetIGDBAppDetailCache(id) if err != nil { return 0, err } hasParent := false for detail.VersionParent != 0 { hasParent = true detail, err = GetIGDBAppDetailCache(detail.VersionParent) if err != nil { return 0, err } } if hasParent { return detail.ID, nil } return id, nil } func GetIGDBAppParetns(ids []int) (map[int]int, error) { var err error idsStr := make([]string, len(ids)) for i, id := range ids { idsStr[i] = strconv.Itoa(id) } resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ","))) if err != nil { return nil, err } var data []struct { ID int `json:"id"` VersionParent int `json:"version_parent"` } if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) } parents := make(map[int]int) for _, item := range data { if item.VersionParent != 0 { pid, err := GetIGDBAppParentCache(item.VersionParent) if err != nil { parents[item.ID] = item.ID } else { parents[item.ID] = pid } } else { parents[item.ID] = item.ID } } return parents, nil } func GetIGDBAppParentCache(id int) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_parent:%d", id) val, exist := cache.Get(key) if exist { id, err := strconv.Atoi(val) if err != nil { return 0, err } return id, nil } else { id, err := GetIGDBAppParent(id) if err != nil { return 0, err } _ = cache.Add(key, id) return id, nil } } return GetIGDBAppParent(id) } // GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search func GetIGDBID(name string) (int, error) { name1 := name name2 := FormatName(name) names := []string{name1} if name1 != name2 { names = append(names, name2) } for _, name := range names { id, err := getIGDBID(name) if err == nil { return id, nil } } for _, name := range names { id, err := getIGDBIDBySteamSearch(name) if err == nil { return id, nil } } return 0, errors.New("IGDB ID not found") } func GetIGDBIDCache(name string) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_id:%s", name) val, exist := cache.Get(key) if exist { id, err := strconv.Atoi(val) if err != nil { return 0, err } return id, nil } else { id, err := GetIGDBID(name) if err != nil { return 0, err } _ = cache.Add(key, id) return id, nil } } else { return GetIGDBID(name) } } func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { var err error resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id)) if err != nil { return nil, err } var data model.IGDBGameDetails if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } if len(data) == 0 { return nil, errors.New("IGDB App not found") } if data[0].Name == "" { return GetIGDBAppDetail(id) } return data[0], nil } func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_game:%v", id) val, exist := cache.Get(key) if exist { var data model.IGDBGameDetail if err := json.Unmarshal([]byte(val), &data); err != nil { return nil, err } return &data, nil } else { data, err := GetIGDBAppDetail(id) if err != nil { return nil, err } dataBytes, err := json.Marshal(data) if err != nil { return nil, err } _ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour) return data, nil } } else { return GetIGDBAppDetail(id) } } func GetIGDBCompany(id int) (string, error) { var err error resp, err := igdbFetch(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id)) if err != nil { return "", err } var data model.IGDBCompanies if err = json.Unmarshal(resp.Data, &data); err != nil { return "", err } if len(data) == 0 { return "", errors.New("not found") } if data[0].Name == "" { return GetIGDBCompany(id) } return data[0].Name, nil } func GetIGDBCompanyCache(id int) (string, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_companies:%v", id) val, exist := cache.Get(key) if exist { return val, nil } else { data, err := GetIGDBCompany(id) if err != nil { return "", err } _ = cache.Add(key, data) return data, nil } } else { return GetIGDBCompany(id) } } func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { item := &model.GameInfo{} detail, err := GetIGDBAppDetailCache(id) if err != nil { return nil, err } item.IGDBID = id item.Name = detail.Name item.Description = detail.Summary item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1) item.InfoUpdatedAt = time.Now() for _, lang := range detail.LanguageSupports { if lang.LanguageSupportType == 3 { l, exist := constant.IGDBLanguages[lang.Language] if !exist { continue } item.Languages = append(item.Languages, l.Name) } } for _, screenshot := range detail.Screenshots { item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1)) } for _, alias := range detail.AlternativeNames { item.Aliases = append(item.Aliases, alias.Name) } for _, company := range detail.InvolvedCompanies { if company.Developer || company.Publisher { companyName, err := GetIGDBCompanyCache(company.Company) if err != nil { continue } if company.Developer { item.Developers = append(item.Developers, companyName) } if company.Publisher { item.Publishers = append(item.Publishers, companyName) } } } return item, nil } // OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { id, err := GetIGDBIDCache(game.Name) if err != nil { return nil, err } d, err := db.GetGameInfoByPlatformID("igdb", id) if err == nil { d.GameIDs = append(d.GameIDs, game.ID) d.GameIDs = utils.Unique(d.GameIDs) return d, nil } info, err := GenerateGameInfo("igdb", id) if err != nil { return nil, err } info.GameIDs = append(info.GameIDs, game.ID) info.GameIDs = utils.Unique(info.GameIDs) return info, nil } func GetIGDBIDBySteamAppID(id int) (int, error) { var err error resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id)) if err != nil { return 0, err } var data []struct { Game int `json:"game"` } if err = json.Unmarshal(resp.Data, &data); err != nil { return 0, err } if len(data) == 0 { return 0, errors.New("not found") } if data[0].Game == 0 { return GetIGDBIDBySteamAppID(id) } return GetIGDBAppParentCache(data[0].Game) } func GetIGDBIDBySteamAppIDCache(id int) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id) val, exist := cache.Get(key) if exist { return strconv.Atoi(val) } else { data, err := GetIGDBIDBySteamAppID(id) if err != nil { return 0, err } _ = cache.Add(key, strconv.Itoa(data)) return data, nil } } else { return GetIGDBIDBySteamAppID(id) } } func GetIGDBIDBySteamBundleID(id int) (int, error) { var err error resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id)) if err != nil { return 0, err } var data []struct { Game int `json:"game"` } if err = json.Unmarshal(resp.Data, &data); err != nil { return 0, err } if len(data) == 0 { return 0, errors.New("not found") } if data[0].Game == 0 { return GetIGDBIDBySteamBundleID(id) } return GetIGDBAppParentCache(data[0].Game) } func GetIGDBIDBySteamBundleIDCache(id int) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id) val, exist := cache.Get(key) if exist { return strconv.Atoi(val) } else { data, err := GetIGDBIDBySteamBundleID(id) if err != nil { return 0, err } _ = cache.Add(key, strconv.Itoa(data)) return data, nil } } else { return GetIGDBIDBySteamBundleID(id) } } func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) { var err error conditionBuilder := strings.Builder{} for _, id := range ids { conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id)) conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id)) } condition := strings.TrimSuffix(conditionBuilder.String(), " | ") respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition) resp, err := igdbFetch(constant.IGDBWebsitesURL, respBody) if err != nil { return nil, err } var data []struct { Game int `json:"game"` Url string `json:"url"` } if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } ret := make(map[int]int) regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`) for _, d := range data { idStr := regex.FindStringSubmatch(d.Url) if len(idStr) < 2 { continue } id, err := strconv.Atoi(idStr[1]) if err == nil { pid, err := GetIGDBAppParentCache(d.Game) if err == nil { ret[id] = pid } else { ret[id] = 0 } } } for _, id := range ids { if _, ok := ret[id]; !ok { ret[id] = 0 } } return ret, nil } func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) { res := make(map[int]int) notExistIDs := make([]int, 0) if config.Config.RedisAvaliable { for _, steamID := range ids { key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID) val, exist := cache.Get(key) if exist { igdbID, _ := strconv.Atoi(val) res[steamID] = igdbID } else { notExistIDs = append(notExistIDs, steamID) } } if len(res) == len(ids) { return res, nil } idMap, err := GetIGDBIDsBySteamIDs(notExistIDs) if err != nil { return nil, err } for steamID, igdbID := range idMap { res[steamID] = igdbID if igdbID != 0 { _ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID) } } return res, nil } else { return GetIGDBIDsBySteamIDs(ids) } } // GetIGDBPopularGameIDs get IGDB popular game IDs // popularity_type = 1 IGDB Visits: Game page visits on IGDB.com. // popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists. // popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists. // popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists. func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) { var err error resp, err := igdbFetch(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType)) if err != nil { return nil, err } type IgdbPopularity struct { GameID int `json:"game_id"` Value float64 `json:"value"` } var data []IgdbPopularity if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } ret := make([]int, 0) for _, d := range data { pid, err := GetIGDBAppParentCache(d.GameID) if err != nil { ret = append(ret, d.GameID) continue } ret = append(ret, pid) } return ret, nil }