package crawler import ( "encoding/json" "errors" "fmt" "net/url" "regexp" "runtime/debug" "strconv" "strings" "pcgamedb/cache" "pcgamedb/config" "pcgamedb/constant" "pcgamedb/db" "pcgamedb/model" "pcgamedb/utils" ) var TwitchToken string func getIGDBID(name string) (int, error) { var err error if TwitchToken == "" { TwitchToken, err = LoginTwitch() if err != nil { return 0, fmt.Errorf("failed to login twitch: %w", err) } } resp, err := utils.Fetch(utils.FetchConfig{ Url: constant.IGDBSearchURL, Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name), Method: "POST", }) if string(resp.Data) == "[]" { resp, err = utils.Fetch(utils.FetchConfig{ Url: constant.IGDBSearchURL, Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: fmt.Sprintf(`search "%s"; fields *; limit 50;`, name), Method: "POST", }) } if err != nil { return 0, err } var data model.IGDBSearches if err = json.Unmarshal(resp.Data, &data); err != nil { return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) } if len(data) == 1 { return data[0].Game, nil } maxSimilairty := 0.0 maxSimilairtyIndex := 0 for i, item := range data { if strings.EqualFold(item.Name, name) { return item.Game, nil } if sim := utils.Similarity(name, item.Name); sim >= 0.8 { if sim > maxSimilairty { maxSimilairty = sim maxSimilairtyIndex = i } } detail, err := GetIGDBAppDetailCache(item.Game) if err != nil { return 0, err } for _, alternativeNames := range detail.AlternativeNames { if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 { if sim > maxSimilairty { maxSimilairty = sim maxSimilairtyIndex = i } } } } if maxSimilairty >= 0.8 { return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game) } return 0, fmt.Errorf("IGDB ID not found: %s", name) } // GetIGDBAppParent returns the parent of the game, if no parent return itself func GetIGDBAppParent(id int) (int, error) { detail, err := GetIGDBAppDetailCache(id) if err != nil { return 0, err } versionParent := detail.VersionParent for versionParent != 0 { detail, err = GetIGDBAppDetailCache(versionParent) if err != nil { return 0, err } versionParent = detail.VersionParent } if versionParent != 0 { return versionParent, nil } return id, nil } func GetIGDBAppParentCache(id int) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_parent:%d", id) val, exist := cache.Get(key) if exist { id, err := strconv.Atoi(val) if err != nil { return 0, err } return id, nil } else { id, err := GetIGDBAppParent(id) if err != nil { return 0, err } _ = cache.Add(key, id) return id, nil } } return GetIGDBAppParent(id) } // GetIGDBID returns the IGDB ID of the game, try raw name first then formated names func GetIGDBID(name string) (int, error) { name1 := name name2 := FormatName(name) names := []string{name1} if name1 != name2 { names = append(names, name2) } for _, name := range names { id, err := getIGDBID(name) if err == nil { return id, nil } } return 0, errors.New("IGDB ID not found") } func GetIGDBIDCache(name string) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_id:%s", name) val, exist := cache.Get(key) if exist { id, err := strconv.Atoi(val) if err != nil { return 0, err } return id, nil } else { id, err := GetIGDBID(name) if err != nil { return 0, err } _ = cache.Add(key, id) return id, nil } } else { return GetIGDBID(name) } } func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { var err error if TwitchToken == "" { TwitchToken, err = LoginTwitch() if err != nil { return nil, err } } resp, err := utils.Fetch(utils.FetchConfig{ Url: constant.IGDBGameURL, Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id), Method: "POST", }) if err != nil { return nil, err } var data model.IGDBGameDetails if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } if len(data) == 0 { return nil, errors.New("IGDB App not found") } if data[0].Name == "" { return GetIGDBAppDetail(id) } return data[0], nil } func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_game:%v", id) val, exist := cache.Get(key) if exist { var data model.IGDBGameDetail if err := json.Unmarshal([]byte(val), &data); err != nil { return nil, err } return &data, nil } else { data, err := GetIGDBAppDetail(id) if err != nil { return nil, err } dataBytes, err := json.Marshal(data) if err != nil { return nil, err } _ = cache.Add(key, dataBytes) return data, nil } } else { return GetIGDBAppDetail(id) } } func LoginTwitch() (string, error) { baseURL, _ := url.Parse(constant.TwitchAuthURL) params := url.Values{} params.Add("client_id", config.Config.Twitch.ClientID) params.Add("client_secret", config.Config.Twitch.ClientSecret) params.Add("grant_type", "client_credentials") baseURL.RawQuery = params.Encode() resp, err := utils.Fetch(utils.FetchConfig{ Url: baseURL.String(), Method: "POST", Headers: map[string]string{ "User-Agent": "", }, }) if err != nil { return "", err } data := struct { AccessToken string `json:"access_token"` }{} err = json.Unmarshal(resp.Data, &data) if err != nil { return "", err } return data.AccessToken, nil } func GetIGDBCompany(id int) (string, error) { var err error if TwitchToken == "" { TwitchToken, err = LoginTwitch() if err != nil { return "", err } } resp, err := utils.Fetch(utils.FetchConfig{ Url: constant.IGDBCompaniesURL, Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: fmt.Sprintf(`where id=%v; fields *;`, id), Method: "POST", }) if err != nil { return "", err } var data model.IGDBCompanies if err = json.Unmarshal(resp.Data, &data); err != nil { return "", err } if len(data) == 0 { return "", errors.New("not found") } if data[0].Name == "" { return GetIGDBCompany(id) } return data[0].Name, nil } func GetIGDBCompanyCache(id int) (string, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_companies:%v", id) val, exist := cache.Get(key) if exist { return val, nil } else { data, err := GetIGDBCompany(id) if err != nil { return "", err } _ = cache.Add(key, data) return data, nil } } else { return GetIGDBCompany(id) } } func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { item := &model.GameInfo{} detail, err := GetIGDBAppDetailCache(id) if err != nil { return nil, err } item.IGDBID = id item.Name = detail.Name item.Description = detail.Summary item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1) for _, lang := range detail.LanguageSupports { if lang.LanguageSupportType == 3 { l, exist := constant.IGDBLanguages[lang.Language] if !exist { continue } item.Languages = append(item.Languages, l.Name) } } for _, screenshot := range detail.Screenshots { item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1)) } for _, alias := range detail.AlternativeNames { item.Aliases = append(item.Aliases, alias.Name) } for _, company := range detail.InvolvedCompanies { if company.Developer || company.Publisher { companyName, err := GetIGDBCompanyCache(company.Company) if err != nil { continue } if company.Developer { item.Developers = append(item.Developers, companyName) } if company.Publisher { item.Publishers = append(item.Publishers, companyName) } } } return item, nil } // OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) { var err error if id == 0 { id, err = GetIGDBIDCache(game.Name) if err != nil { return nil, err } } d, err := db.GetGameInfoByPlatformID("igdb", id) if err == nil { d.GameIDs = append(d.GameIDs, game.ID) d.GameIDs = utils.Unique(d.GameIDs) return d, nil } info, err := GenerateGameInfo("igdb", id) if err != nil { return nil, err } info.GameIDs = append(info.GameIDs, game.ID) info.GameIDs = utils.Unique(info.GameIDs) return info, nil } func GetIGDBIDBySteamID(id int) (int, error) { var err error if TwitchToken == "" { TwitchToken, err = LoginTwitch() if err != nil { return 0, err } } resp, err := utils.Fetch(utils.FetchConfig{ Url: constant.IGDBWebsitesURL, Method: "POST", Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id), }) if err != nil { return 0, err } var data []struct { Game int `json:"game"` } if err = json.Unmarshal(resp.Data, &data); err != nil { return 0, err } if len(data) == 0 { return 0, errors.New("not found") } if data[0].Game == 0 { return GetIGDBIDBySteamID(id) } return GetIGDBAppParentCache(data[0].Game) } func GetIGDBIDBySteamIDCache(id int) (int, error) { if config.Config.RedisAvaliable { key := fmt.Sprintf("igdb_id_by_steam_id:%v", id) val, exist := cache.Get(key) if exist { return strconv.Atoi(val) } else { data, err := GetIGDBIDBySteamID(id) if err != nil { return 0, err } _ = cache.Add(key, strconv.Itoa(data)) return data, nil } } else { return GetIGDBIDBySteamID(id) } } func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) { var err error if TwitchToken == "" { TwitchToken, err = LoginTwitch() if err != nil { return nil, err } } conditionBuilder := strings.Builder{} for _, id := range ids { conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id)) conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id)) } condition := strings.TrimSuffix(conditionBuilder.String(), " | ") respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition) resp, err := utils.Fetch(utils.FetchConfig{ Url: constant.IGDBWebsitesURL, Method: "POST", Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: respBody, }) if err != nil { return nil, err } var data []struct { Game int `json:"game"` Url string `json:"url"` } if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } ret := make(map[int]int) regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`) for _, d := range data { idStr := regex.FindStringSubmatch(d.Url) if len(idStr) < 2 { continue } id, err := strconv.Atoi(idStr[1]) if err == nil { pid, err := GetIGDBAppParentCache(d.Game) if err == nil { ret[id] = pid } else { ret[id] = 0 } } } for _, id := range ids { if _, ok := ret[id]; !ok { ret[id] = 0 } } return ret, nil } func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) { res := make(map[int]int) notExistIDs := make([]int, 0) if config.Config.RedisAvaliable { for _, steamID := range ids { key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID) val, exist := cache.Get(key) if exist { igdbID, _ := strconv.Atoi(val) res[steamID] = igdbID } else { notExistIDs = append(notExistIDs, steamID) } } if len(res) == len(ids) { return res, nil } idMap, err := GetIGDBIDsBySteamIDs(notExistIDs) if err != nil { return nil, err } for steamID, igdbID := range idMap { res[steamID] = igdbID if igdbID != 0 { _ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID) } } return res, nil } else { return GetIGDBIDsBySteamIDs(ids) } } // GetIGDBPopularGameIDs get IGDB popular game IDs // popularity_type = 1 IGDB Visits: Game page visits on IGDB.com. // popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists. // popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists. // popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists. func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) { var err error if TwitchToken == "" { TwitchToken, err = LoginTwitch() if err != nil { return nil, err } } resp, err := utils.Fetch(utils.FetchConfig{ Url: constant.IGDBPopularityURL, Method: "POST", Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + TwitchToken, "User-Agent": "", "Content-Type": "text/plain", }, Data: fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType), }) if err != nil { return nil, err } type IGDBPopularity struct { GameID int `json:"game_id"` Value float64 `json:"value"` } var data []IGDBPopularity if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } ret := make([]int, 0) for _, d := range data { pid, err := GetIGDBAppParentCache(d.GameID) if err != nil { ret = append(ret, d.GameID) continue } ret = append(ret, pid) } return ret, nil }