package crawler import ( "bytes" "encoding/json" "errors" "fmt" "net/url" "runtime/debug" "strconv" "strings" "time" "pcgamedb/cache" "pcgamedb/config" "pcgamedb/constant" "pcgamedb/db" "pcgamedb/model" "pcgamedb/utils" "github.com/PuerkitoBio/goquery" "github.com/go-resty/resty/v2" ) type twitchToken struct { } var token = twitchToken{} func (t *twitchToken) getToken() (string, error) { if val, exist := cache.Get("twitch_token"); exist { return val, nil } token, expires, err := loginTwitch() if err != nil { return "", fmt.Errorf("failed to login twitch: %w", err) } _ = cache.SetWithExpire("twitch_token", token, expires) return token, nil } func loginTwitch() (string, time.Duration, error) { baseURL, _ := url.Parse(constant.TwitchAuthURL) params := url.Values{} params.Add("client_id", config.Config.Twitch.ClientID) params.Add("client_secret", config.Config.Twitch.ClientSecret) params.Add("grant_type", "client_credentials") baseURL.RawQuery = params.Encode() resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String()) if err != nil { return "", 0, err } data := struct { AccessToken string `json:"access_token"` ExpiresIn int64 `json:"expires_in"` TokenType string `json:"token_type"` }{} err = json.Unmarshal(resp.Body(), &data) if err != nil { return "", 0, err } return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil } func igdbRequest(URL string, dataBody any) (*resty.Response, error) { t, err := token.getToken() if err != nil { return nil, err } resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{ "Client-ID": config.Config.Twitch.ClientID, "Authorization": "Bearer " + t, "User-Agent": "", "Content-Type": "text/plain", }).Post(URL) if err != nil { return nil, err } return resp, nil } func getIGDBID(name string) (int, error) { var err error resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name)) if err != nil { return 0, err } if string(resp.Body()) == "[]" { resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name)) if err != nil { return 0, err } } var data model.IGDBSearches if err = json.Unmarshal(resp.Body(), &data); err != nil { return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) } if len(data) == 1 { return GetIGDBAppParent(data[0].Game) } maxSimilairty := 0.0 maxSimilairtyIndex := 0 for i, item := range data { if strings.EqualFold(item.Name, name) { return item.Game, nil } if sim := utils.Similarity(name, item.Name); sim >= 0.8 { if sim > maxSimilairty { maxSimilairty = sim maxSimilairtyIndex = i } } detail, err := GetIGDBAppDetail(item.Game) if err != nil { return 0, err } for _, alternativeNames := range detail.AlternativeNames { if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 { if sim > maxSimilairty { maxSimilairty = sim maxSimilairtyIndex = i } } } } if maxSimilairty >= 0.8 { return GetIGDBAppParent(data[maxSimilairtyIndex].Game) } return 0, fmt.Errorf("IGDB ID not found: %s", name) } func getIGDBIDBySteamSearch(name string) (int, error) { baseURL, _ := url.Parse(constant.SteamSearchURL) params := url.Values{} params.Add("term", name) baseURL.RawQuery = params.Encode() resp, err := utils.Request().Get(baseURL.String()) if err != nil { return 0, err } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) if err != nil { return 0, err } type searchResult struct { ID int Type string Name string } var items []searchResult doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) { if itemKey, exist := s.Attr("data-ds-itemkey"); exist { if strings.HasPrefix(itemKey, "App_") { id, err := strconv.Atoi(itemKey[4:]) if err != nil { return } name := s.Find(".title").Text() items = append(items, searchResult{ ID: id, Type: "App", Name: name, }) } if strings.HasPrefix(itemKey, "Bundle_") { id, err := strconv.Atoi(itemKey[7:]) if err != nil { return } name := s.Find(".title").Text() items = append(items, searchResult{ ID: id, Type: "Bundle", Name: name, }) } } }) maxSim := 0.0 var maxSimItem searchResult for _, item := range items { if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) { maxSimItem = item break } else { sim := utils.Similarity(item.Name, name) if sim >= 0.8 && sim > maxSim { maxSim = sim maxSimItem = item } } } if maxSim != 0 { if maxSimItem.Type == "App" { return GetIGDBIDBySteamAppID(maxSimItem.ID) } if maxSimItem.Type == "Bundle" { return GetIGDBIDBySteamBundleID(maxSimItem.ID) } } return 0, fmt.Errorf("steam ID not found: %s", name) } // GetIGDBAppParent returns the parent of the game, if no parent return itself func GetIGDBAppParent(id int) (int, error) { key := fmt.Sprintf("igdb_parent:%d", id) val, exist := cache.Get(key) if exist { id, err := strconv.Atoi(val) if err != nil { return 0, err } return id, nil } detail, err := GetIGDBAppDetail(id) if err != nil { return 0, err } hasParent := false for detail.VersionParent != 0 { hasParent = true detail, err = GetIGDBAppDetail(detail.VersionParent) if err != nil { return 0, err } } if hasParent { return detail.ID, nil } _ = cache.Set(key, id) return id, nil } // GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search func GetIGDBID(name string) (int, error) { key := fmt.Sprintf("igdb_id:%s", name) val, exist := cache.Get(key) if exist { return strconv.Atoi(val) } name1 := name name2 := FormatName(name) names := []string{name1} if name1 != name2 { names = append(names, name2) } for _, name := range names { id, err := getIGDBID(name) if err == nil { _ = cache.Set(key, id) return id, nil } } for _, name := range names { id, err := getIGDBIDBySteamSearch(name) if err == nil { _ = cache.Set(key, id) return id, nil } } return 0, errors.New("IGDB ID not found") } func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { key := fmt.Sprintf("igdb_game:%v", id) val, exist := cache.Get(key) if exist { var data model.IGDBGameDetail if err := json.Unmarshal([]byte(val), &data); err != nil { return nil, err } return &data, nil } var err error resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id = %v;fields *,alternative_names.*,language_supports.*,screenshots.*,cover.*,involved_companies.*,involved_companies.*,game_engines.*,game_modes.*,genres.*,player_perspectives.*,release_dates.*,videos.*,websites.*,platforms.*,themes.*,collections.*;`, id)) if err != nil { return nil, err } var data model.IGDBGameDetails if err = json.Unmarshal(resp.Body(), &data); err != nil { return nil, err } if len(data) == 0 { return nil, errors.New("IGDB App not found") } if data[0].Name == "" { return GetIGDBAppDetail(id) } jsonBytes, err := json.Marshal(data[0]) if err == nil { _ = cache.Set(key, string(jsonBytes)) } return data[0], nil } func GetIGDBCompany(id int) (string, error) { key := fmt.Sprintf("igdb_companies:%v", id) val, exist := cache.Get(key) if exist { return val, nil } var err error resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id)) if err != nil { return "", err } var data model.IGDBCompanies if err = json.Unmarshal(resp.Body(), &data); err != nil { return "", err } if len(data) == 0 { return "", errors.New("not found") } if data[0].Name == "" { return GetIGDBCompany(id) } _ = cache.Set(key, data[0].Name) return data[0].Name, nil } func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { item := &model.GameInfo{} detail, err := GetIGDBAppDetail(id) if err != nil { return nil, err } item.IGDBID = id item.Name = detail.Name item.Description = detail.Summary item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1) for _, lang := range detail.LanguageSupports { if lang.LanguageSupportType == 3 { l, exist := constant.IGDBLanguages[lang.Language] if !exist { continue } item.Languages = append(item.Languages, l.Name) } } for _, screenshot := range detail.Screenshots { item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1)) } for _, alias := range detail.AlternativeNames { item.Aliases = append(item.Aliases, alias.Name) } for _, company := range detail.InvolvedCompanies { if company.Developer || company.Publisher { companyName, err := GetIGDBCompany(company.Company) if err != nil { continue } if company.Developer { item.Developers = append(item.Developers, companyName) } if company.Publisher { item.Publishers = append(item.Publishers, companyName) } } } item.GameEngines = make([]string, 0) for _, engine := range detail.GameEngines { item.GameEngines = append(item.GameEngines, engine.Name) } item.GameModes = make([]string, 0) for _, mode := range detail.GameModes { item.GameModes = append(item.GameModes, mode.Name) } item.Genres = make([]string, 0) for _, genre := range detail.Genres { item.Genres = append(item.Genres, genre.Name) } item.Themes = make([]string, 0) for _, theme := range detail.Themes { item.Themes = append(item.Themes, theme.Name) } item.Platforms = make([]string, 0) for _, platform := range detail.Platforms { item.Platforms = append(item.Platforms, platform.Name) } item.PlayerPerspectives = make([]string, 0) for _, perspective := range detail.PlayerPerspectives { item.PlayerPerspectives = append(item.PlayerPerspectives, perspective.Name) } item.SimilarGames = detail.SimilarGames item.Videos = make([]string, 0) for _, video := range detail.Videos { item.Videos = append(item.Videos, fmt.Sprintf("https://www.youtube.com/watch?v=%s", video.VideoID)) } item.Websites = make([]string, 0) for _, website := range detail.Websites { item.Websites = append(item.Websites, website.URL) } item.Collections = make([]model.GameCollection, 0) for _, collection := range detail.Collections { item.Collections = append(item.Collections, model.GameCollection{ Games: collection.Games, Name: collection.Name, }) } return item, nil } // OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { id, err := GetIGDBID(game.Name) if err != nil { return nil, err } d, err := db.GetGameInfoByPlatformID("igdb", id) if err == nil { d.GameIDs = append(d.GameIDs, game.ID) d.GameIDs = utils.Unique(d.GameIDs) return d, nil } info, err := GenerateGameInfo("igdb", id) if err != nil { return nil, err } info.GameIDs = append(info.GameIDs, game.ID) info.GameIDs = utils.Unique(info.GameIDs) return info, nil } func GetIGDBIDBySteamAppID(id int) (int, error) { key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id) val, exist := cache.Get(key) if exist { return strconv.Atoi(val) } var err error resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id)) if err != nil { return 0, err } var data []struct { Game int `json:"game"` } if err = json.Unmarshal(resp.Body(), &data); err != nil { return 0, err } if len(data) == 0 { return 0, errors.New("not found") } if data[0].Game == 0 { return GetIGDBIDBySteamAppID(id) } _ = cache.Set(key, strconv.Itoa(data[0].Game)) return GetIGDBAppParent(data[0].Game) } func GetIGDBIDBySteamBundleID(id int) (int, error) { key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id) val, exist := cache.Get(key) if exist { return strconv.Atoi(val) } var err error resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id)) if err != nil { return 0, err } var data []struct { Game int `json:"game"` } if err = json.Unmarshal(resp.Body(), &data); err != nil { return 0, err } if len(data) == 0 { return 0, errors.New("not found") } if data[0].Game == 0 { return GetIGDBIDBySteamBundleID(id) } _ = cache.Set(key, strconv.Itoa(data[0].Game)) return GetIGDBAppParent(data[0].Game) } // GetIGDBPopularGameIDs get IGDB popular game IDs // popularity_type = 1 IGDB Visits: Game page visits on IGDB.com. // popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists. // popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists. // popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists. func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) { var err error resp, err := igdbRequest(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType)) if err != nil { return nil, err } type IgdbPopularity struct { GameID int `json:"game_id"` Value float64 `json:"value"` } var data []IgdbPopularity if err = json.Unmarshal(resp.Body(), &data); err != nil { return nil, err } ret := make([]int, 0) for _, d := range data { pid, err := GetIGDBAppParent(d.GameID) if err != nil { ret = append(ret, d.GameID) continue } ret = append(ret, pid) } return ret, nil }