pcgamedb/crawler/igdb.go

490 lines
12 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package crawler
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/url"
"runtime/debug"
"strconv"
"strings"
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/constant"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"github.com/PuerkitoBio/goquery"
"github.com/go-resty/resty/v2"
)
type twitchToken struct {
}
var token = twitchToken{}
func (t *twitchToken) getToken() (string, error) {
if val, exist := cache.Get("twitch_token"); exist {
return val, nil
}
token, expires, err := loginTwitch()
if err != nil {
return "", fmt.Errorf("failed to login twitch: %w", err)
}
_ = cache.SetWithExpire("twitch_token", token, expires)
return token, nil
}
func loginTwitch() (string, time.Duration, error) {
baseURL, _ := url.Parse(constant.TwitchAuthURL)
params := url.Values{}
params.Add("client_id", config.Config.Twitch.ClientID)
params.Add("client_secret", config.Config.Twitch.ClientSecret)
params.Add("grant_type", "client_credentials")
baseURL.RawQuery = params.Encode()
resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
if err != nil {
return "", 0, err
}
data := struct {
AccessToken string `json:"access_token"`
ExpiresIn int64 `json:"expires_in"`
TokenType string `json:"token_type"`
}{}
err = json.Unmarshal(resp.Body(), &data)
if err != nil {
return "", 0, err
}
return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
}
func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
t, err := token.getToken()
if err != nil {
return nil, err
}
resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{
"Client-ID": config.Config.Twitch.ClientID,
"Authorization": "Bearer " + t,
"User-Agent": "",
"Content-Type": "text/plain",
}).Post(URL)
if err != nil {
return nil, err
}
return resp, nil
}
func getIGDBID(name string) (int, error) {
var err error
resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
if err != nil {
return 0, err
}
if string(resp.Body()) == "[]" {
resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
if err != nil {
return 0, err
}
}
var data model.IGDBSearches
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
}
if len(data) == 1 {
return GetIGDBAppParent(data[0].Game)
}
maxSimilairty := 0.0
maxSimilairtyIndex := 0
for i, item := range data {
if strings.EqualFold(item.Name, name) {
return item.Game, nil
}
if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
if sim > maxSimilairty {
maxSimilairty = sim
maxSimilairtyIndex = i
}
}
detail, err := GetIGDBAppDetail(item.Game)
if err != nil {
return 0, err
}
for _, alternativeNames := range detail.AlternativeNames {
if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 {
if sim > maxSimilairty {
maxSimilairty = sim
maxSimilairtyIndex = i
}
}
}
}
if maxSimilairty >= 0.8 {
return GetIGDBAppParent(data[maxSimilairtyIndex].Game)
}
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
func getIGDBIDBySteamSearch(name string) (int, error) {
baseURL, _ := url.Parse(constant.SteamSearchURL)
params := url.Values{}
params.Add("term", name)
baseURL.RawQuery = params.Encode()
resp, err := utils.Request().Get(baseURL.String())
if err != nil {
return 0, err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
if err != nil {
return 0, err
}
type searchResult struct {
ID int
Type string
Name string
}
var items []searchResult
doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
if strings.HasPrefix(itemKey, "App_") {
id, err := strconv.Atoi(itemKey[4:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "App",
Name: name,
})
}
if strings.HasPrefix(itemKey, "Bundle_") {
id, err := strconv.Atoi(itemKey[7:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "Bundle",
Name: name,
})
}
}
})
maxSim := 0.0
var maxSimItem searchResult
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
maxSimItem = item
break
} else {
sim := utils.Similarity(item.Name, name)
if sim >= 0.8 && sim > maxSim {
maxSim = sim
maxSimItem = item
}
}
}
if maxSim != 0 {
if maxSimItem.Type == "App" {
return GetIGDBIDBySteamAppID(maxSimItem.ID)
}
if maxSimItem.Type == "Bundle" {
return GetIGDBIDBySteamBundleID(maxSimItem.ID)
}
}
return 0, fmt.Errorf("steam ID not found: %s", name)
}
// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
key := fmt.Sprintf("igdb_parent:%d", id)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
}
detail, err := GetIGDBAppDetail(id)
if err != nil {
return 0, err
}
hasParent := false
for detail.VersionParent != 0 {
hasParent = true
detail, err = GetIGDBAppDetail(detail.VersionParent)
if err != nil {
return 0, err
}
}
if hasParent {
return detail.ID, nil
}
_ = cache.Set(key, id)
return id, nil
}
// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
func GetIGDBID(name string) (int, error) {
key := fmt.Sprintf("igdb_id:%s", name)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
}
name1 := name
name2 := FormatName(name)
names := []string{name1}
if name1 != name2 {
names = append(names, name2)
}
for _, name := range names {
id, err := getIGDBID(name)
if err == nil {
_ = cache.Set(key, id)
return id, nil
}
}
for _, name := range names {
id, err := getIGDBIDBySteamSearch(name)
if err == nil {
_ = cache.Set(key, id)
return id, nil
}
}
return 0, errors.New("IGDB ID not found")
}
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
key := fmt.Sprintf("igdb_game:%v", id)
val, exist := cache.Get(key)
if exist {
var data model.IGDBGameDetail
if err := json.Unmarshal([]byte(val), &data); err != nil {
return nil, err
}
return &data, nil
}
var err error
resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))
if err != nil {
return nil, err
}
var data model.IGDBGameDetails
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return nil, err
}
if len(data) == 0 {
return nil, errors.New("IGDB App not found")
}
if data[0].Name == "" {
return GetIGDBAppDetail(id)
}
jsonBytes, err := json.Marshal(data[0])
if err == nil {
_ = cache.Set(key, string(jsonBytes))
}
return data[0], nil
}
func GetIGDBCompany(id int) (string, error) {
key := fmt.Sprintf("igdb_companies:%v", id)
val, exist := cache.Get(key)
if exist {
return val, nil
}
var err error
resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
if err != nil {
return "", err
}
var data model.IGDBCompanies
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return "", err
}
if len(data) == 0 {
return "", errors.New("not found")
}
if data[0].Name == "" {
return GetIGDBCompany(id)
}
_ = cache.Set(key, data[0].Name)
return data[0].Name, nil
}
func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
item := &model.GameInfo{}
detail, err := GetIGDBAppDetail(id)
if err != nil {
return nil, err
}
item.IGDBID = id
item.Name = detail.Name
item.Description = detail.Summary
item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)
item.InfoUpdatedAt = time.Now()
for _, lang := range detail.LanguageSupports {
if lang.LanguageSupportType == 3 {
l, exist := constant.IGDBLanguages[lang.Language]
if !exist {
continue
}
item.Languages = append(item.Languages, l.Name)
}
}
for _, screenshot := range detail.Screenshots {
item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1))
}
for _, alias := range detail.AlternativeNames {
item.Aliases = append(item.Aliases, alias.Name)
}
for _, company := range detail.InvolvedCompanies {
if company.Developer || company.Publisher {
companyName, err := GetIGDBCompany(company.Company)
if err != nil {
continue
}
if company.Developer {
item.Developers = append(item.Developers, companyName)
}
if company.Publisher {
item.Publishers = append(item.Publishers, companyName)
}
}
}
return item, nil
}
// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
id, err := GetIGDBID(game.Name)
if err != nil {
return nil, err
}
d, err := db.GetGameInfoByPlatformID("igdb", id)
if err == nil {
d.GameIDs = append(d.GameIDs, game.ID)
d.GameIDs = utils.Unique(d.GameIDs)
return d, nil
}
info, err := GenerateGameInfo("igdb", id)
if err != nil {
return nil, err
}
info.GameIDs = append(info.GameIDs, game.ID)
info.GameIDs = utils.Unique(info.GameIDs)
return info, nil
}
func GetIGDBIDBySteamAppID(id int) (int, error) {
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
}
var err error
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
return 0, err
}
var data []struct {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return 0, err
}
if len(data) == 0 {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamAppID(id)
}
_ = cache.Set(key, strconv.Itoa(data[0].Game))
return GetIGDBAppParent(data[0].Game)
}
func GetIGDBIDBySteamBundleID(id int) (int, error) {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
}
var err error
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
return 0, err
}
var data []struct {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return 0, err
}
if len(data) == 0 {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamBundleID(id)
}
_ = cache.Set(key, strconv.Itoa(data[0].Game))
return GetIGDBAppParent(data[0].Game)
}
// GetIGDBPopularGameIDs get IGDB popular game IDs
// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users “Want to Play” lists.
// popularity_type = 3 IGDB Playing: Additions to IGDB.com users “Playing” lists.
// popularity_type = 4 IGDB Played: Additions to IGDB.com users “Played” lists.
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
var err error
resp, err := igdbRequest(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType))
if err != nil {
return nil, err
}
type IgdbPopularity struct {
GameID int `json:"game_id"`
Value float64 `json:"value"`
}
var data []IgdbPopularity
if err = json.Unmarshal(resp.Body(), &data); err != nil {
return nil, err
}
ret := make([]int, 0)
for _, d := range data {
pid, err := GetIGDBAppParent(d.GameID)
if err != nil {
ret = append(ret, d.GameID)
continue
}
ret = append(ret, pid)
}
return ret, nil
}