pcgamedb/crawler/igdb.go
nite07 8702d3e93f fix: OrganizeGameItem
fix: save twitchtoken
remove: getSteamID
remove: update cmd&api
add: getIGDBIDBySteamSearch
add: update game info task
2024-11-28 18:37:01 +08:00

700 lines
17 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package crawler
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/url"
"regexp"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/constant"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"github.com/PuerkitoBio/goquery"
)
type twitchToken struct {
Token string `json:"token"`
Expires time.Time `json:"expires"`
once sync.Once
}
var token = twitchToken{}
func (t *twitchToken) getToken() (string, error) {
t.once.Do(func() {
if config.Config.RedisAvaliable {
if dataBytes, exist := cache.Get("twitch_token"); exist {
_ = json.Unmarshal([]byte(dataBytes), &token)
}
}
})
if t.Token == "" || time.Now().After(t.Expires) {
token, expires, err := loginTwitch()
if err != nil {
return "", fmt.Errorf("failed to login twitch: %w", err)
}
t.Token = token
t.Expires = expires
j, err := json.Marshal(t)
if err == nil {
_ = cache.Add("twitch_token", j)
}
}
return t.Token, nil
}
func loginTwitch() (string, time.Time, error) {
baseURL, _ := url.Parse(constant.TwitchAuthURL)
params := url.Values{}
params.Add("client_id", config.Config.Twitch.ClientID)
params.Add("client_secret", config.Config.Twitch.ClientSecret)
params.Add("grant_type", "client_credentials")
baseURL.RawQuery = params.Encode()
resp, err := utils.Fetch(utils.FetchConfig{
Url: baseURL.String(),
Method: "POST",
Headers: map[string]string{
"User-Agent": "",
},
})
if err != nil {
return "", time.Time{}, err
}
data := struct {
AccessToken string `json:"access_token"`
ExpiresIn int64 `json:"expires_in"`
TokenType string `json:"token_type"`
}{}
err = json.Unmarshal(resp.Data, &data)
if err != nil {
return "", time.Time{}, err
}
return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil
}
func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) {
t, err := token.getToken()
if err != nil {
return nil, err
}
resp, err := utils.Fetch(utils.FetchConfig{
Url: URL,
Headers: map[string]string{
"Client-ID": config.Config.Twitch.ClientID,
"Authorization": "Bearer " + t,
"User-Agent": "",
"Content-Type": "text/plain",
},
Data: dataBody,
Method: "POST",
})
if err != nil {
return nil, err
}
return resp, nil
}
func getIGDBID(name string) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
if err != nil {
return 0, err
}
if string(resp.Data) == "[]" {
resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
if err != nil {
return 0, err
}
}
var data model.IGDBSearches
if err = json.Unmarshal(resp.Data, &data); err != nil {
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
}
if len(data) == 1 {
return GetIGDBAppParentCache(data[0].Game)
}
maxSimilairty := 0.0
maxSimilairtyIndex := 0
for i, item := range data {
if strings.EqualFold(item.Name, name) {
return item.Game, nil
}
if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
if sim > maxSimilairty {
maxSimilairty = sim
maxSimilairtyIndex = i
}
}
detail, err := GetIGDBAppDetailCache(item.Game)
if err != nil {
return 0, err
}
for _, alternativeNames := range detail.AlternativeNames {
if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 {
if sim > maxSimilairty {
maxSimilairty = sim
maxSimilairtyIndex = i
}
}
}
}
if maxSimilairty >= 0.8 {
return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game)
}
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
func getIGDBIDBySteamSearch(name string) (int, error) {
baseURL, _ := url.Parse(constant.SteamSearchURL)
params := url.Values{}
params.Add("term", name)
baseURL.RawQuery = params.Encode()
resp, err := utils.Fetch(utils.FetchConfig{
Url: baseURL.String(),
})
if err != nil {
return 0, err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
if err != nil {
return 0, err
}
type searchResult struct {
ID int
Type string
Name string
}
var items []searchResult
doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
if strings.HasPrefix(itemKey, "App_") {
id, err := strconv.Atoi(itemKey[4:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "App",
Name: name,
})
}
if strings.HasPrefix(itemKey, "Bundle_") {
id, err := strconv.Atoi(itemKey[7:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "Bundle",
Name: name,
})
}
}
})
maxSim := 0.0
var maxSimItem searchResult
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
maxSimItem = item
break
} else {
sim := utils.Similarity(item.Name, name)
if sim >= 0.8 && sim > maxSim {
maxSim = sim
maxSimItem = item
}
}
}
if maxSim != 0 {
if maxSimItem.Type == "App" {
return GetIGDBIDBySteamAppIDCache(maxSimItem.ID)
}
if maxSimItem.Type == "Bundle" {
return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID)
}
}
return 0, fmt.Errorf("steam ID not found: %s", name)
}
// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
detail, err := GetIGDBAppDetailCache(id)
if err != nil {
return 0, err
}
hasParent := false
for detail.VersionParent != 0 {
hasParent = true
detail, err = GetIGDBAppDetailCache(detail.VersionParent)
if err != nil {
return 0, err
}
}
if hasParent {
return detail.ID, nil
}
return id, nil
}
func GetIGDBAppParetns(ids []int) (map[int]int, error) {
var err error
idsStr := make([]string, len(ids))
for i, id := range ids {
idsStr[i] = strconv.Itoa(id)
}
resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")))
if err != nil {
return nil, err
}
var data []struct {
ID int `json:"id"`
VersionParent int `json:"version_parent"`
}
if err = json.Unmarshal(resp.Data, &data); err != nil {
return nil, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
}
parents := make(map[int]int)
for _, item := range data {
if item.VersionParent != 0 {
pid, err := GetIGDBAppParentCache(item.VersionParent)
if err != nil {
parents[item.ID] = item.ID
} else {
parents[item.ID] = pid
}
} else {
parents[item.ID] = item.ID
}
}
return parents, nil
}
func GetIGDBAppParentCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_parent:%d", id)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetIGDBAppParent(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, id)
return id, nil
}
}
return GetIGDBAppParent(id)
}
// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
func GetIGDBID(name string) (int, error) {
name1 := name
name2 := FormatName(name)
names := []string{name1}
if name1 != name2 {
names = append(names, name2)
}
for _, name := range names {
id, err := getIGDBID(name)
if err == nil {
return id, nil
}
}
for _, name := range names {
id, err := getIGDBIDBySteamSearch(name)
if err == nil {
return id, nil
}
}
return 0, errors.New("IGDB ID not found")
}
func GetIGDBIDCache(name string) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id:%s", name)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetIGDBID(name)
if err != nil {
return 0, err
}
_ = cache.Add(key, id)
return id, nil
}
} else {
return GetIGDBID(name)
}
}
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
var err error
resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))
if err != nil {
return nil, err
}
var data model.IGDBGameDetails
if err = json.Unmarshal(resp.Data, &data); err != nil {
return nil, err
}
if len(data) == 0 {
return nil, errors.New("IGDB App not found")
}
if data[0].Name == "" {
return GetIGDBAppDetail(id)
}
return data[0], nil
}
func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_game:%v", id)
val, exist := cache.Get(key)
if exist {
var data model.IGDBGameDetail
if err := json.Unmarshal([]byte(val), &data); err != nil {
return nil, err
}
return &data, nil
} else {
data, err := GetIGDBAppDetail(id)
if err != nil {
return nil, err
}
dataBytes, err := json.Marshal(data)
if err != nil {
return nil, err
}
_ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour)
return data, nil
}
} else {
return GetIGDBAppDetail(id)
}
}
func GetIGDBCompany(id int) (string, error) {
var err error
resp, err := igdbFetch(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
if err != nil {
return "", err
}
var data model.IGDBCompanies
if err = json.Unmarshal(resp.Data, &data); err != nil {
return "", err
}
if len(data) == 0 {
return "", errors.New("not found")
}
if data[0].Name == "" {
return GetIGDBCompany(id)
}
return data[0].Name, nil
}
func GetIGDBCompanyCache(id int) (string, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_companies:%v", id)
val, exist := cache.Get(key)
if exist {
return val, nil
} else {
data, err := GetIGDBCompany(id)
if err != nil {
return "", err
}
_ = cache.Add(key, data)
return data, nil
}
} else {
return GetIGDBCompany(id)
}
}
func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
item := &model.GameInfo{}
detail, err := GetIGDBAppDetailCache(id)
if err != nil {
return nil, err
}
item.IGDBID = id
item.Name = detail.Name
item.Description = detail.Summary
item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)
item.InfoUpdatedAt = time.Now()
for _, lang := range detail.LanguageSupports {
if lang.LanguageSupportType == 3 {
l, exist := constant.IGDBLanguages[lang.Language]
if !exist {
continue
}
item.Languages = append(item.Languages, l.Name)
}
}
for _, screenshot := range detail.Screenshots {
item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1))
}
for _, alias := range detail.AlternativeNames {
item.Aliases = append(item.Aliases, alias.Name)
}
for _, company := range detail.InvolvedCompanies {
if company.Developer || company.Publisher {
companyName, err := GetIGDBCompanyCache(company.Company)
if err != nil {
continue
}
if company.Developer {
item.Developers = append(item.Developers, companyName)
}
if company.Publisher {
item.Publishers = append(item.Publishers, companyName)
}
}
}
return item, nil
}
// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
id, err := GetIGDBIDCache(game.Name)
if err != nil {
return nil, err
}
d, err := db.GetGameInfoByPlatformID("igdb", id)
if err == nil {
d.GameIDs = append(d.GameIDs, game.ID)
d.GameIDs = utils.Unique(d.GameIDs)
return d, nil
}
info, err := GenerateGameInfo("igdb", id)
if err != nil {
return nil, err
}
info.GameIDs = append(info.GameIDs, game.ID)
info.GameIDs = utils.Unique(info.GameIDs)
return info, nil
}
func GetIGDBIDBySteamAppID(id int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
return 0, err
}
var data []struct {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Data, &data); err != nil {
return 0, err
}
if len(data) == 0 {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamAppID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamAppIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamAppID(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, strconv.Itoa(data))
return data, nil
}
} else {
return GetIGDBIDBySteamAppID(id)
}
}
func GetIGDBIDBySteamBundleID(id int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
return 0, err
}
var data []struct {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Data, &data); err != nil {
return 0, err
}
if len(data) == 0 {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamBundleID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamBundleIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamBundleID(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, strconv.Itoa(data))
return data, nil
}
} else {
return GetIGDBIDBySteamBundleID(id)
}
}
func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
var err error
conditionBuilder := strings.Builder{}
for _, id := range ids {
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id))
}
condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
resp, err := igdbFetch(constant.IGDBWebsitesURL, respBody)
if err != nil {
return nil, err
}
var data []struct {
Game int `json:"game"`
Url string `json:"url"`
}
if err = json.Unmarshal(resp.Data, &data); err != nil {
return nil, err
}
ret := make(map[int]int)
regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
for _, d := range data {
idStr := regex.FindStringSubmatch(d.Url)
if len(idStr) < 2 {
continue
}
id, err := strconv.Atoi(idStr[1])
if err == nil {
pid, err := GetIGDBAppParentCache(d.Game)
if err == nil {
ret[id] = pid
} else {
ret[id] = 0
}
}
}
for _, id := range ids {
if _, ok := ret[id]; !ok {
ret[id] = 0
}
}
return ret, nil
}
func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
res := make(map[int]int)
notExistIDs := make([]int, 0)
if config.Config.RedisAvaliable {
for _, steamID := range ids {
key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID)
val, exist := cache.Get(key)
if exist {
igdbID, _ := strconv.Atoi(val)
res[steamID] = igdbID
} else {
notExistIDs = append(notExistIDs, steamID)
}
}
if len(res) == len(ids) {
return res, nil
}
idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
if err != nil {
return nil, err
}
for steamID, igdbID := range idMap {
res[steamID] = igdbID
if igdbID != 0 {
_ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID)
}
}
return res, nil
} else {
return GetIGDBIDsBySteamIDs(ids)
}
}
// GetIGDBPopularGameIDs get IGDB popular game IDs
// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users “Want to Play” lists.
// popularity_type = 3 IGDB Playing: Additions to IGDB.com users “Playing” lists.
// popularity_type = 4 IGDB Played: Additions to IGDB.com users “Played” lists.
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
var err error
resp, err := igdbFetch(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType))
if err != nil {
return nil, err
}
type IgdbPopularity struct {
GameID int `json:"game_id"`
Value float64 `json:"value"`
}
var data []IgdbPopularity
if err = json.Unmarshal(resp.Data, &data); err != nil {
return nil, err
}
ret := make([]int, 0)
for _, d := range data {
pid, err := GetIGDBAppParentCache(d.GameID)
if err != nil {
ret = append(ret, d.GameID)
continue
}
ret = append(ret, pid)
}
return ret, nil
}