519 lines
14 KiB
Go
519 lines
14 KiB
Go
package crawler
|
||
|
||
import (
|
||
"bytes"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"net/url"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"game-crawler/cache"
|
||
"game-crawler/config"
|
||
"game-crawler/constant"
|
||
"game-crawler/db"
|
||
"game-crawler/model"
|
||
"game-crawler/utils"
|
||
|
||
"github.com/PuerkitoBio/goquery"
|
||
"github.com/go-resty/resty/v2"
|
||
)
|
||
|
||
type twitchToken struct {
|
||
}
|
||
|
||
var token = twitchToken{}
|
||
|
||
func (t *twitchToken) getToken() (string, error) {
|
||
if val, exist := cache.Get("twitch_token"); exist {
|
||
return val, nil
|
||
}
|
||
token, expires, err := loginTwitch()
|
||
if err != nil {
|
||
return "", fmt.Errorf("failed to login twitch: %w", err)
|
||
}
|
||
_ = cache.SetWithExpire("twitch_token", token, expires)
|
||
return token, nil
|
||
}
|
||
|
||
func loginTwitch() (string, time.Duration, error) {
|
||
baseURL, _ := url.Parse(constant.TwitchAuthURL)
|
||
params := url.Values{}
|
||
params.Add("client_id", config.Config.Twitch.ClientID)
|
||
params.Add("client_secret", config.Config.Twitch.ClientSecret)
|
||
params.Add("grant_type", "client_credentials")
|
||
baseURL.RawQuery = params.Encode()
|
||
|
||
resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
|
||
if err != nil {
|
||
return "", 0, fmt.Errorf("failed to make request: %s: %w", baseURL.String(), err)
|
||
}
|
||
|
||
data := struct {
|
||
AccessToken string `json:"access_token"`
|
||
ExpiresIn int64 `json:"expires_in"`
|
||
TokenType string `json:"token_type"`
|
||
}{}
|
||
err = json.Unmarshal(resp.Body(), &data)
|
||
if err != nil {
|
||
return "", 0, fmt.Errorf("failed to parse response: %w", err)
|
||
}
|
||
return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
|
||
}
|
||
|
||
func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
|
||
t, err := token.getToken()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to get Twitch token: %w", err)
|
||
}
|
||
|
||
resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{
|
||
"Client-ID": config.Config.Twitch.ClientID,
|
||
"Authorization": "Bearer " + t,
|
||
"User-Agent": "",
|
||
"Content-Type": "text/plain",
|
||
}).Post(URL)
|
||
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to make request: %s: %w", URL, err)
|
||
}
|
||
return resp, nil
|
||
}
|
||
|
||
func getIGDBID(name string) (int, error) {
|
||
resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to search IGDB ID: %s: %w", name, err)
|
||
}
|
||
|
||
if string(resp.Body()) == "[]" {
|
||
resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to fallback search: %s: %w", name, err)
|
||
}
|
||
}
|
||
|
||
var data model.IGDBSearches
|
||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||
return 0, fmt.Errorf("failed to parse IGDB search response: %w", err)
|
||
}
|
||
|
||
if len(data) == 1 {
|
||
return GetIGDBAppParent(data[0].Game)
|
||
}
|
||
|
||
maxSimilarity := 0.0
|
||
maxSimilarityIndex := 0
|
||
for i, item := range data {
|
||
if strings.EqualFold(item.Name, name) {
|
||
return GetIGDBAppParent(item.Game)
|
||
}
|
||
if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
|
||
if sim > maxSimilarity {
|
||
maxSimilarity = sim
|
||
maxSimilarityIndex = i
|
||
}
|
||
}
|
||
|
||
detail, err := GetIGDBAppDetail(item.Game)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to get IGDB app detail: %d: %w", item.Game, err)
|
||
}
|
||
for _, altName := range detail.AlternativeNames {
|
||
if sim := utils.Similarity(altName.Name, name); sim >= 0.8 {
|
||
if sim > maxSimilarity {
|
||
maxSimilarity = sim
|
||
maxSimilarityIndex = i
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if maxSimilarity >= 0.8 {
|
||
return GetIGDBAppParent(data[maxSimilarityIndex].Game)
|
||
}
|
||
|
||
return 0, fmt.Errorf("IGDB ID not found: %s", name)
|
||
}
|
||
|
||
func getIGDBIDBySteamSearch(name string) (int, error) {
|
||
baseURL, _ := url.Parse(constant.SteamSearchURL)
|
||
params := url.Values{}
|
||
params.Add("term", name)
|
||
baseURL.RawQuery = params.Encode()
|
||
|
||
resp, err := utils.Request().Get(baseURL.String())
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
|
||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
type searchResult struct {
|
||
ID int
|
||
Type string
|
||
Name string
|
||
}
|
||
var items []searchResult
|
||
doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
|
||
if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
|
||
if strings.HasPrefix(itemKey, "App_") {
|
||
id, err := strconv.Atoi(itemKey[4:])
|
||
if err != nil {
|
||
return
|
||
}
|
||
name := s.Find(".title").Text()
|
||
items = append(items, searchResult{
|
||
ID: id,
|
||
Type: "App",
|
||
Name: name,
|
||
})
|
||
}
|
||
if strings.HasPrefix(itemKey, "Bundle_") {
|
||
id, err := strconv.Atoi(itemKey[7:])
|
||
if err != nil {
|
||
return
|
||
}
|
||
name := s.Find(".title").Text()
|
||
items = append(items, searchResult{
|
||
ID: id,
|
||
Type: "Bundle",
|
||
Name: name,
|
||
})
|
||
}
|
||
}
|
||
})
|
||
|
||
maxSim := 0.0
|
||
var maxSimItem searchResult
|
||
for _, item := range items {
|
||
if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
|
||
maxSimItem = item
|
||
break
|
||
} else {
|
||
sim := utils.Similarity(item.Name, name)
|
||
if sim >= 0.8 && sim > maxSim {
|
||
maxSim = sim
|
||
maxSimItem = item
|
||
}
|
||
}
|
||
}
|
||
if maxSim != 0 {
|
||
if maxSimItem.Type == "App" {
|
||
return GetIGDBIDBySteamAppID(maxSimItem.ID)
|
||
}
|
||
if maxSimItem.Type == "Bundle" {
|
||
return GetIGDBIDBySteamBundleID(maxSimItem.ID)
|
||
}
|
||
}
|
||
return 0, fmt.Errorf("steam ID not found: %s", name)
|
||
}
|
||
|
||
// GetIGDBAppParent returns the parent of the game, if no parent return itself
|
||
func GetIGDBAppParent(id int) (int, error) {
|
||
key := fmt.Sprintf("igdb_parent:%d", id)
|
||
val, exist := cache.Get(key)
|
||
if exist {
|
||
id, err := strconv.Atoi(val)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to parse cached IGDB parent ID: %s: %w", key, err)
|
||
}
|
||
return id, nil
|
||
}
|
||
detail, err := GetIGDBAppDetail(id)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to fetch IGDB app detail for parent: %d: %w", id, err)
|
||
}
|
||
hasParent := false
|
||
if detail.ParentGame != 0 {
|
||
hasParent = true
|
||
detail, err = GetIGDBAppDetail(detail.ParentGame)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to fetch IGDB version parent: %d: %w", detail.VersionParent, err)
|
||
}
|
||
}
|
||
for detail.VersionParent != 0 {
|
||
hasParent = true
|
||
detail, err = GetIGDBAppDetail(detail.VersionParent)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to fetch IGDB version parent: %d: %w", detail.VersionParent, err)
|
||
}
|
||
}
|
||
if hasParent {
|
||
return detail.ID, nil
|
||
}
|
||
|
||
_ = cache.Set(key, id)
|
||
|
||
return id, nil
|
||
}
|
||
|
||
// GetIGDBID retrieves the IGDB ID of a game by its name using IGDB API and fallback mechanisms.
|
||
func GetIGDBID(name string) (int, error) {
|
||
key := fmt.Sprintf("igdb_id:%s", name)
|
||
if val, exist := cache.Get(key); exist {
|
||
return strconv.Atoi(val)
|
||
}
|
||
|
||
// Normalize game name and try multiple variations
|
||
normalizedNames := []string{name, FormatName(name)}
|
||
for _, n := range normalizedNames {
|
||
id, err := getIGDBID(n)
|
||
if err == nil {
|
||
_ = cache.Set(key, id)
|
||
return id, nil
|
||
}
|
||
}
|
||
|
||
// Fallback to Steam search if IGDB search fails
|
||
for _, n := range normalizedNames {
|
||
id, err := getIGDBIDBySteamSearch(n)
|
||
if err == nil {
|
||
_ = cache.Set(key, id)
|
||
return id, nil
|
||
}
|
||
}
|
||
|
||
return 0, fmt.Errorf("IGDB ID not found: %s", name)
|
||
}
|
||
|
||
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
|
||
key := fmt.Sprintf("igdb_game:%v", id)
|
||
val, exist := cache.Get(key)
|
||
if exist {
|
||
var data model.IGDBGameDetail
|
||
if err := json.Unmarshal([]byte(val), &data); err != nil {
|
||
return nil, fmt.Errorf("failed to parse cached IGDB game detail: %s: %w", key, err)
|
||
}
|
||
return &data, nil
|
||
}
|
||
|
||
resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id = %v; fields *,alternative_names.*,language_supports.*,screenshots.*,cover.*,involved_companies.*,game_engines.*,game_modes.*,genres.*,player_perspectives.*,release_dates.*,videos.*,websites.*,platforms.*,themes.*,collections.*;`, id))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to fetch IGDB game detail for ID %d: %w", id, err)
|
||
}
|
||
|
||
var data model.IGDBGameDetails
|
||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||
return nil, fmt.Errorf("failed to unmarshal IGDB game detail response: %w", err)
|
||
}
|
||
|
||
if len(data) == 0 {
|
||
return nil, fmt.Errorf("IGDB game detail not found for ID %d", id)
|
||
}
|
||
|
||
if data[0].Name == "" {
|
||
return GetIGDBAppDetail(id)
|
||
}
|
||
|
||
jsonBytes, err := json.Marshal(data[0])
|
||
if err == nil {
|
||
_ = cache.Set(key, string(jsonBytes))
|
||
}
|
||
|
||
return data[0], nil
|
||
}
|
||
|
||
// GetIGDBCompany retrieves the company name from IGDB by its ID.
|
||
func GetIGDBCompany(id int) (string, error) {
|
||
key := fmt.Sprintf("igdb_companies:%d", id)
|
||
if val, exist := cache.Get(key); exist {
|
||
return val, nil
|
||
}
|
||
|
||
query := fmt.Sprintf(`where id=%d; fields *;`, id)
|
||
resp, err := igdbRequest(constant.IGDBCompaniesURL, query)
|
||
if err != nil {
|
||
return "", fmt.Errorf("failed to fetch IGDB company for ID %d: %w", id, err)
|
||
}
|
||
|
||
var data model.IGDBCompanies
|
||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||
return "", fmt.Errorf("failed to unmarshal IGDB companies response: %w", err)
|
||
}
|
||
|
||
if len(data) == 0 {
|
||
return "", errors.New("company not found")
|
||
}
|
||
|
||
companyName := data[0].Name
|
||
_ = cache.Set(key, companyName)
|
||
return companyName, nil
|
||
}
|
||
|
||
// GenerateIGDBGameInfo generates detailed game information based on an IGDB ID.
|
||
func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
|
||
detail, err := GetIGDBAppDetail(id)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to fetch IGDB app detail for ID %d: %w", id, err)
|
||
}
|
||
|
||
gameInfo := &model.GameInfo{
|
||
IGDBID: id,
|
||
Name: detail.Name,
|
||
Description: detail.Summary,
|
||
Cover: strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1),
|
||
}
|
||
|
||
for _, lang := range detail.LanguageSupports {
|
||
if lang.LanguageSupportType == 3 {
|
||
if l, exist := constant.IGDBLanguages[lang.Language]; exist {
|
||
gameInfo.Languages = append(gameInfo.Languages, l.Name)
|
||
}
|
||
}
|
||
}
|
||
|
||
for _, screenshot := range detail.Screenshots {
|
||
gameInfo.Screenshots = append(gameInfo.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1))
|
||
}
|
||
|
||
for _, alias := range detail.AlternativeNames {
|
||
gameInfo.Aliases = append(gameInfo.Aliases, alias.Name)
|
||
}
|
||
|
||
for _, company := range detail.InvolvedCompanies {
|
||
companyName, err := GetIGDBCompany(company.Company)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if company.Developer {
|
||
gameInfo.Developers = append(gameInfo.Developers, companyName)
|
||
}
|
||
if company.Publisher {
|
||
gameInfo.Publishers = append(gameInfo.Publishers, companyName)
|
||
}
|
||
}
|
||
|
||
for _, mode := range detail.GameModes {
|
||
gameInfo.GameModes = append(gameInfo.GameModes, mode.Name)
|
||
}
|
||
|
||
for _, genre := range detail.Genres {
|
||
gameInfo.Genres = append(gameInfo.Genres, genre.Name)
|
||
}
|
||
|
||
for _, platform := range detail.Platforms {
|
||
gameInfo.Platforms = append(gameInfo.Platforms, platform.Name)
|
||
}
|
||
|
||
return gameInfo, nil
|
||
}
|
||
|
||
// OrganizeGameItemWithIGDB links a game item with its corresponding IGDB game information.
|
||
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
|
||
id, err := GetIGDBID(game.Name)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to get IGDB ID for game '%s': %w", game.Name, err)
|
||
}
|
||
|
||
info, err := db.GetGameInfoByPlatformID("igdb", id)
|
||
if err == nil {
|
||
info.GameIDs = utils.Unique(append(info.GameIDs, game.ID))
|
||
return info, nil
|
||
}
|
||
|
||
info, err = GenerateIGDBGameInfo(id)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to generate IGDB game info for ID %d: %w", id, err)
|
||
}
|
||
|
||
info.GameIDs = utils.Unique(append(info.GameIDs, game.ID))
|
||
return info, nil
|
||
}
|
||
|
||
// GetIGDBIDBySteamAppID retrieves the IGDB ID of a game using its Steam App ID.
|
||
func GetIGDBIDBySteamAppID(id int) (int, error) {
|
||
key := fmt.Sprintf("igdb_id_by_steam_app_id:%d", id)
|
||
if val, exist := cache.Get(key); exist {
|
||
return strconv.Atoi(val)
|
||
}
|
||
|
||
query := fmt.Sprintf(`where url = "https://store.steampowered.com/app/%d" | url = "https://store.steampowered.com/app/%d/"; fields game;`, id, id)
|
||
resp, err := igdbRequest(constant.IGDBWebsitesURL, query)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to fetch IGDB ID by Steam App ID %d: %w", id, err)
|
||
}
|
||
|
||
var data []struct {
|
||
Game int `json:"game"`
|
||
}
|
||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||
return 0, fmt.Errorf("failed to unmarshal IGDB response for Steam App ID %d: %w", id, err)
|
||
}
|
||
|
||
if len(data) == 0 || data[0].Game == 0 {
|
||
return 0, errors.New("no matching IGDB game found")
|
||
}
|
||
|
||
igdbID := data[0].Game
|
||
_ = cache.Set(key, strconv.Itoa(igdbID))
|
||
|
||
return GetIGDBAppParent(igdbID)
|
||
}
|
||
|
||
// GetIGDBIDBySteamBundleID retrieves the IGDB ID of a game using its Steam Bundle ID.
|
||
func GetIGDBIDBySteamBundleID(id int) (int, error) {
|
||
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%d", id)
|
||
if val, exist := cache.Get(key); exist {
|
||
return strconv.Atoi(val)
|
||
}
|
||
|
||
query := fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%d" | url = "https://store.steampowered.com/bundle/%d/"; fields game;`, id, id)
|
||
resp, err := igdbRequest(constant.IGDBWebsitesURL, query)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("failed to fetch IGDB ID by Steam Bundle ID %d: %w", id, err)
|
||
}
|
||
|
||
var data []struct {
|
||
Game int `json:"game"`
|
||
}
|
||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||
return 0, fmt.Errorf("failed to unmarshal IGDB response for Steam Bundle ID %d: %w", id, err)
|
||
}
|
||
|
||
if len(data) == 0 || data[0].Game == 0 {
|
||
return 0, errors.New("no matching IGDB game found")
|
||
}
|
||
|
||
igdbID := data[0].Game
|
||
_ = cache.Set(key, strconv.Itoa(igdbID))
|
||
|
||
return GetIGDBAppParent(igdbID)
|
||
}
|
||
|
||
// GetIGDBPopularGameIDs retrieves popular IGDB game IDs based on a given popularity type.
|
||
// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
|
||
// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists.
|
||
// popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists.
|
||
// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
|
||
func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) {
|
||
query := fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %d; offset %d; where popularity_type = %d;", limit, offset, popularityType)
|
||
resp, err := igdbRequest(constant.IGDBPopularityURL, query)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("failed to fetch popular IGDB game IDs for type %d: %w", popularityType, err)
|
||
}
|
||
|
||
var data []struct {
|
||
GameID int `json:"game_id"`
|
||
Value float64 `json:"value"`
|
||
}
|
||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||
return nil, fmt.Errorf("failed to unmarshal IGDB popular games response: %w", err)
|
||
}
|
||
|
||
gameIDs := make([]int, 0, len(data))
|
||
for _, d := range data {
|
||
parentID, err := GetIGDBAppParent(d.GameID)
|
||
if err != nil {
|
||
gameIDs = append(gameIDs, d.GameID)
|
||
} else {
|
||
gameIDs = append(gameIDs, parentID)
|
||
}
|
||
}
|
||
return gameIDs, nil
|
||
}
|