pcgamedb/crawler/igdb.go

package crawler

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"runtime/debug"
	"strconv"
	"strings"
	"time"

	"pcgamedb/cache"
	"pcgamedb/config"
	"pcgamedb/constant"
	"pcgamedb/db"
	"pcgamedb/model"
	"pcgamedb/utils"

	"github.com/PuerkitoBio/goquery"
	"github.com/go-resty/resty/v2"
)

type twitchToken struct {
}

var token = twitchToken{}

func (t *twitchToken) getToken() (string, error) {
	if val, exist := cache.Get("twitch_token"); exist {
		return val, nil
	}
	token, expires, err := loginTwitch()
	if err != nil {
		return "", fmt.Errorf("failed to login twitch: %w", err)
	}
	_ = cache.SetWithExpire("twitch_token", token, expires)
	return token, nil
}

func loginTwitch() (string, time.Duration, error) {
	baseURL, _ := url.Parse(constant.TwitchAuthURL)
	params := url.Values{}
	params.Add("client_id", config.Config.Twitch.ClientID)
	params.Add("client_secret", config.Config.Twitch.ClientSecret)
	params.Add("grant_type", "client_credentials")
	baseURL.RawQuery = params.Encode()
	resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
	if err != nil {
		return "", 0, err
	}
	data := struct {
		AccessToken string `json:"access_token"`
		ExpiresIn   int64  `json:"expires_in"`
		TokenType   string `json:"token_type"`
	}{}
	err = json.Unmarshal(resp.Body(), &data)
	if err != nil {
		return "", 0, err
	}
	return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
}

func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
	t, err := token.getToken()
	if err != nil {
		return nil, err
	}
	resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{
		"Client-ID":     config.Config.Twitch.ClientID,
		"Authorization": "Bearer " + t,
		"User-Agent":    "",
		"Content-Type":  "text/plain",
	}).Post(URL)
	if err != nil {
		return nil, err
	}
	return resp, nil
}

func getIGDBID(name string) (int, error) {
	var err error
	resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
	if err != nil {
		return 0, err
	}
	if string(resp.Body()) == "[]" {
		resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
		if err != nil {
			return 0, err
		}
	}

	var data model.IGDBSearches
	if err = json.Unmarshal(resp.Body(), &data); err != nil {
		return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
	}
	if len(data) == 1 {
		return GetIGDBAppParent(data[0].Game)
	}
	maxSimilairty := 0.0
	maxSimilairtyIndex := 0
	for i, item := range data {
		if strings.EqualFold(item.Name, name) {
			return item.Game, nil
		}
		if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
			if sim > maxSimilairty {
				maxSimilairty = sim
				maxSimilairtyIndex = i
			}
		}
		detail, err := GetIGDBAppDetail(item.Game)
		if err != nil {
			return 0, err
		}
		for _, alternativeNames := range detail.AlternativeNames {
			if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 {
				if sim > maxSimilairty {
					maxSimilairty = sim
					maxSimilairtyIndex = i
				}
			}
		}
	}
	if maxSimilairty >= 0.8 {
		return GetIGDBAppParent(data[maxSimilairtyIndex].Game)
	}
	return 0, fmt.Errorf("IGDB ID not found: %s", name)
}

func getIGDBIDBySteamSearch(name string) (int, error) {
	baseURL, _ := url.Parse(constant.SteamSearchURL)
	params := url.Values{}
	params.Add("term", name)
	baseURL.RawQuery = params.Encode()

	resp, err := utils.Request().Get(baseURL.String())
	if err != nil {
		return 0, err
	}

	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
	if err != nil {
		return 0, err
	}
	type searchResult struct {
		ID   int
		Type string
		Name string
	}
	var items []searchResult
	doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
		if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
			if strings.HasPrefix(itemKey, "App_") {
				id, err := strconv.Atoi(itemKey[4:])
				if err != nil {
					return
				}
				name := s.Find(".title").Text()
				items = append(items, searchResult{
					ID:   id,
					Type: "App",
					Name: name,
				})
			}
			if strings.HasPrefix(itemKey, "Bundle_") {
				id, err := strconv.Atoi(itemKey[7:])
				if err != nil {
					return
				}
				name := s.Find(".title").Text()
				items = append(items, searchResult{
					ID:   id,
					Type: "Bundle",
					Name: name,
				})
			}
		}
	})

	maxSim := 0.0
	var maxSimItem searchResult
	for _, item := range items {
		if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
			maxSimItem = item
			break
		} else {
			sim := utils.Similarity(item.Name, name)
			if sim >= 0.8 && sim > maxSim {
				maxSim = sim
				maxSimItem = item
			}
		}
	}
	if maxSim != 0 {
		if maxSimItem.Type == "App" {
			return GetIGDBIDBySteamAppID(maxSimItem.ID)
		}
		if maxSimItem.Type == "Bundle" {
			return GetIGDBIDBySteamBundleID(maxSimItem.ID)
		}
	}
	return 0, fmt.Errorf("steam ID not found: %s", name)
}

// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
	key := fmt.Sprintf("igdb_parent:%d", id)
	val, exist := cache.Get(key)
	if exist {
		id, err := strconv.Atoi(val)
		if err != nil {
			return 0, err
		}
		return id, nil
	}
	detail, err := GetIGDBAppDetail(id)
	if err != nil {
		return 0, err
	}
	hasParent := false
	for detail.VersionParent != 0 {
		hasParent = true
		detail, err = GetIGDBAppDetail(detail.VersionParent)
		if err != nil {
			return 0, err
		}
	}
	if hasParent {
		return detail.ID, nil
	}

	_ = cache.Set(key, id)

	return id, nil
}

// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
func GetIGDBID(name string) (int, error) {
	key := fmt.Sprintf("igdb_id:%s", name)
	val, exist := cache.Get(key)
	if exist {
		return strconv.Atoi(val)
	}
	name1 := name
	name2 := FormatName(name)
	names := []string{name1}
	if name1 != name2 {
		names = append(names, name2)
	}
	for _, name := range names {
		id, err := getIGDBID(name)
		if err == nil {
			_ = cache.Set(key, id)
			return id, nil
		}
	}
	for _, name := range names {
		id, err := getIGDBIDBySteamSearch(name)
		if err == nil {
			_ = cache.Set(key, id)
			return id, nil
		}
	}
	return 0, errors.New("IGDB ID not found")
}

func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
	key := fmt.Sprintf("igdb_game:%v", id)
	val, exist := cache.Get(key)
	if exist {
		var data model.IGDBGameDetail
		if err := json.Unmarshal([]byte(val), &data); err != nil {
			return nil, err
		}
		return &data, nil
	}
	var err error
	resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))

	if err != nil {
		return nil, err
	}
	var data model.IGDBGameDetails
	if err = json.Unmarshal(resp.Body(), &data); err != nil {
		return nil, err
	}
	if len(data) == 0 {
		return nil, errors.New("IGDB App not found")
	}
	if data[0].Name == "" {
		return GetIGDBAppDetail(id)
	}

	jsonBytes, err := json.Marshal(data[0])
	if err == nil {
		_ = cache.Set(key, string(jsonBytes))
	}

	return data[0], nil
}

func GetIGDBCompany(id int) (string, error) {
	key := fmt.Sprintf("igdb_companies:%v", id)
	val, exist := cache.Get(key)
	if exist {
		return val, nil
	}
	var err error
	resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
	if err != nil {
		return "", err
	}
	var data model.IGDBCompanies
	if err = json.Unmarshal(resp.Body(), &data); err != nil {
		return "", err
	}
	if len(data) == 0 {
		return "", errors.New("not found")
	}
	if data[0].Name == "" {
		return GetIGDBCompany(id)
	}

	_ = cache.Set(key, data[0].Name)

	return data[0].Name, nil
}

func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
	item := &model.GameInfo{}
	detail, err := GetIGDBAppDetail(id)
	if err != nil {
		return nil, err
	}
	item.IGDBID = id
	item.Name = detail.Name
	item.Description = detail.Summary
	item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)
	item.InfoUpdatedAt = time.Now()

	for _, lang := range detail.LanguageSupports {
		if lang.LanguageSupportType == 3 {
			l, exist := constant.IGDBLanguages[lang.Language]
			if !exist {
				continue
			}
			item.Languages = append(item.Languages, l.Name)
		}
	}

	for _, screenshot := range detail.Screenshots {
		item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1))
	}

	for _, alias := range detail.AlternativeNames {
		item.Aliases = append(item.Aliases, alias.Name)
	}

	for _, company := range detail.InvolvedCompanies {
		if company.Developer || company.Publisher {
			companyName, err := GetIGDBCompany(company.Company)
			if err != nil {
				continue
			}
			if company.Developer {
				item.Developers = append(item.Developers, companyName)
			}
			if company.Publisher {
				item.Publishers = append(item.Publishers, companyName)
			}
		}
	}

	return item, nil
}

// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
	id, err := GetIGDBID(game.Name)
	if err != nil {
		return nil, err
	}
	d, err := db.GetGameInfoByPlatformID("igdb", id)
	if err == nil {
		d.GameIDs = append(d.GameIDs, game.ID)
		d.GameIDs = utils.Unique(d.GameIDs)
		return d, nil
	}
	info, err := GenerateGameInfo("igdb", id)
	if err != nil {
		return nil, err
	}
	info.GameIDs = append(info.GameIDs, game.ID)
	info.GameIDs = utils.Unique(info.GameIDs)
	return info, nil
}

func GetIGDBIDBySteamAppID(id int) (int, error) {
	key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
	val, exist := cache.Get(key)
	if exist {
		return strconv.Atoi(val)
	}
	var err error
	resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
	if err != nil {
		return 0, err
	}
	var data []struct {
		Game int `json:"game"`
	}
	if err = json.Unmarshal(resp.Body(), &data); err != nil {
		return 0, err
	}
	if len(data) == 0 {
		return 0, errors.New("not found")
	}
	if data[0].Game == 0 {
		return GetIGDBIDBySteamAppID(id)
	}

	_ = cache.Set(key, strconv.Itoa(data[0].Game))

	return GetIGDBAppParent(data[0].Game)
}

func GetIGDBIDBySteamBundleID(id int) (int, error) {
	key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
	val, exist := cache.Get(key)
	if exist {
		return strconv.Atoi(val)
	}

	var err error
	resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
	if err != nil {
		return 0, err
	}
	var data []struct {
		Game int `json:"game"`
	}
	if err = json.Unmarshal(resp.Body(), &data); err != nil {
		return 0, err
	}
	if len(data) == 0 {
		return 0, errors.New("not found")
	}
	if data[0].Game == 0 {
		return GetIGDBIDBySteamBundleID(id)
	}

	_ = cache.Set(key, strconv.Itoa(data[0].Game))

	return GetIGDBAppParent(data[0].Game)
}

// GetIGDBPopularGameIDs get IGDB popular game IDs
// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists.
// popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists.
// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
	var err error
	resp, err := igdbRequest(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType))
	if err != nil {
		return nil, err
	}
	type IgdbPopularity struct {
		GameID int     `json:"game_id"`
		Value  float64 `json:"value"`
	}
	var data []IgdbPopularity
	if err = json.Unmarshal(resp.Body(), &data); err != nil {
		return nil, err
	}
	ret := make([]int, 0)
	for _, d := range data {
		pid, err := GetIGDBAppParent(d.GameID)
		if err != nil {
			ret = append(ret, d.GameID)
			continue
		}
		ret = append(ret, pid)
	}
	return ret, nil
}