pcgamedb/crawler/igdb.go

package crawler

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"regexp"
	"runtime/debug"
	"strconv"
	"strings"

	"pcgamedb/cache"
	"pcgamedb/config"
	"pcgamedb/constant"
	"pcgamedb/db"
	"pcgamedb/model"
	"pcgamedb/utils"
)

var TwitchToken string

func getIGDBID(name string) (int, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return 0, fmt.Errorf("failed to login twitch: %w", err)
		}
	}
	resp, err := utils.Fetch(utils.FetchConfig{
		Url: constant.IGDBSearchURL,
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data:   fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name),
		Method: "POST",
	})
	if string(resp.Data) == "[]" {
		resp, err = utils.Fetch(utils.FetchConfig{
			Url: constant.IGDBSearchURL,
			Headers: map[string]string{
				"Client-ID":     config.Config.Twitch.ClientID,
				"Authorization": "Bearer " + TwitchToken,
				"User-Agent":    "",
				"Content-Type":  "text/plain",
			},
			Data:   fmt.Sprintf(`search "%s"; fields *; limit 50;`, name),
			Method: "POST",
		})
	}
	if err != nil {
		return 0, err
	}
	var data model.IGDBSearches
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
	}
	if len(data) == 1 {
		return data[0].Game, nil
	}
	maxSimilairty := 0.0
	maxSimilairtyIndex := 0
	for i, item := range data {
		if strings.EqualFold(item.Name, name) {
			return item.Game, nil
		}
		if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
			if sim > maxSimilairty {
				maxSimilairty = sim
				maxSimilairtyIndex = i
			}
		}
		detail, err := GetIGDBAppDetailCache(item.Game)
		if err != nil {
			return 0, err
		}
		for _, alternativeNames := range detail.AlternativeNames {
			if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 {
				if sim > maxSimilairty {
					maxSimilairty = sim
					maxSimilairtyIndex = i
				}
			}
		}
	}
	if maxSimilairty >= 0.8 {
		return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game)
	}
	return 0, fmt.Errorf("IGDB ID not found: %s", name)
}

// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
	detail, err := GetIGDBAppDetailCache(id)
	if err != nil {
		return 0, err
	}
	hasParent := false
	for detail.VersionParent != 0 {
		hasParent = true
		detail, err = GetIGDBAppDetailCache(detail.VersionParent)
		if err != nil {
			return 0, err
		}
	}
	if hasParent {
		return detail.ID, nil
	}
	return id, nil
}

func GetIGDBAppParetns(ids []int) (map[int]int, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return nil, err
		}
	}
	idsStr := make([]string, len(ids))
	for i, id := range ids {
		idsStr[i] = strconv.Itoa(id)
	}
	resp, err := utils.Fetch(utils.FetchConfig{
		Url: constant.IGDBGameURL,
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data:   fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")),
		Method: "POST",
	})
	if err != nil {
		return nil, err
	}
	var data []struct {
		ID            int `json:"id"`
		VersionParent int `json:"version_parent"`
	}
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return nil, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
	}
	parents := make(map[int]int)
	for _, item := range data {
		if item.VersionParent != 0 {
			pid, err := GetIGDBAppParentCache(item.VersionParent)
			if err != nil {
				parents[item.ID] = item.ID
			} else {
				parents[item.ID] = pid
			}
		} else {
			parents[item.ID] = item.ID
		}
	}
	return parents, nil
}

func GetIGDBAppParentCache(id int) (int, error) {
	if config.Config.RedisAvaliable {
		key := fmt.Sprintf("igdb_parent:%d", id)
		val, exist := cache.Get(key)
		if exist {
			id, err := strconv.Atoi(val)
			if err != nil {
				return 0, err
			}
			return id, nil
		} else {
			id, err := GetIGDBAppParent(id)
			if err != nil {
				return 0, err
			}
			_ = cache.Add(key, id)
			return id, nil
		}
	}
	return GetIGDBAppParent(id)
}

// GetIGDBID returns the IGDB ID of the game, try raw name first then formated names
func GetIGDBID(name string) (int, error) {
	name1 := name
	name2 := FormatName(name)
	names := []string{name1}
	if name1 != name2 {
		names = append(names, name2)
	}
	for _, name := range names {
		id, err := getIGDBID(name)
		if err == nil {
			return id, nil
		}
	}
	return 0, errors.New("IGDB ID not found")
}

func GetIGDBIDCache(name string) (int, error) {
	if config.Config.RedisAvaliable {
		key := fmt.Sprintf("igdb_id:%s", name)
		val, exist := cache.Get(key)
		if exist {
			id, err := strconv.Atoi(val)
			if err != nil {
				return 0, err
			}
			return id, nil
		} else {
			id, err := GetIGDBID(name)
			if err != nil {
				return 0, err
			}
			_ = cache.Add(key, id)
			return id, nil
		}
	} else {
		return GetIGDBID(name)
	}
}

func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return nil, err
		}
	}
	resp, err := utils.Fetch(utils.FetchConfig{
		Url: constant.IGDBGameURL,
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data:   fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id),
		Method: "POST",
	})
	if err != nil {
		return nil, err
	}
	var data model.IGDBGameDetails
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return nil, err
	}
	if len(data) == 0 {
		return nil, errors.New("IGDB App not found")
	}
	if data[0].Name == "" {
		return GetIGDBAppDetail(id)
	}
	return data[0], nil
}

func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
	if config.Config.RedisAvaliable {
		key := fmt.Sprintf("igdb_game:%v", id)
		val, exist := cache.Get(key)
		if exist {
			var data model.IGDBGameDetail
			if err := json.Unmarshal([]byte(val), &data); err != nil {
				return nil, err
			}
			return &data, nil
		} else {
			data, err := GetIGDBAppDetail(id)
			if err != nil {
				return nil, err
			}
			dataBytes, err := json.Marshal(data)
			if err != nil {
				return nil, err
			}
			_ = cache.Add(key, dataBytes)
			return data, nil
		}
	} else {
		return GetIGDBAppDetail(id)
	}
}

func LoginTwitch() (string, error) {
	baseURL, _ := url.Parse(constant.TwitchAuthURL)
	params := url.Values{}
	params.Add("client_id", config.Config.Twitch.ClientID)
	params.Add("client_secret", config.Config.Twitch.ClientSecret)
	params.Add("grant_type", "client_credentials")
	baseURL.RawQuery = params.Encode()
	resp, err := utils.Fetch(utils.FetchConfig{
		Url:    baseURL.String(),
		Method: "POST",
		Headers: map[string]string{
			"User-Agent": "",
		},
	})
	if err != nil {
		return "", err
	}
	data := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.Unmarshal(resp.Data, &data)
	if err != nil {
		return "", err
	}
	return data.AccessToken, nil
}

func GetIGDBCompany(id int) (string, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return "", err
		}
	}
	resp, err := utils.Fetch(utils.FetchConfig{
		Url: constant.IGDBCompaniesURL,
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data:   fmt.Sprintf(`where id=%v; fields *;`, id),
		Method: "POST",
	})
	if err != nil {
		return "", err
	}
	var data model.IGDBCompanies
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return "", err
	}
	if len(data) == 0 {
		return "", errors.New("not found")
	}
	if data[0].Name == "" {
		return GetIGDBCompany(id)
	}
	return data[0].Name, nil
}

func GetIGDBCompanyCache(id int) (string, error) {
	if config.Config.RedisAvaliable {
		key := fmt.Sprintf("igdb_companies:%v", id)
		val, exist := cache.Get(key)
		if exist {
			return val, nil
		} else {
			data, err := GetIGDBCompany(id)
			if err != nil {
				return "", err
			}
			_ = cache.Add(key, data)
			return data, nil
		}
	} else {
		return GetIGDBCompany(id)
	}
}

func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
	item := &model.GameInfo{}
	detail, err := GetIGDBAppDetailCache(id)
	if err != nil {
		return nil, err
	}
	item.IGDBID = id
	item.Name = detail.Name
	item.Description = detail.Summary
	item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)

	for _, lang := range detail.LanguageSupports {
		if lang.LanguageSupportType == 3 {
			l, exist := constant.IGDBLanguages[lang.Language]
			if !exist {
				continue
			}
			item.Languages = append(item.Languages, l.Name)
		}
	}

	for _, screenshot := range detail.Screenshots {
		item.Screenshots = append(item.Screenshots, strings.Replace(screenshot.URL, "t_thumb", "t_original", 1))
	}

	for _, alias := range detail.AlternativeNames {
		item.Aliases = append(item.Aliases, alias.Name)
	}

	for _, company := range detail.InvolvedCompanies {
		if company.Developer || company.Publisher {
			companyName, err := GetIGDBCompanyCache(company.Company)
			if err != nil {
				continue
			}
			if company.Developer {
				item.Developers = append(item.Developers, companyName)
			}
			if company.Publisher {
				item.Publishers = append(item.Publishers, companyName)
			}
		}
	}

	return item, nil
}

// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
	var err error
	if id == 0 {
		id, err = GetIGDBIDCache(game.Name)
		if err != nil {
			return nil, err
		}
	}
	d, err := db.GetGameInfoByPlatformID("igdb", id)
	if err == nil {
		d.GameIDs = append(d.GameIDs, game.ID)
		d.GameIDs = utils.Unique(d.GameIDs)
		return d, nil
	}
	info, err := GenerateGameInfo("igdb", id)
	if err != nil {
		return nil, err
	}
	info.GameIDs = append(info.GameIDs, game.ID)
	info.GameIDs = utils.Unique(info.GameIDs)
	return info, nil
}

func GetIGDBIDBySteamID(id int) (int, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return 0, err
		}
	}
	resp, err := utils.Fetch(utils.FetchConfig{
		Url:    constant.IGDBWebsitesURL,
		Method: "POST",
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data: fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id),
	})
	if err != nil {
		return 0, err
	}
	var data []struct {
		Game int `json:"game"`
	}
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return 0, err
	}
	if len(data) == 0 {
		return 0, errors.New("not found")
	}
	if data[0].Game == 0 {
		return GetIGDBIDBySteamID(id)
	}
	return GetIGDBAppParentCache(data[0].Game)
}

func GetIGDBIDBySteamIDCache(id int) (int, error) {
	if config.Config.RedisAvaliable {
		key := fmt.Sprintf("igdb_id_by_steam_id:%v", id)
		val, exist := cache.Get(key)
		if exist {
			return strconv.Atoi(val)
		} else {
			data, err := GetIGDBIDBySteamID(id)
			if err != nil {
				return 0, err
			}
			_ = cache.Add(key, strconv.Itoa(data))
			return data, nil
		}
	} else {
		return GetIGDBIDBySteamID(id)
	}
}

func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return nil, err
		}
	}
	conditionBuilder := strings.Builder{}
	for _, id := range ids {
		conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
		conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id))
	}
	condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
	respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
	resp, err := utils.Fetch(utils.FetchConfig{
		Url:    constant.IGDBWebsitesURL,
		Method: "POST",
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data: respBody,
	})
	if err != nil {
		return nil, err
	}
	var data []struct {
		Game int    `json:"game"`
		Url  string `json:"url"`
	}
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return nil, err
	}
	ret := make(map[int]int)
	regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
	for _, d := range data {
		idStr := regex.FindStringSubmatch(d.Url)
		if len(idStr) < 2 {
			continue
		}
		id, err := strconv.Atoi(idStr[1])
		if err == nil {
			pid, err := GetIGDBAppParentCache(d.Game)
			if err == nil {
				ret[id] = pid
			} else {
				ret[id] = 0
			}
		}
	}
	for _, id := range ids {
		if _, ok := ret[id]; !ok {
			ret[id] = 0
		}
	}
	return ret, nil
}

func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
	res := make(map[int]int)
	notExistIDs := make([]int, 0)
	if config.Config.RedisAvaliable {
		for _, steamID := range ids {
			key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID)
			val, exist := cache.Get(key)
			if exist {
				igdbID, _ := strconv.Atoi(val)
				res[steamID] = igdbID
			} else {
				notExistIDs = append(notExistIDs, steamID)
			}
		}
		if len(res) == len(ids) {
			return res, nil
		}
		idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
		if err != nil {
			return nil, err
		}
		for steamID, igdbID := range idMap {
			res[steamID] = igdbID
			if igdbID != 0 {
				_ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID)
			}
		}
		return res, nil
	} else {
		return GetIGDBIDsBySteamIDs(ids)
	}
}

// GetIGDBPopularGameIDs get IGDB popular game IDs
// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists.
// popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists.
// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
	var err error
	if TwitchToken == "" {
		TwitchToken, err = LoginTwitch()
		if err != nil {
			return nil, err
		}
	}
	resp, err := utils.Fetch(utils.FetchConfig{
		Url:    constant.IGDBPopularityURL,
		Method: "POST",
		Headers: map[string]string{
			"Client-ID":     config.Config.Twitch.ClientID,
			"Authorization": "Bearer " + TwitchToken,
			"User-Agent":    "",
			"Content-Type":  "text/plain",
		},
		Data: fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType),
	})
	if err != nil {
		return nil, err
	}
	type IGDBPopularity struct {
		GameID int     `json:"game_id"`
		Value  float64 `json:"value"`
	}
	var data []IGDBPopularity
	if err = json.Unmarshal(resp.Data, &data); err != nil {
		return nil, err
	}
	ret := make([]int, 0)
	for _, d := range data {
		pid, err := GetIGDBAppParentCache(d.GameID)
		if err != nil {
			ret = append(ret, d.GameID)
			continue
		}
		ret = append(ret, pid)
	}
	return ret, nil
}