modify igdb.go steam.go
This commit is contained in:
parent
fa206a1bb7
commit
71a2ac545b
260
crawler/igdb.go
260
crawler/igdb.go
@ -9,6 +9,8 @@ import (
|
|||||||
"runtime/debug"
|
"runtime/debug"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"pcgamedb/cache"
|
"pcgamedb/cache"
|
||||||
"pcgamedb/config"
|
"pcgamedb/config"
|
||||||
@ -18,43 +20,98 @@ import (
|
|||||||
"pcgamedb/utils"
|
"pcgamedb/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
var TwitchToken string
|
type twitchToken struct {
|
||||||
|
token string
|
||||||
|
expires time.Time
|
||||||
|
once sync.Once
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *twitchToken) getToken() (string, error) {
|
||||||
|
t.once.Do(func() {
|
||||||
|
if config.Config.RedisAvaliable {
|
||||||
|
if dataBytes, exist := cache.Get("twitch_token"); exist {
|
||||||
|
_ = json.Unmarshal([]byte(dataBytes), &token)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if t.token == "" || time.Now().After(t.expires) {
|
||||||
|
token, expires, err := loginTwitch()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to login twitch: %w", err)
|
||||||
|
}
|
||||||
|
t.token = token
|
||||||
|
t.expires = expires
|
||||||
|
j, err := json.Marshal(t)
|
||||||
|
if err == nil {
|
||||||
|
_ = cache.Add("twitch_token", j)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return t.token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func loginTwitch() (string, time.Time, error) {
|
||||||
|
baseURL, _ := url.Parse(constant.TwitchAuthURL)
|
||||||
|
params := url.Values{}
|
||||||
|
params.Add("client_id", config.Config.Twitch.ClientID)
|
||||||
|
params.Add("client_secret", config.Config.Twitch.ClientSecret)
|
||||||
|
params.Add("grant_type", "client_credentials")
|
||||||
|
baseURL.RawQuery = params.Encode()
|
||||||
|
resp, err := utils.Fetch(utils.FetchConfig{
|
||||||
|
Url: baseURL.String(),
|
||||||
|
Method: "POST",
|
||||||
|
Headers: map[string]string{
|
||||||
|
"User-Agent": "",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", time.Time{}, err
|
||||||
|
}
|
||||||
|
data := struct {
|
||||||
|
AccessToken string `json:"access_token"`
|
||||||
|
ExpiresIn int64 `json:"expires_in"`
|
||||||
|
TokenType string `json:"token_type"`
|
||||||
|
}{}
|
||||||
|
err = json.Unmarshal(resp.Data, &data)
|
||||||
|
if err != nil {
|
||||||
|
return "", time.Time{}, err
|
||||||
|
}
|
||||||
|
return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) {
|
||||||
|
t, err := token.getToken()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resp, err := utils.Fetch(utils.FetchConfig{
|
||||||
|
Url: URL,
|
||||||
|
Headers: map[string]string{
|
||||||
|
"Client-ID": config.Config.Twitch.ClientID,
|
||||||
|
"Authorization": "Bearer " + t,
|
||||||
|
"User-Agent": "",
|
||||||
|
"Content-Type": "text/plain",
|
||||||
|
},
|
||||||
|
Data: dataBody,
|
||||||
|
Method: "POST",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var token = twitchToken{}
|
||||||
|
|
||||||
func getIGDBID(name string) (int, error) {
|
func getIGDBID(name string) (int, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("failed to login twitch: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBSearchURL,
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name),
|
|
||||||
Method: "POST",
|
|
||||||
})
|
|
||||||
if string(resp.Data) == "[]" {
|
|
||||||
resp, err = utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBSearchURL,
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`search "%s"; fields *; limit 50;`, name),
|
|
||||||
Method: "POST",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
if string(resp.Data) == "[]" {
|
||||||
|
resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
|
||||||
|
}
|
||||||
|
|
||||||
var data model.IGDBSearches
|
var data model.IGDBSearches
|
||||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||||
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
|
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
|
||||||
@ -115,27 +172,12 @@ func GetIGDBAppParent(id int) (int, error) {
|
|||||||
|
|
||||||
func GetIGDBAppParetns(ids []int) (map[int]int, error) {
|
func GetIGDBAppParetns(ids []int) (map[int]int, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
idsStr := make([]string, len(ids))
|
idsStr := make([]string, len(ids))
|
||||||
for i, id := range ids {
|
for i, id := range ids {
|
||||||
idsStr[i] = strconv.Itoa(id)
|
idsStr[i] = strconv.Itoa(id)
|
||||||
}
|
}
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")))
|
||||||
Url: constant.IGDBGameURL,
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")),
|
|
||||||
Method: "POST",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -226,23 +268,8 @@ func GetIGDBIDCache(name string) (int, error) {
|
|||||||
|
|
||||||
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
|
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBGameURL,
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id),
|
|
||||||
Method: "POST",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -286,52 +313,9 @@ func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoginTwitch() (string, error) {
|
|
||||||
baseURL, _ := url.Parse(constant.TwitchAuthURL)
|
|
||||||
params := url.Values{}
|
|
||||||
params.Add("client_id", config.Config.Twitch.ClientID)
|
|
||||||
params.Add("client_secret", config.Config.Twitch.ClientSecret)
|
|
||||||
params.Add("grant_type", "client_credentials")
|
|
||||||
baseURL.RawQuery = params.Encode()
|
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: baseURL.String(),
|
|
||||||
Method: "POST",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"User-Agent": "",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
data := struct {
|
|
||||||
AccessToken string `json:"access_token"`
|
|
||||||
}{}
|
|
||||||
err = json.Unmarshal(resp.Data, &data)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return data.AccessToken, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetIGDBCompany(id int) (string, error) {
|
func GetIGDBCompany(id int) (string, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
resp, err := igdbFetch(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBCompaniesURL,
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`where id=%v; fields *;`, id),
|
|
||||||
Method: "POST",
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@ -440,23 +424,7 @@ func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, er
|
|||||||
|
|
||||||
func GetIGDBIDBySteamID(id int) (int, error) {
|
func GetIGDBIDBySteamID(id int) (int, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBWebsitesURL,
|
|
||||||
Method: "POST",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
@ -496,12 +464,6 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
|
|||||||
|
|
||||||
func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
|
func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
conditionBuilder := strings.Builder{}
|
conditionBuilder := strings.Builder{}
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
|
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
|
||||||
@ -509,17 +471,7 @@ func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
|
|||||||
}
|
}
|
||||||
condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
|
condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
|
||||||
respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
|
respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
resp, err := igdbFetch(constant.IGDBWebsitesURL, respBody)
|
||||||
Url: constant.IGDBWebsitesURL,
|
|
||||||
Method: "POST",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: respBody,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -595,31 +547,15 @@ func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
|
|||||||
// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
|
// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
|
||||||
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
|
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
resp, err := igdbFetch(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType))
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
type IgdbPopularity struct {
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBPopularityURL,
|
|
||||||
Method: "POST",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
type IGDBPopularity struct {
|
|
||||||
GameID int `json:"game_id"`
|
GameID int `json:"game_id"`
|
||||||
Value float64 `json:"value"`
|
Value float64 `json:"value"`
|
||||||
}
|
}
|
||||||
var data []IGDBPopularity
|
var data []IgdbPopularity
|
||||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -202,23 +202,7 @@ func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, e
|
|||||||
|
|
||||||
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
|
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
|
||||||
var err error
|
var err error
|
||||||
if TwitchToken == "" {
|
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))
|
||||||
TwitchToken, err = LoginTwitch()
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
resp, err := utils.Fetch(utils.FetchConfig{
|
|
||||||
Url: constant.IGDBWebsitesURL,
|
|
||||||
Method: "POST",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"Client-ID": config.Config.Twitch.ClientID,
|
|
||||||
"Authorization": "Bearer " + TwitchToken,
|
|
||||||
"User-Agent": "",
|
|
||||||
"Content-Type": "text/plain",
|
|
||||||
},
|
|
||||||
Data: fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user