From 71a2ac545b3cf1877d7850f8ced0551eab9295dc Mon Sep 17 00:00:00 2001 From: nite07 Date: Tue, 26 Nov 2024 23:14:15 +0800 Subject: [PATCH] modify igdb.go steam.go --- crawler/igdb.go | 240 +++++++++++++++++------------------------------ crawler/steam.go | 18 +--- 2 files changed, 89 insertions(+), 169 deletions(-) diff --git a/crawler/igdb.go b/crawler/igdb.go index 1254b45..027c6e3 100644 --- a/crawler/igdb.go +++ b/crawler/igdb.go @@ -9,6 +9,8 @@ import ( "runtime/debug" "strconv" "strings" + "sync" + "time" "pcgamedb/cache" "pcgamedb/config" @@ -18,43 +20,98 @@ import ( "pcgamedb/utils" ) -var TwitchToken string +type twitchToken struct { + token string + expires time.Time + once sync.Once +} -func getIGDBID(name string) (int, error) { - var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() +func (t *twitchToken) getToken() (string, error) { + t.once.Do(func() { + if config.Config.RedisAvaliable { + if dataBytes, exist := cache.Get("twitch_token"); exist { + _ = json.Unmarshal([]byte(dataBytes), &token) + } + } + }) + if t.token == "" || time.Now().After(t.expires) { + token, expires, err := loginTwitch() if err != nil { - return 0, fmt.Errorf("failed to login twitch: %w", err) + return "", fmt.Errorf("failed to login twitch: %w", err) + } + t.token = token + t.expires = expires + j, err := json.Marshal(t) + if err == nil { + _ = cache.Add("twitch_token", j) } } + return t.token, nil +} + +func loginTwitch() (string, time.Time, error) { + baseURL, _ := url.Parse(constant.TwitchAuthURL) + params := url.Values{} + params.Add("client_id", config.Config.Twitch.ClientID) + params.Add("client_secret", config.Config.Twitch.ClientSecret) + params.Add("grant_type", "client_credentials") + baseURL.RawQuery = params.Encode() resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBSearchURL, + Url: baseURL.String(), + Method: "POST", + Headers: map[string]string{ + "User-Agent": "", + }, + }) + if err != nil { + return "", time.Time{}, err + } + data := struct { + AccessToken string `json:"access_token"` + ExpiresIn int64 `json:"expires_in"` + TokenType string `json:"token_type"` + }{} + err = json.Unmarshal(resp.Data, &data) + if err != nil { + return "", time.Time{}, err + } + return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil +} + +func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) { + t, err := token.getToken() + if err != nil { + return nil, err + } + resp, err := utils.Fetch(utils.FetchConfig{ + Url: URL, Headers: map[string]string{ "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, + "Authorization": "Bearer " + t, "User-Agent": "", "Content-Type": "text/plain", }, - Data: fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name), + Data: dataBody, Method: "POST", }) - if string(resp.Data) == "[]" { - resp, err = utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBSearchURL, - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf(`search "%s"; fields *; limit 50;`, name), - Method: "POST", - }) + if err != nil { + return nil, err } + return resp, nil +} + +var token = twitchToken{} + +func getIGDBID(name string) (int, error) { + var err error + resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name)) if err != nil { return 0, err } + if string(resp.Data) == "[]" { + resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name)) + } + var data model.IGDBSearches if err = json.Unmarshal(resp.Data, &data); err != nil { return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) @@ -115,27 +172,12 @@ func GetIGDBAppParent(id int) (int, error) { func GetIGDBAppParetns(ids []int) (map[int]int, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return nil, err - } - } idsStr := make([]string, len(ids)) for i, id := range ids { idsStr[i] = strconv.Itoa(id) } - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBGameURL, - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")), - Method: "POST", - }) + resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ","))) + if err != nil { return nil, err } @@ -226,23 +268,8 @@ func GetIGDBIDCache(name string) (int, error) { func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return nil, err - } - } - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBGameURL, - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id), - Method: "POST", - }) + resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id)) + if err != nil { return nil, err } @@ -286,52 +313,9 @@ func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) { } } -func LoginTwitch() (string, error) { - baseURL, _ := url.Parse(constant.TwitchAuthURL) - params := url.Values{} - params.Add("client_id", config.Config.Twitch.ClientID) - params.Add("client_secret", config.Config.Twitch.ClientSecret) - params.Add("grant_type", "client_credentials") - baseURL.RawQuery = params.Encode() - resp, err := utils.Fetch(utils.FetchConfig{ - Url: baseURL.String(), - Method: "POST", - Headers: map[string]string{ - "User-Agent": "", - }, - }) - if err != nil { - return "", err - } - data := struct { - AccessToken string `json:"access_token"` - }{} - err = json.Unmarshal(resp.Data, &data) - if err != nil { - return "", err - } - return data.AccessToken, nil -} - func GetIGDBCompany(id int) (string, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return "", err - } - } - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBCompaniesURL, - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf(`where id=%v; fields *;`, id), - Method: "POST", - }) + resp, err := igdbFetch(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id)) if err != nil { return "", err } @@ -440,23 +424,7 @@ func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, er func GetIGDBIDBySteamID(id int) (int, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return 0, err - } - } - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBWebsitesURL, - Method: "POST", - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id), - }) + resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id)) if err != nil { return 0, err } @@ -496,12 +464,6 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) { func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return nil, err - } - } conditionBuilder := strings.Builder{} for _, id := range ids { conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id)) @@ -509,17 +471,7 @@ func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) { } condition := strings.TrimSuffix(conditionBuilder.String(), " | ") respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition) - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBWebsitesURL, - Method: "POST", - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: respBody, - }) + resp, err := igdbFetch(constant.IGDBWebsitesURL, respBody) if err != nil { return nil, err } @@ -595,31 +547,15 @@ func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) { // popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists. func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return nil, err - } - } - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBPopularityURL, - Method: "POST", - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType), - }) + resp, err := igdbFetch(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType)) if err != nil { return nil, err } - type IGDBPopularity struct { + type IgdbPopularity struct { GameID int `json:"game_id"` Value float64 `json:"value"` } - var data []IGDBPopularity + var data []IgdbPopularity if err = json.Unmarshal(resp.Data, &data); err != nil { return nil, err } diff --git a/crawler/steam.go b/crawler/steam.go index fd5e0d1..1975d7a 100644 --- a/crawler/steam.go +++ b/crawler/steam.go @@ -202,23 +202,7 @@ func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, e func GetSteamIDByIGDBID(IGDBID int) (int, error) { var err error - if TwitchToken == "" { - TwitchToken, err = LoginTwitch() - if err != nil { - return 0, err - } - } - resp, err := utils.Fetch(utils.FetchConfig{ - Url: constant.IGDBWebsitesURL, - Method: "POST", - Headers: map[string]string{ - "Client-ID": config.Config.Twitch.ClientID, - "Authorization": "Bearer " + TwitchToken, - "User-Agent": "", - "Content-Type": "text/plain", - }, - Data: fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID), - }) + resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID)) if err != nil { return 0, err }