modify igdb.go steam.go

crawler/igdb.go

@@ -9,6 +9,8 @@ import (
 	"runtime/debug"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
 
 	"pcgamedb/cache"
 	"pcgamedb/config"
@@ -18,43 +20,98 @@ import (
 	"pcgamedb/utils"
 )
 
-var TwitchToken string
+type twitchToken struct {
+	token   string
+	expires time.Time
+	once    sync.Once
+}
 
-func getIGDBID(name string) (int, error) {
+func (t *twitchToken) getToken() (string, error) {
-	var err error
+	t.once.Do(func() {
-	if TwitchToken == "" {
+		if config.Config.RedisAvaliable {
-		TwitchToken, err = LoginTwitch()
+			if dataBytes, exist := cache.Get("twitch_token"); exist {
+				_ = json.Unmarshal([]byte(dataBytes), &token)
+			}
+		}
+	})
+	if t.token == "" || time.Now().After(t.expires) {
+		token, expires, err := loginTwitch()
 		if err != nil {
-			return 0, fmt.Errorf("failed to login twitch: %w", err)
+			return "", fmt.Errorf("failed to login twitch: %w", err)
+		}
+		t.token = token
+		t.expires = expires
+		j, err := json.Marshal(t)
+		if err == nil {
+			_ = cache.Add("twitch_token", j)
 		}
 	}
+	return t.token, nil
+}
+
+func loginTwitch() (string, time.Time, error) {
+	baseURL, _ := url.Parse(constant.TwitchAuthURL)
+	params := url.Values{}
+	params.Add("client_id", config.Config.Twitch.ClientID)
+	params.Add("client_secret", config.Config.Twitch.ClientSecret)
+	params.Add("grant_type", "client_credentials")
+	baseURL.RawQuery = params.Encode()
 	resp, err := utils.Fetch(utils.FetchConfig{
-		Url: constant.IGDBSearchURL,
+		Url:    baseURL.String(),
+		Method: "POST",
+		Headers: map[string]string{
+			"User-Agent": "",
+		},
+	})
+	if err != nil {
+		return "", time.Time{}, err
+	}
+	data := struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int64  `json:"expires_in"`
+		TokenType   string `json:"token_type"`
+	}{}
+	err = json.Unmarshal(resp.Data, &data)
+	if err != nil {
+		return "", time.Time{}, err
+	}
+	return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil
+}
+
+func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) {
+	t, err := token.getToken()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := utils.Fetch(utils.FetchConfig{
+		Url: URL,
 		Headers: map[string]string{
 			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
+			"Authorization": "Bearer " + t,
 			"User-Agent":    "",
 			"Content-Type":  "text/plain",
 		},
-		Data:   fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name),
+		Data:   dataBody,
 		Method: "POST",
 	})
-	if string(resp.Data) == "[]" {
+	if err != nil {
-		resp, err = utils.Fetch(utils.FetchConfig{
+		return nil, err
-			Url: constant.IGDBSearchURL,
-			Headers: map[string]string{
-				"Client-ID":     config.Config.Twitch.ClientID,
-				"Authorization": "Bearer " + TwitchToken,
-				"User-Agent":    "",
-				"Content-Type":  "text/plain",
-			},
-			Data:   fmt.Sprintf(`search "%s"; fields *; limit 50;`, name),
-			Method: "POST",
-		})
 	}
+	return resp, nil
+}
+
+var token = twitchToken{}
+
+func getIGDBID(name string) (int, error) {
+	var err error
+	resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
 	if err != nil {
 		return 0, err
 	}
+	if string(resp.Data) == "[]" {
+		resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
+	}
+
 	var data model.IGDBSearches
 	if err = json.Unmarshal(resp.Data, &data); err != nil {
 		return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
@@ -115,27 +172,12 @@ func GetIGDBAppParent(id int) (int, error) {
 
 func GetIGDBAppParetns(ids []int) (map[int]int, error) {
 	var err error
-	if TwitchToken == "" {
-		TwitchToken, err = LoginTwitch()
-		if err != nil {
-			return nil, err
-		}
-	}
 	idsStr := make([]string, len(ids))
 	for i, id := range ids {
 		idsStr[i] = strconv.Itoa(id)
 	}
-	resp, err := utils.Fetch(utils.FetchConfig{
+	resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")))
-		Url: constant.IGDBGameURL,
+
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data:   fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")),
-		Method: "POST",
-	})
 	if err != nil {
 		return nil, err
 	}
@@ -226,23 +268,8 @@ func GetIGDBIDCache(name string) (int, error) {
 
 func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
 	var err error
-	if TwitchToken == "" {
+	resp, err := igdbFetch(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))
-		TwitchToken, err = LoginTwitch()
+
-		if err != nil {
-			return nil, err
-		}
-	}
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url: constant.IGDBGameURL,
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data:   fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id),
-		Method: "POST",
-	})
 	if err != nil {
 		return nil, err
 	}
@@ -286,52 +313,9 @@ func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
 	}
 }
 
-func LoginTwitch() (string, error) {
-	baseURL, _ := url.Parse(constant.TwitchAuthURL)
-	params := url.Values{}
-	params.Add("client_id", config.Config.Twitch.ClientID)
-	params.Add("client_secret", config.Config.Twitch.ClientSecret)
-	params.Add("grant_type", "client_credentials")
-	baseURL.RawQuery = params.Encode()
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url:    baseURL.String(),
-		Method: "POST",
-		Headers: map[string]string{
-			"User-Agent": "",
-		},
-	})
-	if err != nil {
-		return "", err
-	}
-	data := struct {
-		AccessToken string `json:"access_token"`
-	}{}
-	err = json.Unmarshal(resp.Data, &data)
-	if err != nil {
-		return "", err
-	}
-	return data.AccessToken, nil
-}
-
 func GetIGDBCompany(id int) (string, error) {
 	var err error
-	if TwitchToken == "" {
+	resp, err := igdbFetch(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
-		TwitchToken, err = LoginTwitch()
-		if err != nil {
-			return "", err
-		}
-	}
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url: constant.IGDBCompaniesURL,
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data:   fmt.Sprintf(`where id=%v; fields *;`, id),
-		Method: "POST",
-	})
 	if err != nil {
 		return "", err
 	}
@@ -440,23 +424,7 @@ func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, er
 
 func GetIGDBIDBySteamID(id int) (int, error) {
 	var err error
-	if TwitchToken == "" {
+	resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
-		TwitchToken, err = LoginTwitch()
-		if err != nil {
-			return 0, err
-		}
-	}
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url:    constant.IGDBWebsitesURL,
-		Method: "POST",
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data: fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id),
-	})
 	if err != nil {
 		return 0, err
 	}
@@ -496,12 +464,6 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
 
 func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
 	var err error
-	if TwitchToken == "" {
-		TwitchToken, err = LoginTwitch()
-		if err != nil {
-			return nil, err
-		}
-	}
 	conditionBuilder := strings.Builder{}
 	for _, id := range ids {
 		conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
@@ -509,17 +471,7 @@ func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
 	}
 	condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
 	respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
-	resp, err := utils.Fetch(utils.FetchConfig{
+	resp, err := igdbFetch(constant.IGDBWebsitesURL, respBody)
-		Url:    constant.IGDBWebsitesURL,
-		Method: "POST",
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data: respBody,
-	})
 	if err != nil {
 		return nil, err
 	}
@@ -595,31 +547,15 @@ func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
 // popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
 func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
 	var err error
-	if TwitchToken == "" {
+	resp, err := igdbFetch(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType))
-		TwitchToken, err = LoginTwitch()
-		if err != nil {
-			return nil, err
-		}
-	}
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url:    constant.IGDBPopularityURL,
-		Method: "POST",
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data: fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType),
-	})
 	if err != nil {
 		return nil, err
 	}
-	type IGDBPopularity struct {
+	type IgdbPopularity struct {
 		GameID int     `json:"game_id"`
 		Value  float64 `json:"value"`
 	}
-	var data []IGDBPopularity
+	var data []IgdbPopularity
 	if err = json.Unmarshal(resp.Data, &data); err != nil {
 		return nil, err
 	}

crawler/steam.go

@@ -202,23 +202,7 @@ func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, e
 
 func GetSteamIDByIGDBID(IGDBID int) (int, error) {
 	var err error
-	if TwitchToken == "" {
+	resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))
-		TwitchToken, err = LoginTwitch()
-		if err != nil {
-			return 0, err
-		}
-	}
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url:    constant.IGDBWebsitesURL,
-		Method: "POST",
-		Headers: map[string]string{
-			"Client-ID":     config.Config.Twitch.ClientID,
-			"Authorization": "Bearer " + TwitchToken,
-			"User-Agent":    "",
-			"Content-Type":  "text/plain",
-		},
-		Data: fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID),
-	})
 	if err != nil {
 		return 0, err
 	}