remove cache-related judgments and simplify the code

cache/redis.go

@@ -16,9 +16,6 @@ var cache *redis.Client
 var mutx = &sync.RWMutex{}
 
 func connect() {
-	if !config.Config.RedisAvaliable {
-		return
-	}
 	cache = redis.NewClient(&redis.Options{
 		Addr:     fmt.Sprintf("%s:%d", config.Config.Redis.Host, config.Config.Redis.Port),
 		Password: config.Config.Redis.Password,
@@ -69,14 +66,14 @@ func Get(key string) (string, bool) {
 	return value, true
 }
 
-func Add(key string, value interface{}) error {
+func Set(key string, value interface{}) error {
 	CheckConnect()
 	ctx := context.Background()
 	cmd := cache.Set(ctx, key, value, 0)
 	return cmd.Err()
 }
 
-func AddWithExpire(key string, value interface{}, expire time.Duration) error {
+func SetWithExpire(key string, value interface{}, expire time.Duration) error {
 	CheckConnect()
 	ctx := context.Background()
 	cmd := cache.Set(ctx, key, value, expire)

config/config.go

@@ -20,10 +20,7 @@ type config struct {
 	Twitch             twitch             `json:"twitch"`
 	Webhooks           webhooks           `json:"webhooks"`
 	CFClearanceScraper cfClearanceScraper `json:"cf_clearance_scraper"`
-	DatabaseAvaliable  bool
-	OnlineFixAvaliable bool
 	MegaAvaliable      bool
-	RedisAvaliable     bool
 }
 
 type cfClearanceScraper struct {
@@ -97,11 +94,20 @@ func init() {
 		}
 	}
 	loadEnvVariables(&Config)
-	Config.OnlineFixAvaliable = Config.OnlineFix.User != "" && Config.OnlineFix.Password != ""
-	Config.RedisAvaliable = Config.Redis.Host != ""
-	Config.DatabaseAvaliable = Config.Database.Database != "" && Config.Database.Host != ""
 
-	if Config.CFClearanceScraper.Url != "" && !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") {
+	if Config.OnlineFix.User == "" || Config.OnlineFix.Password == "" {
+		panic("Need OnlineFix User and Password")
+	}
+	if Config.Redis.Host == "" {
+		panic("Need Redis Host")
+	}
+	if Config.Database.Database == "" || Config.Database.Host == "" {
+		panic("Need Database Name and Host")
+	}
+	if Config.CFClearanceScraper.Url == "" {
+		panic("Need CF Clearance Scraper URL")
+	}
+	if !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") {
 		Config.CFClearanceScraper.Url += "/cf-clearance-scraper"
 	}
 }

crawler/freegog.go

@@ -32,7 +32,7 @@ func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
 	}
 }
 
-func (c *FreeGOGCrawler) getWAFSession() (*ccs.Session, error) {
+func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) {
 	var session ccs.Session
 	var err error
 	if val, exist := cache.Get("freegog_waf_session"); exist {
@@ -45,8 +45,10 @@ func (c *FreeGOGCrawler) getWAFSession() (*ccs.Session, error) {
 		if err != nil {
 			return nil, err
 		}
-		jsonBytes, _ := json.Marshal(session)
+		jsonBytes, err := json.Marshal(session)
-		_ = cache.AddWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour)
+		if err == nil {
+			_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 24*time.Hour)
+		}
 	}
 	return &session, nil
 }
@@ -57,7 +59,7 @@ func (c *FreeGOGCrawler) Name() string {
 
 func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
 	count := 0
-	session, err := c.getWAFSession()
+	session, err := c.getSession()
 	if err != nil {
 		c.logger.Error("Failed to create session", zap.Error(err))
 		return nil, err
@@ -111,7 +113,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
 }
 
 func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
-	session, err := c.getWAFSession()
+	session, err := c.getSession()
 	if err != nil {
 		return nil, err
 	}

crawler/game.go

@@ -38,7 +38,7 @@ func OrganizeGameItem(game *model.GameItem) error {
 	if err == nil {
 		if item.SteamID == 0 {
 			// get steam id from igdb
-			steamID, err := GetSteamIDByIGDBIDCache(item.IGDBID)
+			steamID, err := GetSteamIDByIGDBID(item.IGDBID)
 			if err == nil {
 				item.SteamID = steamID
 			}
@@ -81,13 +81,13 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo
 		return nil, err
 	}
 	if platform == "igdb" {
-		steamID, err := GetSteamIDByIGDBIDCache(platformID)
+		steamID, err := GetSteamIDByIGDBID(platformID)
 		if err == nil {
 			info.SteamID = steamID
 		}
 	}
 	if platform == "steam" {
-		igdbID, err := GetIGDBIDBySteamAppIDCache(platformID)
+		igdbID, err := GetIGDBIDBySteamAppID(platformID)
 		if err == nil {
 			info.IGDBID = igdbID
 		}
@@ -116,7 +116,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
 	for _, info := range infos {
 		changed := false
 		if info.IGDBID != 0 && info.SteamID == 0 {
-			steamID, err := GetSteamIDByIGDBIDCache(info.IGDBID)
+			steamID, err := GetSteamIDByIGDBID(info.IGDBID)
 			time.Sleep(time.Millisecond * 100)
 			if err != nil {
 				continue
@@ -125,7 +125,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
 			changed = true
 		}
 		if info.SteamID != 0 && info.IGDBID == 0 {
-			igdbID, err := GetIGDBIDBySteamAppIDCache(info.SteamID)
+			igdbID, err := GetIGDBIDBySteamAppID(info.SteamID)
 			time.Sleep(time.Millisecond * 100)
 			if err != nil {
 				continue

crawler/igdb.go

@@ -6,11 +6,9 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"regexp"
 	"runtime/debug"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"pcgamedb/cache"
@@ -25,37 +23,23 @@ import (
 )
 
 type twitchToken struct {
-	Token   string    `json:"token"`
-	Expires time.Time `json:"expires"`
-	once    sync.Once
 }
 
 var token = twitchToken{}
 
 func (t *twitchToken) getToken() (string, error) {
-	t.once.Do(func() {
+	if val, exist := cache.Get("twitch_token"); exist {
-		if config.Config.RedisAvaliable {
+		return val, nil
-			if dataBytes, exist := cache.Get("twitch_token"); exist {
-				_ = json.Unmarshal([]byte(dataBytes), &token)
 	}
-		}
-	})
-	if t.Token == "" || time.Now().After(t.Expires) {
 	token, expires, err := loginTwitch()
 	if err != nil {
 		return "", fmt.Errorf("failed to login twitch: %w", err)
 	}
-		t.Token = token
+	_ = cache.SetWithExpire("twitch_token", token, expires)
-		t.Expires = expires
+	return token, nil
-		j, err := json.Marshal(t)
-		if err == nil {
-			_ = cache.Add("twitch_token", j)
-		}
-	}
-	return t.Token, nil
 }
 
-func loginTwitch() (string, time.Time, error) {
+func loginTwitch() (string, time.Duration, error) {
 	baseURL, _ := url.Parse(constant.TwitchAuthURL)
 	params := url.Values{}
 	params.Add("client_id", config.Config.Twitch.ClientID)
@@ -64,7 +48,7 @@ func loginTwitch() (string, time.Time, error) {
 	baseURL.RawQuery = params.Encode()
 	resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
 	if err != nil {
-		return "", time.Time{}, err
+		return "", 0, err
 	}
 	data := struct {
 		AccessToken string `json:"access_token"`
@@ -73,9 +57,9 @@ func loginTwitch() (string, time.Time, error) {
 	}{}
 	err = json.Unmarshal(resp.Body(), &data)
 	if err != nil {
-		return "", time.Time{}, err
+		return "", 0, err
 	}
-	return data.AccessToken, time.Now().Add(time.Second * time.Duration(data.ExpiresIn)), nil
+	return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
 }
 
 func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
@@ -113,7 +97,7 @@ func getIGDBID(name string) (int, error) {
 		return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
 	}
 	if len(data) == 1 {
-		return GetIGDBAppParentCache(data[0].Game)
+		return GetIGDBAppParent(data[0].Game)
 	}
 	maxSimilairty := 0.0
 	maxSimilairtyIndex := 0
@@ -127,7 +111,7 @@ func getIGDBID(name string) (int, error) {
 				maxSimilairtyIndex = i
 			}
 		}
-		detail, err := GetIGDBAppDetailCache(item.Game)
+		detail, err := GetIGDBAppDetail(item.Game)
 		if err != nil {
 			return 0, err
 		}
@@ -141,7 +125,7 @@ func getIGDBID(name string) (int, error) {
 		}
 	}
 	if maxSimilairty >= 0.8 {
-		return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game)
+		return GetIGDBAppParent(data[maxSimilairtyIndex].Game)
 	}
 	return 0, fmt.Errorf("IGDB ID not found: %s", name)
 }
@@ -212,10 +196,10 @@ func getIGDBIDBySteamSearch(name string) (int, error) {
 	}
 	if maxSim != 0 {
 		if maxSimItem.Type == "App" {
-			return GetIGDBIDBySteamAppIDCache(maxSimItem.ID)
+			return GetIGDBIDBySteamAppID(maxSimItem.ID)
 		}
 		if maxSimItem.Type == "Bundle" {
-			return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID)
+			return GetIGDBIDBySteamBundleID(maxSimItem.ID)
 		}
 	}
 	return 0, fmt.Errorf("steam ID not found: %s", name)
@@ -223,60 +207,6 @@ func getIGDBIDBySteamSearch(name string) (int, error) {
 
 // GetIGDBAppParent returns the parent of the game, if no parent return itself
 func GetIGDBAppParent(id int) (int, error) {
-	detail, err := GetIGDBAppDetailCache(id)
-	if err != nil {
-		return 0, err
-	}
-	hasParent := false
-	for detail.VersionParent != 0 {
-		hasParent = true
-		detail, err = GetIGDBAppDetailCache(detail.VersionParent)
-		if err != nil {
-			return 0, err
-		}
-	}
-	if hasParent {
-		return detail.ID, nil
-	}
-	return id, nil
-}
-
-func GetIGDBAppParetns(ids []int) (map[int]int, error) {
-	var err error
-	idsStr := make([]string, len(ids))
-	for i, id := range ids {
-		idsStr[i] = strconv.Itoa(id)
-	}
-	resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=(%s) ;fields version_parent;`, strings.Join(idsStr, ",")))
-
-	if err != nil {
-		return nil, err
-	}
-	var data []struct {
-		ID            int `json:"id"`
-		VersionParent int `json:"version_parent"`
-	}
-	if err = json.Unmarshal(resp.Body(), &data); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
-	}
-	parents := make(map[int]int)
-	for _, item := range data {
-		if item.VersionParent != 0 {
-			pid, err := GetIGDBAppParentCache(item.VersionParent)
-			if err != nil {
-				parents[item.ID] = item.ID
-			} else {
-				parents[item.ID] = pid
-			}
-		} else {
-			parents[item.ID] = item.ID
-		}
-	}
-	return parents, nil
-}
-
-func GetIGDBAppParentCache(id int) (int, error) {
-	if config.Config.RedisAvaliable {
 	key := fmt.Sprintf("igdb_parent:%d", id)
 	val, exist := cache.Get(key)
 	if exist {
@@ -285,20 +215,35 @@ func GetIGDBAppParentCache(id int) (int, error) {
 			return 0, err
 		}
 		return id, nil
-		} else {
+	}
-			id, err := GetIGDBAppParent(id)
+	detail, err := GetIGDBAppDetail(id)
 	if err != nil {
 		return 0, err
 	}
-			_ = cache.Add(key, id)
+	hasParent := false
+	for detail.VersionParent != 0 {
+		hasParent = true
+		detail, err = GetIGDBAppDetail(detail.VersionParent)
+		if err != nil {
+			return 0, err
+		}
+	}
+	if hasParent {
+		return detail.ID, nil
+	}
+
+	_ = cache.Set(key, id)
+
 	return id, nil
-		}
-	}
-	return GetIGDBAppParent(id)
 }
 
 // GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
 func GetIGDBID(name string) (int, error) {
+	key := fmt.Sprintf("igdb_id:%s", name)
+	val, exist := cache.Get(key)
+	if exist {
+		return strconv.Atoi(val)
+	}
 	name1 := name
 	name2 := FormatName(name)
 	names := []string{name1}
@@ -308,42 +253,30 @@ func GetIGDBID(name string) (int, error) {
 	for _, name := range names {
 		id, err := getIGDBID(name)
 		if err == nil {
+			_ = cache.Set(key, id)
 			return id, nil
 		}
 	}
 	for _, name := range names {
 		id, err := getIGDBIDBySteamSearch(name)
 		if err == nil {
+			_ = cache.Set(key, id)
 			return id, nil
 		}
 	}
 	return 0, errors.New("IGDB ID not found")
 }
 
-func GetIGDBIDCache(name string) (int, error) {
+func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
-	if config.Config.RedisAvaliable {
+	key := fmt.Sprintf("igdb_game:%v", id)
-		key := fmt.Sprintf("igdb_id:%s", name)
 	val, exist := cache.Get(key)
 	if exist {
-			id, err := strconv.Atoi(val)
+		var data model.IGDBGameDetail
-			if err != nil {
+		if err := json.Unmarshal([]byte(val), &data); err != nil {
-				return 0, err
+			return nil, err
 		}
-			return id, nil
+		return &data, nil
-		} else {
-			id, err := GetIGDBID(name)
-			if err != nil {
-				return 0, err
 	}
-			_ = cache.Add(key, id)
-			return id, nil
-		}
-	} else {
-		return GetIGDBID(name)
-	}
-}
-
-func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
 	var err error
 	resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id))
 
@@ -360,37 +293,21 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
 	if data[0].Name == "" {
 		return GetIGDBAppDetail(id)
 	}
+
+	jsonBytes, err := json.Marshal(data[0])
+	if err == nil {
+		_ = cache.Set(key, string(jsonBytes))
+	}
+
 	return data[0], nil
 }
 
-func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
+func GetIGDBCompany(id int) (string, error) {
-	if config.Config.RedisAvaliable {
+	key := fmt.Sprintf("igdb_companies:%v", id)
-		key := fmt.Sprintf("igdb_game:%v", id)
 	val, exist := cache.Get(key)
 	if exist {
-			var data model.IGDBGameDetail
+		return val, nil
-			if err := json.Unmarshal([]byte(val), &data); err != nil {
-				return nil, err
 	}
-			return &data, nil
-		} else {
-			data, err := GetIGDBAppDetail(id)
-			if err != nil {
-				return nil, err
-			}
-			dataBytes, err := json.Marshal(data)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour)
-			return data, nil
-		}
-	} else {
-		return GetIGDBAppDetail(id)
-	}
-}
-
-func GetIGDBCompany(id int) (string, error) {
 	var err error
 	resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
 	if err != nil {
@@ -406,31 +323,15 @@ func GetIGDBCompany(id int) (string, error) {
 	if data[0].Name == "" {
 		return GetIGDBCompany(id)
 	}
-	return data[0].Name, nil
-}
 
-func GetIGDBCompanyCache(id int) (string, error) {
+	_ = cache.Set(key, data[0].Name)
-	if config.Config.RedisAvaliable {
+
-		key := fmt.Sprintf("igdb_companies:%v", id)
+	return data[0].Name, nil
-		val, exist := cache.Get(key)
-		if exist {
-			return val, nil
-		} else {
-			data, err := GetIGDBCompany(id)
-			if err != nil {
-				return "", err
-			}
-			_ = cache.Add(key, data)
-			return data, nil
-		}
-	} else {
-		return GetIGDBCompany(id)
-	}
 }
 
 func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
 	item := &model.GameInfo{}
-	detail, err := GetIGDBAppDetailCache(id)
+	detail, err := GetIGDBAppDetail(id)
 	if err != nil {
 		return nil, err
 	}
@@ -460,7 +361,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
 
 	for _, company := range detail.InvolvedCompanies {
 		if company.Developer || company.Publisher {
-			companyName, err := GetIGDBCompanyCache(company.Company)
+			companyName, err := GetIGDBCompany(company.Company)
 			if err != nil {
 				continue
 			}
@@ -478,7 +379,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
 
 // OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
 func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
-	id, err := GetIGDBIDCache(game.Name)
+	id, err := GetIGDBID(game.Name)
 	if err != nil {
 		return nil, err
 	}
@@ -498,6 +399,11 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
 }
 
 func GetIGDBIDBySteamAppID(id int) (int, error) {
+	key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
+	val, exist := cache.Get(key)
+	if exist {
+		return strconv.Atoi(val)
+	}
 	var err error
 	resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
 	if err != nil {
@@ -515,29 +421,19 @@ func GetIGDBIDBySteamAppID(id int) (int, error) {
 	if data[0].Game == 0 {
 		return GetIGDBIDBySteamAppID(id)
 	}
-	return GetIGDBAppParentCache(data[0].Game)
-}
 
-func GetIGDBIDBySteamAppIDCache(id int) (int, error) {
+	_ = cache.Set(key, strconv.Itoa(data[0].Game))
-	if config.Config.RedisAvaliable {
+
-		key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
+	return GetIGDBAppParent(data[0].Game)
-		val, exist := cache.Get(key)
-		if exist {
-			return strconv.Atoi(val)
-		} else {
-			data, err := GetIGDBIDBySteamAppID(id)
-			if err != nil {
-				return 0, err
-			}
-			_ = cache.Add(key, strconv.Itoa(data))
-			return data, nil
-		}
-	} else {
-		return GetIGDBIDBySteamAppID(id)
-	}
 }
 
 func GetIGDBIDBySteamBundleID(id int) (int, error) {
+	key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
+	val, exist := cache.Get(key)
+	if exist {
+		return strconv.Atoi(val)
+	}
+
 	var err error
 	resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
 	if err != nil {
@@ -555,104 +451,10 @@ func GetIGDBIDBySteamBundleID(id int) (int, error) {
 	if data[0].Game == 0 {
 		return GetIGDBIDBySteamBundleID(id)
 	}
-	return GetIGDBAppParentCache(data[0].Game)
-}
 
-func GetIGDBIDBySteamBundleIDCache(id int) (int, error) {
+	_ = cache.Set(key, strconv.Itoa(data[0].Game))
-	if config.Config.RedisAvaliable {
-		key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
-		val, exist := cache.Get(key)
-		if exist {
-			return strconv.Atoi(val)
-		} else {
-			data, err := GetIGDBIDBySteamBundleID(id)
-			if err != nil {
-				return 0, err
-			}
-			_ = cache.Add(key, strconv.Itoa(data))
-			return data, nil
-		}
-	} else {
-		return GetIGDBIDBySteamBundleID(id)
-	}
-}
 
-func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
+	return GetIGDBAppParent(data[0].Game)
-	var err error
-	conditionBuilder := strings.Builder{}
-	for _, id := range ids {
-		conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
-		conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id))
-	}
-	condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
-	respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
-	resp, err := igdbRequest(constant.IGDBWebsitesURL, respBody)
-	if err != nil {
-		return nil, err
-	}
-	var data []struct {
-		Game int    `json:"game"`
-		Url  string `json:"url"`
-	}
-	if err = json.Unmarshal(resp.Body(), &data); err != nil {
-		return nil, err
-	}
-	ret := make(map[int]int)
-	regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
-	for _, d := range data {
-		idStr := regex.FindStringSubmatch(d.Url)
-		if len(idStr) < 2 {
-			continue
-		}
-		id, err := strconv.Atoi(idStr[1])
-		if err == nil {
-			pid, err := GetIGDBAppParentCache(d.Game)
-			if err == nil {
-				ret[id] = pid
-			} else {
-				ret[id] = 0
-			}
-		}
-	}
-	for _, id := range ids {
-		if _, ok := ret[id]; !ok {
-			ret[id] = 0
-		}
-	}
-	return ret, nil
-}
-
-func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
-	res := make(map[int]int)
-	notExistIDs := make([]int, 0)
-	if config.Config.RedisAvaliable {
-		for _, steamID := range ids {
-			key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID)
-			val, exist := cache.Get(key)
-			if exist {
-				igdbID, _ := strconv.Atoi(val)
-				res[steamID] = igdbID
-			} else {
-				notExistIDs = append(notExistIDs, steamID)
-			}
-		}
-		if len(res) == len(ids) {
-			return res, nil
-		}
-		idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
-		if err != nil {
-			return nil, err
-		}
-		for steamID, igdbID := range idMap {
-			res[steamID] = igdbID
-			if igdbID != 0 {
-				_ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID)
-			}
-		}
-		return res, nil
-	} else {
-		return GetIGDBIDsBySteamIDs(ids)
-	}
 }
 
 // GetIGDBPopularGameIDs get IGDB popular game IDs
@@ -676,7 +478,7 @@ func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, er
 	}
 	ret := make([]int, 0)
 	for _, d := range data {
-		pid, err := GetIGDBAppParentCache(d.GameID)
+		pid, err := GetIGDBAppParent(d.GameID)
 		if err != nil {
 			ret = append(ret, d.GameID)
 			continue
@@ -685,30 +487,3 @@ func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, er
 	}
 	return ret, nil
 }
-
-func GetIGDBPopularGameIDsCache(popularityType int, offset int, limit int) ([]int, error) {
-	if config.Config.RedisAvaliable {
-		key := fmt.Sprintf("igdb_popular_game_ids:%v:%v:%v", popularityType, offset, limit)
-		val, exist := cache.Get(key)
-		if exist {
-			var data []int
-			if err := json.Unmarshal([]byte(val), &data); err != nil {
-				return nil, err
-			}
-			return data, nil
-		} else {
-			data, err := GetIGDBPopularGameIDs(popularityType, offset, limit)
-			if err != nil {
-				return nil, err
-			}
-			jsonBytes, err := json.Marshal(data)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, jsonBytes, 12*time.Hour)
-			return data, nil
-		}
-	} else {
-		return GetIGDBPopularGameIDs(popularityType, offset, limit)
-	}
-}

crawler/onlinefix.go

@@ -39,10 +39,6 @@ func (c *OnlineFixCrawler) Name() string {
 }
 
 func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
-	if !config.Config.OnlineFixAvaliable {
-		c.logger.Error("Need Online Fix account")
-		return nil, errors.New("online Fix is not available")
-	}
 	cookies, err := c.getCookies()
 	if err != nil {
 		return nil, err

crawler/steam.go

@@ -11,13 +11,22 @@ import (
 	"time"
 
 	"pcgamedb/cache"
-	"pcgamedb/config"
 	"pcgamedb/constant"
 	"pcgamedb/model"
 	"pcgamedb/utils"
 )
 
 func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
+	key := fmt.Sprintf("steam_game:%d", id)
+	val, exist := cache.Get(key)
+	if exist {
+		var detail model.SteamAppDetail
+		if err := json.Unmarshal([]byte(val), &detail); err != nil {
+			return nil, err
+		}
+		return &detail, nil
+	}
+
 	baseURL, _ := url.Parse(constant.SteamAppDetailURL)
 	params := url.Values{}
 	params.Add("appids", strconv.Itoa(id))
@@ -39,39 +48,18 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
 	if detail[strconv.Itoa(id)] == nil {
 		return nil, fmt.Errorf("steam App not found: %d", id)
 	}
-	return detail[strconv.Itoa(id)], nil
-}
 
-func GetSteamAppDetailCache(id int) (*model.SteamAppDetail, error) {
+	jsonBytes, err := json.Marshal(detail[strconv.Itoa(id)])
-	if config.Config.RedisAvaliable {
+	if err == nil {
-		key := fmt.Sprintf("steam_game:%d", id)
+		_ = cache.Set(key, string(jsonBytes))
-		val, exist := cache.Get(key)
-		if exist {
-			var detail model.SteamAppDetail
-			if err := json.Unmarshal([]byte(val), &detail); err != nil {
-				return nil, err
-			}
-			return &detail, nil
-		} else {
-			data, err := GetSteamAppDetail(id)
-			if err != nil {
-				return nil, err
-			}
-			dataBytes, err := json.Marshal(data)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.Add(key, dataBytes)
-			return data, nil
-		}
-	} else {
-		return GetSteamAppDetail(id)
 	}
+
+	return detail[strconv.Itoa(id)], nil
 }
 
 func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
 	item := &model.GameInfo{}
-	detail, err := GetSteamAppDetailCache(id)
+	detail, err := GetSteamAppDetail(id)
 	if err != nil {
 		return nil, err
 	}
@@ -91,6 +79,16 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
 }
 
 func GetSteamIDByIGDBID(IGDBID int) (int, error) {
+	key := fmt.Sprintf("steam_game:%d", IGDBID)
+	val, exist := cache.Get(key)
+	if exist {
+		id, err := strconv.Atoi(val)
+		if err != nil {
+			return 0, err
+		}
+		return id, nil
+	}
+
 	var err error
 	resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))
 	if err != nil {
@@ -117,32 +115,9 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) {
 			if err != nil {
 				return 0, err
 			}
+			_ = cache.Set(key, strconv.Itoa(steamID))
 			return steamID, nil
 		}
 	}
 	return 0, errors.New("not found")
 }
-
-func GetSteamIDByIGDBIDCache(IGDBID int) (int, error) {
-	if config.Config.RedisAvaliable {
-		key := fmt.Sprintf("steam_game:%d", IGDBID)
-		val, exist := cache.Get(key)
-		if exist {
-			id, err := strconv.Atoi(val)
-			if err != nil {
-				return 0, err
-			}
-			return id, nil
-		} else {
-			id, err := GetSteamIDByIGDBID(IGDBID)
-			if err != nil {
-				return 0, err
-			}
-			dataBytes := strconv.Itoa(id)
-			_ = cache.Add(key, dataBytes)
-			return id, nil
-		}
-	} else {
-		return GetSteamIDByIGDBID(IGDBID)
-	}
-}

crawler/steam250.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net/url"
 	"pcgamedb/cache"
-	"pcgamedb/config"
 	"pcgamedb/db"
 	"regexp"
 	"strconv"
@@ -19,6 +19,16 @@ import (
 )
 
 func GetSteam250(URL string) ([]*model.GameInfo, error) {
+	key := "steam250:" + url.QueryEscape(URL)
+	if val, ok := cache.Get(key); ok {
+		var infos []*model.GameInfo
+		err := json.Unmarshal([]byte(val), &infos)
+		if err != nil {
+			return nil, err
+		}
+		return infos, nil
+	}
+
 	resp, err := utils.Request().Get(URL)
 	if err != nil {
 		return nil, err
@@ -48,6 +58,12 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
 	if len(infos) > 10 {
 		return infos[:10], nil
 	}
+
+	jsonBytes, err := json.Marshal(infos)
+	if err == nil {
+		_ = cache.SetWithExpire(key, string(jsonBytes), 12*time.Hour)
+	}
+
 	return infos, nil
 }
 
@@ -55,158 +71,18 @@ func GetSteam250Top250() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250Top250URL)
 }
 
-func GetSteam250Top250Cache() ([]*model.GameInfo, error) {
-	if config.Config.RedisAvaliable {
-		key := "steam250_top250"
-		val, exist := cache.Get(key)
-		if exist {
-			var infos []*model.GameInfo
-			err := json.Unmarshal([]byte(val), &infos)
-			if err != nil {
-				return nil, err
-			}
-			return infos, nil
-		} else {
-			infos, err := GetSteam250Top250()
-			if err != nil {
-				return nil, err
-			}
-			jsonBytes, err := json.Marshal(infos)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
-			return infos, nil
-		}
-	} else {
-		return GetSteam250Top250()
-	}
-}
-
 func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) {
 	return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, time.Now().UTC().Year()))
 }
 
-func GetSteam250BestOfTheYearCache() ([]*model.GameInfo, error) {
-	if config.Config.RedisAvaliable {
-		key := "steam250_best_of_the_year"
-		val, exist := cache.Get(key)
-		if exist {
-			var infos []*model.GameInfo
-			err := json.Unmarshal([]byte(val), &infos)
-			if err != nil {
-				return nil, err
-			}
-			return infos, nil
-		} else {
-			infos, err := GetSteam250BestOfTheYear()
-			if err != nil {
-				return nil, err
-			}
-			jsonBytes, err := json.Marshal(infos)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
-			return infos, nil
-		}
-	} else {
-		return GetSteam250BestOfTheYear()
-	}
-}
-
 func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250WeekTop50URL)
 }
 
-func GetSteam250WeekTop50Cache() ([]*model.GameInfo, error) {
-	if config.Config.RedisAvaliable {
-		key := "steam250_week_top50"
-		val, exist := cache.Get(key)
-		if exist {
-			var infos []*model.GameInfo
-			err := json.Unmarshal([]byte(val), &infos)
-			if err != nil {
-				return nil, err
-			}
-			return infos, nil
-		} else {
-			infos, err := GetSteam250WeekTop50()
-			if err != nil {
-				return nil, err
-			}
-			jsonBytes, err := json.Marshal(infos)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
-			return infos, nil
-		}
-	} else {
-		return GetSteam250WeekTop50()
-	}
-}
-
 func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250MonthTop50URL)
 }
 
-func GetSteam250MonthTop50Cache() ([]*model.GameInfo, error) {
-	if config.Config.RedisAvaliable {
-		key := "steam250_month_top50"
-		val, exist := cache.Get(key)
-		if exist {
-			var infos []*model.GameInfo
-			err := json.Unmarshal([]byte(val), &infos)
-			if err != nil {
-				return nil, err
-			}
-			return infos, nil
-		} else {
-			infos, err := GetSteam250MonthTop50()
-			if err != nil {
-				return nil, err
-			}
-			jsonBytes, err := json.Marshal(infos)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
-			return infos, nil
-		}
-	} else {
-		return GetSteam250MonthTop50()
-	}
-}
-
 func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250MostPlayedURL)
 }
-
-func GetSteam250MostPlayedCache() ([]*model.GameInfo, error) {
-	if config.Config.RedisAvaliable {
-		key := "steam250_most_played"
-		val, exist := cache.Get(key)
-		if exist {
-			var infos []*model.GameInfo
-			err := json.Unmarshal([]byte(val), &infos)
-			if err != nil {
-				return nil, err
-			}
-			return infos, nil
-		} else {
-			infos, err := GetSteam250MostPlayed()
-			if err != nil {
-				return nil, err
-			}
-			jsonBytes, err := json.Marshal(infos)
-			if err != nil {
-				return nil, err
-			}
-			_ = cache.AddWithExpire(key, string(jsonBytes), 12*time.Hour)
-			return infos, nil
-		}
-	} else {
-		return GetSteam250MostPlayed()
-	}
-}

db/db.go

@@ -32,10 +32,6 @@ var (
 )
 
 func connect() {
-	if !config.Config.DatabaseAvaliable {
-		log.Logger.Panic("Missing database configuration information")
-		return
-	}
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	clientOptions := options.Client().ApplyURI(fmt.Sprintf(

db/game.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"pcgamedb/cache"
-	"pcgamedb/config"
 	"pcgamedb/model"
 
 	"go.mongodb.org/mongo-driver/bson"
@@ -299,6 +298,21 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
 	name = strings.TrimSpace(name)
 	name = strings.Replace(name, " ", ".*", -1)
 	name = fmt.Sprintf("%s.*", name)
+
+	key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize)
+	val, exist := cache.Get(key)
+	if exist {
+		var data struct {
+			Items     []*model.GameInfo
+			TotalPage int
+		}
+		err := json.Unmarshal([]byte(val), &data)
+		if err != nil {
+			return nil, 0, err
+		}
+		return data.Items, data.TotalPage, nil
+	}
+
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 
@@ -334,40 +348,19 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
 	if err := cursor.Err(); err != nil {
 		return nil, 0, err
 	}
-	return items, int(totalPage), nil
-}
 
-func SearchGameInfosCache(name string, page int, pageSize int) ([]*model.GameInfo, int, error) {
+	jsonBytes, err := json.Marshal(struct {
-	type res struct {
 		Items     []*model.GameInfo
 		TotalPage int
+	}{
+		Items:     items,
+		TotalPage: int(totalPage),
+	})
+	if err == nil {
+		_ = cache.SetWithExpire(key, string(jsonBytes), time.Minute*5)
 	}
-	name = strings.ToLower(name)
+
-	if config.Config.RedisAvaliable {
+	return items, int(totalPage), nil
-		key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize)
-		val, exist := cache.Get(key)
-		if exist {
-			var data res
-			err := json.Unmarshal([]byte(val), &data)
-			if err != nil {
-				return nil, 0, err
-			}
-			return data.Items, data.TotalPage, nil
-		} else {
-			data, totalPage, err := SearchGameInfos(name, page, pageSize)
-			if err != nil {
-				return nil, 0, err
-			}
-			dataBytes, err := json.Marshal(res{Items: data, TotalPage: totalPage})
-			if err != nil {
-				return nil, 0, err
-			}
-			_ = cache.AddWithExpire(key, string(dataBytes), 5*time.Minute)
-			return data, totalPage, nil
-		}
-	} else {
-		return SearchGameInfos(name, page, pageSize)
-	}
 }
 
 func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {

go.mod

@@ -25,7 +25,7 @@ require (
 )
 
 require (
-	git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 // indirect
+	git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 // indirect
 	github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 // indirect
 	github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect

go.sum

@@ -21,6 +21,10 @@ git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff h1:f5OEMRc/zhMxSfHdTJ
 git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
 git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 h1:B2ewPM44DgyrkycIrUfyTRLM7mXggA0JE8pNbvxjFKw=
 git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
+git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c h1:UcIxgKmcQGZqjTJWsQf9MVDviUQFlU+ZK6HJjpW+nAU=
+git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
+git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 h1:ZCMgQt2ILohQ3MSk6RVhnRY4fbQTJVLREbWZjLtVs+Y=
+git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 h1:6fexoGmvzoXMSk14BZ0AirapVm5c3KUsEjE0jLlVKi8=
 github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26/go.mod h1:QFi/EVO7qqru3Ftxz1LR+96jIc91Tifv0DnskF/gWQ8=

server/handler/get_popular_game_infos.go

@@ -87,7 +87,7 @@ func GetPopularGameInfosHandler(c *gin.Context) {
 			offset += 20
 			pids := make([]int, 20)
 			for _, id := range ids {
-				pid, err := crawler.GetIGDBAppParentCache(id)
+				pid, err := crawler.GetIGDBAppParent(id)
 				if err != nil {
 					continue
 				}

server/handler/healthcheck.go

@@ -24,8 +24,6 @@ type HealthCheckResponse struct {
 	GameItem      int64  `json:"game_num"`
 	GameInfo      int64  `json:"game_info_num"`
 	Unorganized   int64  `json:"unorganized_game_num"`
-	RedisAvaliable     bool   `json:"redis_avaliable"`
-	OnlineFixAvaliable bool   `json:"online_fix_avaliable"`
 	MegaAvaliable bool   `json:"mega_avaliable"`
 }
 
@@ -59,8 +57,6 @@ func HealthCheckHandler(c *gin.Context) {
 		GameItem:      downloadCount,
 		GameInfo:      infoCount,
 		Unorganized:   unorganizedCount,
-		RedisAvaliable:     config.Config.RedisAvaliable,
-		OnlineFixAvaliable: config.Config.OnlineFixAvaliable,
 		MegaAvaliable: config.Config.MegaAvaliable,
 	})
 }

server/handler/search_games.go

@@ -53,7 +53,7 @@ func SearchGamesHandler(c *gin.Context) {
 	if req.PageSize > 10 {
 		req.PageSize = 10
 	}
-	items, totalPage, err := db.SearchGameInfosCache(req.Keyword, req.Page, req.PageSize)
+	items, totalPage, err := db.SearchGameInfos(req.Keyword, req.Page, req.PageSize)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, SearchGamesResponse{
 			Status:  "error",

server/route.go

@@ -83,17 +83,17 @@ func initFrontend(app *gin.Engine) {
 
 	// Load routes
 	app.GET("/", func(ctx *gin.Context) {
-		monthTop, err := crawler.GetSteam250MonthTop50Cache()
+		monthTop, err := crawler.GetSteam250MonthTop50()
 		if err != nil {
 			ctx.HTML(500, "500.html", err)
 			return
 		}
-		mostPlayed, err := crawler.GetSteam250MostPlayedCache()
+		mostPlayed, err := crawler.GetSteam250MostPlayed()
 		if err != nil {
 			ctx.HTML(500, "500.html", err)
 			return
 		}
-		bestOfTheYear, err := crawler.GetSteam250BestOfTheYearCache()
+		bestOfTheYear, err := crawler.GetSteam250BestOfTheYear()
 		if err != nil {
 			ctx.HTML(500, "500.html", err)
 			return