fix: OrganizeGameItem

fix: save twitchtoken
remove: getSteamID
remove: update cmd&api
add: getIGDBIDBySteamSearch
add: update game info task

cmd/task.go

@@ -10,32 +10,63 @@ import (
 )
 
 type taskCommandConfig struct {
-	Crawl     bool
+	Cron string
-	CrawlCron string
+	Now  bool
 }
 
-var taskCmdCfg taskCommandConfig
+var taskCommandCfg taskCommandConfig
 
-var taskCmd = &cobra.Command{
+var crawlTaskCmd = &cobra.Command{
-	Use:   "task",
+	Use:   "crawl",
-	Long:  "Start task",
+	Long:  "Start crawl task",
-	Short: "Start task",
+	Short: "Start crawl task",
 	Run: func(cmd *cobra.Command, args []string) {
-		if taskCmdCfg.Crawl {
+		if taskCommandCfg.Now {
 			task.Crawl(log.Logger)
+		}
 		c := cron.New()
-			_, err := c.AddFunc(taskCmdCfg.CrawlCron, func() { task.Crawl(log.Logger) })
+		_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.Crawl(log.Logger) })
 		if err != nil {
 			log.Logger.Error("Failed to add task", zap.Error(err))
 		}
 		c.Start()
 		select {}
-		}
 	},
 }
 
+var updateTaskCmd = &cobra.Command{
+	Use:   "update",
+	Long:  "Start update outdated game infos task",
+	Short: "Start update outdated game infos task",
+	Run: func(cmd *cobra.Command, args []string) {
+		if taskCommandCfg.Now {
+			task.UpdateOutdatedGameInfos(log.Logger)
+		}
+		c := cron.New()
+		_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.UpdateOutdatedGameInfos(log.Logger) })
+		if err != nil {
+			log.Logger.Error("Failed to add task", zap.Error(err))
+		}
+		c.Start()
+		select {}
+	},
+}
+
+var taskCmd = &cobra.Command{
+	Use:   "task",
+	Long:  "Start task",
+	Short: "Start task",
+}
+
 func init() {
-	taskCmd.Flags().BoolVar(&taskCmdCfg.Crawl, "crawl", false, "enable auto crawl")
+	crawlTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
-	taskCmd.Flags().StringVar(&taskCmdCfg.CrawlCron, "crawl-cron", "0 */3 * * *", "crawl cron expression")
+	crawlTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
+
+	updateTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
+	updateTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
+
+	taskCmd.AddCommand(crawlTaskCmd)
+	taskCmd.AddCommand(updateTaskCmd)
+
 	RootCmd.AddCommand(taskCmd)
 }

cmd/update.go

@@ -1,57 +0,0 @@
-package cmd
-
-import (
-	"pcgamedb/crawler"
-	"pcgamedb/db"
-	"pcgamedb/log"
-
-	"github.com/spf13/cobra"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-	"go.uber.org/zap"
-)
-
-var updateCmd = &cobra.Command{
-	Use:   "update",
-	Long:  "Update game info by game data platform",
-	Short: "Update game info by game data platform",
-	Run:   updateRun,
-}
-
-type updateCommandConfig struct {
-	PlatformID int
-	Platform   string
-	ID         string
-}
-
-var updateCmdcfx updateCommandConfig
-
-func init() {
-	updateCmd.Flags().IntVarP(&updateCmdcfx.PlatformID, "platform-id", "p", 0, "platform id")
-	updateCmd.Flags().StringVarP(&updateCmdcfx.Platform, "platform", "t", "", "platform")
-	updateCmd.Flags().StringVarP(&updateCmdcfx.ID, "game-id", "i", "", "game info id")
-	RootCmd.AddCommand(updateCmd)
-}
-
-func updateRun(cmd *cobra.Command, args []string) {
-	id, err := primitive.ObjectIDFromHex(updateCmdcfx.ID)
-	if err != nil {
-		log.Logger.Error("Failed to parse game info id", zap.Error(err))
-		return
-	}
-	oldInfo, err := db.GetGameInfoByID(id)
-	if err != nil {
-		log.Logger.Error("Failed to get game info", zap.Error(err))
-		return
-	}
-	newInfo, err := crawler.GenerateGameInfo(updateCmdcfx.Platform, updateCmdcfx.PlatformID)
-	if err != nil {
-		log.Logger.Error("Failed to generate game info", zap.Error(err))
-		return
-	}
-	newInfo.ID = id
-	newInfo.GameIDs = oldInfo.GameIDs
-	err = db.SaveGameInfo(newInfo)
-	if err != nil {
-		log.Logger.Error("Failed to save game info", zap.Error(err))
-	}
-}

constant/url.go

@@ -24,6 +24,7 @@ const (
 	Steam250Top250URL        = "https://steam250.com/top250"
 	Steam250BestOfTheYearURL = "https://steam250.com/%v"
 	Steam250WeekTop50URL     = "https://steam250.com/7day"
+	Steam250MonthTop50URL    = "https://steam250.com/30day"
 	Steam250MostPlayedURL    = "https://steam250.com/most_played"
 	FitGirlURL               = "https://fitgirl-repacks.site/page/%v/"
 	SteamRIPBaseURL          = "https://steamrip.com"

crawler/game.go

@@ -6,11 +6,12 @@ import (
 	"strings"
 	"time"
 
-	"go.uber.org/zap"
 	"pcgamedb/db"
 	"pcgamedb/model"
 	"pcgamedb/utils"
 
+	"go.uber.org/zap"
+
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
 )
@@ -26,14 +27,14 @@ func GenerateGameInfo(platform string, id int) (*model.GameInfo, error) {
 	}
 }
 
-// OrganizeGameItem Organize and save GameInfo
+// OrganizeGameItem Organize game item and save game info to database
 func OrganizeGameItem(game *model.GameItem) error {
 	hasOriganized, _ := db.HasGameItemOrganized(game.ID)
 	if hasOriganized {
 		return nil
 	}
 
-	item, err := OrganizeGameItemWithIGDB(0, game)
+	item, err := OrganizeGameItemWithIGDB(game)
 	if err == nil {
 		if item.SteamID == 0 {
 			// get steam id from igdb
@@ -41,20 +42,6 @@ func OrganizeGameItem(game *model.GameItem) error {
 			if err == nil {
 				item.SteamID = steamID
 			}
-			err = db.SaveGameInfo(item)
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-	}
-	item, err = OrganizeGameItemWithSteam(0, game)
-	if err == nil {
-		if item.IGDBID == 0 {
-			igdbID, err := GetIGDBIDBySteamIDCache(item.SteamID)
-			if err == nil {
-				item.IGDBID = igdbID
-			}
 		}
 		err = db.SaveGameInfo(item)
 		if err != nil {
@@ -100,7 +87,7 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo
 		}
 	}
 	if platform == "steam" {
-		igdbID, err := GetIGDBIDBySteamIDCache(platformID)
+		igdbID, err := GetIGDBIDBySteamAppIDCache(platformID)
 		if err == nil {
 			info.IGDBID = igdbID
 		}
@@ -138,7 +125,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
 			changed = true
 		}
 		if info.SteamID != 0 && info.IGDBID == 0 {
-			igdbID, err := GetIGDBIDBySteamIDCache(info.SteamID)
+			igdbID, err := GetIGDBIDBySteamAppIDCache(info.SteamID)
 			time.Sleep(time.Millisecond * 100)
 			if err != nil {
 				continue
@@ -147,9 +134,38 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
 			changed = true
 		}
 		if changed {
-			logger.Info("Supplemented platform id for game info", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
+			logger.Info("supp", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
 			_ = db.SaveGameInfo(info)
+		} else {
+			logger.Info("skip", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
 		}
 	}
 	return nil
 }
+
+func UpdateGameInfo(num int) (chan *model.GameInfo, error) {
+	infos, err := db.GetOutdatedGameInfos(num)
+	if err != nil {
+		return nil, err
+	}
+	updateChan := make(chan *model.GameInfo)
+
+	go func() {
+		for _, info := range infos {
+			if info.IGDBID != 0 {
+				newInfo, err := GenerateIGDBGameInfo(info.IGDBID)
+				if err != nil {
+					continue
+				}
+				db.MergeGameInfo(info, newInfo)
+				err = db.SaveGameInfo(newInfo)
+				if err != nil {
+					continue
+				}
+				updateChan <- newInfo
+			}
+		}
+	}()
+
+	return updateChan, nil
+}

crawler/igdb.go

@@ -1,6 +1,7 @@
 package crawler
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -18,14 +19,18 @@ import (
 	"pcgamedb/db"
 	"pcgamedb/model"
 	"pcgamedb/utils"
+
+	"github.com/PuerkitoBio/goquery"
 )
 
 type twitchToken struct {
-	token   string
+	Token   string    `json:"token"`
-	expires time.Time
+	Expires time.Time `json:"expires"`
 	once    sync.Once
 }
 
+var token = twitchToken{}
+
 func (t *twitchToken) getToken() (string, error) {
 	t.once.Do(func() {
 		if config.Config.RedisAvaliable {
@@ -34,19 +39,19 @@ func (t *twitchToken) getToken() (string, error) {
 			}
 		}
 	})
-	if t.token == "" || time.Now().After(t.expires) {
+	if t.Token == "" || time.Now().After(t.Expires) {
 		token, expires, err := loginTwitch()
 		if err != nil {
 			return "", fmt.Errorf("failed to login twitch: %w", err)
 		}
-		t.token = token
+		t.Token = token
-		t.expires = expires
+		t.Expires = expires
 		j, err := json.Marshal(t)
 		if err == nil {
 			_ = cache.Add("twitch_token", j)
 		}
 	}
-	return t.token, nil
+	return t.Token, nil
 }
 
 func loginTwitch() (string, time.Time, error) {
@@ -100,8 +105,6 @@ func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) {
 	return resp, nil
 }
 
-var token = twitchToken{}
-
 func getIGDBID(name string) (int, error) {
 	var err error
 	resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
@@ -110,6 +113,9 @@ func getIGDBID(name string) (int, error) {
 	}
 	if string(resp.Data) == "[]" {
 		resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
+		if err != nil {
+			return 0, err
+		}
 	}
 
 	var data model.IGDBSearches
@@ -150,6 +156,83 @@ func getIGDBID(name string) (int, error) {
 	return 0, fmt.Errorf("IGDB ID not found: %s", name)
 }
 
+func getIGDBIDBySteamSearch(name string) (int, error) {
+	baseURL, _ := url.Parse(constant.SteamSearchURL)
+	params := url.Values{}
+	params.Add("term", name)
+	baseURL.RawQuery = params.Encode()
+
+	resp, err := utils.Fetch(utils.FetchConfig{
+		Url: baseURL.String(),
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
+	if err != nil {
+		return 0, err
+	}
+	type searchResult struct {
+		ID   int
+		Type string
+		Name string
+	}
+	var items []searchResult
+	doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
+		if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
+			if strings.HasPrefix(itemKey, "App_") {
+				id, err := strconv.Atoi(itemKey[4:])
+				if err != nil {
+					return
+				}
+				name := s.Find(".title").Text()
+				items = append(items, searchResult{
+					ID:   id,
+					Type: "App",
+					Name: name,
+				})
+			}
+			if strings.HasPrefix(itemKey, "Bundle_") {
+				id, err := strconv.Atoi(itemKey[7:])
+				if err != nil {
+					return
+				}
+				name := s.Find(".title").Text()
+				items = append(items, searchResult{
+					ID:   id,
+					Type: "Bundle",
+					Name: name,
+				})
+			}
+		}
+	})
+
+	maxSim := 0.0
+	var maxSimItem searchResult
+	for _, item := range items {
+		if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
+			maxSimItem = item
+			break
+		} else {
+			sim := utils.Similarity(item.Name, name)
+			if sim >= 0.8 && sim > maxSim {
+				maxSim = sim
+				maxSimItem = item
+			}
+		}
+	}
+	if maxSim != 0 {
+		if maxSimItem.Type == "App" {
+			return GetIGDBIDBySteamAppIDCache(maxSimItem.ID)
+		}
+		if maxSimItem.Type == "Bundle" {
+			return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID)
+		}
+	}
+	return 0, fmt.Errorf("steam ID not found: %s", name)
+}
+
 // GetIGDBAppParent returns the parent of the game, if no parent return itself
 func GetIGDBAppParent(id int) (int, error) {
 	detail, err := GetIGDBAppDetailCache(id)
@@ -226,7 +309,7 @@ func GetIGDBAppParentCache(id int) (int, error) {
 	return GetIGDBAppParent(id)
 }
 
-// GetIGDBID returns the IGDB ID of the game, try raw name first then formated names
+// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
 func GetIGDBID(name string) (int, error) {
 	name1 := name
 	name2 := FormatName(name)
@@ -240,6 +323,12 @@ func GetIGDBID(name string) (int, error) {
 			return id, nil
 		}
 	}
+	for _, name := range names {
+		id, err := getIGDBIDBySteamSearch(name)
+		if err == nil {
+			return id, nil
+		}
+	}
 	return 0, errors.New("IGDB ID not found")
 }
 
@@ -305,7 +394,7 @@ func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
 			if err != nil {
 				return nil, err
 			}
-			_ = cache.Add(key, dataBytes)
+			_ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour)
 			return data, nil
 		}
 	} else {
@@ -361,6 +450,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
 	item.Name = detail.Name
 	item.Description = detail.Summary
 	item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)
+	item.InfoUpdatedAt = time.Now()
 
 	for _, lang := range detail.LanguageSupports {
 		if lang.LanguageSupportType == 3 {
@@ -399,14 +489,11 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
 }
 
 // OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
-func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
+func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
-	var err error
+	id, err := GetIGDBIDCache(game.Name)
-	if id == 0 {
-		id, err = GetIGDBIDCache(game.Name)
 	if err != nil {
 		return nil, err
 	}
-	}
 	d, err := db.GetGameInfoByPlatformID("igdb", id)
 	if err == nil {
 		d.GameIDs = append(d.GameIDs, game.ID)
@@ -422,7 +509,7 @@ func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, er
 	return info, nil
 }
 
-func GetIGDBIDBySteamID(id int) (int, error) {
+func GetIGDBIDBySteamAppID(id int) (int, error) {
 	var err error
 	resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
 	if err != nil {
@@ -438,19 +525,19 @@ func GetIGDBIDBySteamID(id int) (int, error) {
 		return 0, errors.New("not found")
 	}
 	if data[0].Game == 0 {
-		return GetIGDBIDBySteamID(id)
+		return GetIGDBIDBySteamAppID(id)
 	}
 	return GetIGDBAppParentCache(data[0].Game)
 }
 
-func GetIGDBIDBySteamIDCache(id int) (int, error) {
+func GetIGDBIDBySteamAppIDCache(id int) (int, error) {
 	if config.Config.RedisAvaliable {
-		key := fmt.Sprintf("igdb_id_by_steam_id:%v", id)
+		key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
 		val, exist := cache.Get(key)
 		if exist {
 			return strconv.Atoi(val)
 		} else {
-			data, err := GetIGDBIDBySteamID(id)
+			data, err := GetIGDBIDBySteamAppID(id)
 			if err != nil {
 				return 0, err
 			}
@@ -458,7 +545,47 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
 			return data, nil
 		}
 	} else {
-		return GetIGDBIDBySteamID(id)
+		return GetIGDBIDBySteamAppID(id)
+	}
+}
+
+func GetIGDBIDBySteamBundleID(id int) (int, error) {
+	var err error
+	resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
+	if err != nil {
+		return 0, err
+	}
+	var data []struct {
+		Game int `json:"game"`
+	}
+	if err = json.Unmarshal(resp.Data, &data); err != nil {
+		return 0, err
+	}
+	if len(data) == 0 {
+		return 0, errors.New("not found")
+	}
+	if data[0].Game == 0 {
+		return GetIGDBIDBySteamBundleID(id)
+	}
+	return GetIGDBAppParentCache(data[0].Game)
+}
+
+func GetIGDBIDBySteamBundleIDCache(id int) (int, error) {
+	if config.Config.RedisAvaliable {
+		key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
+		val, exist := cache.Get(key)
+		if exist {
+			return strconv.Atoi(val)
+		} else {
+			data, err := GetIGDBIDBySteamBundleID(id)
+			if err != nil {
+				return 0, err
+			}
+			_ = cache.Add(key, strconv.Itoa(data))
+			return data, nil
+		}
+	} else {
+		return GetIGDBIDBySteamBundleID(id)
 	}
 }
 

crawler/steam.go

@@ -8,99 +8,15 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"time"
 
 	"pcgamedb/cache"
 	"pcgamedb/config"
 	"pcgamedb/constant"
-	"pcgamedb/db"
 	"pcgamedb/model"
 	"pcgamedb/utils"
 )
 
-func getSteamID(name string) (int, error) {
-	baseURL, _ := url.Parse(constant.SteamSearchURL)
-	params := url.Values{}
-	params.Add("term", name)
-	baseURL.RawQuery = params.Encode()
-
-	resp, err := utils.Fetch(utils.FetchConfig{
-		Url: baseURL.String(),
-	})
-	if err != nil {
-		return 0, err
-	}
-	idRegex := regexp.MustCompile(`data-ds-appid="(.*?)"`)
-	nameRegex := regexp.MustCompile(`<span class="title">(.*?)</span>`)
-	idRegexRes := idRegex.FindAllStringSubmatch(string(resp.Data), -1)
-	nameRegexRes := nameRegex.FindAllStringSubmatch(string(resp.Data), -1)
-
-	if len(idRegexRes) == 0 {
-		return 0, fmt.Errorf("steam ID not found: %s", name)
-	}
-
-	maxSim := 0.0
-	maxSimID := 0
-	for i, id := range idRegexRes {
-		idStr := id[1]
-		nameStr := nameRegexRes[i][1]
-		if index := strings.Index(idStr, ","); index != -1 {
-			idStr = idStr[:index]
-		}
-		if strings.EqualFold(strings.TrimSpace(nameStr), strings.TrimSpace(name)) {
-			return strconv.Atoi(idStr)
-		} else {
-			sim := utils.Similarity(nameStr, name)
-			if sim >= 0.8 && sim > maxSim {
-				maxSim = sim
-				maxSimID, _ = strconv.Atoi(idStr)
-			}
-		}
-	}
-	if maxSimID != 0 {
-		return maxSimID, nil
-	}
-	return 0, fmt.Errorf("steam ID not found: %s", name)
-}
-
-func GetSteamID(name string) (int, error) {
-	name1 := name
-	name2 := FormatName(name)
-	names := []string{name1}
-	if name1 != name2 {
-		names = append(names, name2)
-	}
-	for _, n := range names {
-		id, err := getSteamID(n)
-		if err == nil {
-			return id, nil
-		}
-	}
-	return 0, errors.New("steam ID not found")
-}
-
-func GetSteamIDCache(name string) (int, error) {
-	if config.Config.RedisAvaliable {
-		key := fmt.Sprintf("steam_id:%s", name)
-		val, exist := cache.Get(key)
-		if exist {
-			id, err := strconv.Atoi(val)
-			if err != nil {
-				return 0, err
-			}
-			return id, nil
-		} else {
-			id, err := GetSteamID(name)
-			if err != nil {
-				return 0, err
-			}
-			_ = cache.Add(key, id)
-			return id, nil
-		}
-	} else {
-		return GetSteamID(name)
-	}
-}
-
 func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
 	baseURL, _ := url.Parse(constant.SteamAppDetailURL)
 	params := url.Values{}
@@ -168,6 +84,7 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
 	item.Cover = fmt.Sprintf("https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/%v/library_600x900_2x.jpg", id)
 	item.Developers = detail.Data.Developers
 	item.Publishers = detail.Data.Publishers
+	item.InfoUpdatedAt = time.Now()
 	var screenshots []string
 	for _, screenshot := range detail.Data.Screenshots {
 		screenshots = append(screenshots, screenshot.PathFull)
@@ -176,30 +93,6 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
 	return item, nil
 }
 
-// OrganizeGameItemWithSteam Will add GameItem.ID to the newly added GameInfo.GameIDs
-func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, error) {
-	var err error
-	if id == 0 {
-		id, err = GetSteamIDCache(game.Name)
-		if err != nil {
-			return nil, err
-		}
-	}
-	d, err := db.GetGameInfoByPlatformID("steam", id)
-	if err == nil {
-		d.GameIDs = append(d.GameIDs, game.ID)
-		d.GameIDs = utils.Unique(d.GameIDs)
-		return d, nil
-	}
-	detail, err := GenerateGameInfo("steam", id)
-	if err != nil {
-		return nil, err
-	}
-	detail.GameIDs = append(detail.GameIDs, game.ID)
-	detail.GameIDs = utils.Unique(detail.GameIDs)
-	return detail, nil
-}
-
 func GetSteamIDByIGDBID(IGDBID int) (int, error) {
 	var err error
 	resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))

crawler/steam250.go

@@ -62,6 +62,10 @@ func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250WeekTop50URL)
 }
 
+func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
+	return GetSteam250(constant.Steam250MonthTop50URL)
+}
+
 func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250MostPlayedURL)
 }

db/game.go

@@ -141,6 +141,9 @@ func SaveGameInfo(item *model.GameInfo) error {
 	if item.CreatedAt.IsZero() {
 		item.CreatedAt = time.Now()
 	}
+	if item.InfoUpdatedAt.IsZero() {
+		item.InfoUpdatedAt = item.CreatedAt
+	}
 	item.UpdatedAt = time.Now()
 	filter := bson.M{"_id": item.ID}
 	update := bson.M{"$set": item}
@@ -153,7 +156,7 @@ func SaveGameInfo(item *model.GameInfo) error {
 }
 
 func SaveGameInfos(items []*model.GameInfo) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
 	defer cancel()
 
 	operations := make([]mongo.WriteModel, len(items))
@@ -164,6 +167,9 @@ func SaveGameInfos(items []*model.GameInfo) error {
 		if item.CreatedAt.IsZero() {
 			item.CreatedAt = time.Now()
 		}
+		if item.InfoUpdatedAt.IsZero() {
+			item.InfoUpdatedAt = item.CreatedAt
+		}
 		item.UpdatedAt = time.Now()
 		operations[i] = mongo.NewUpdateOneModel().
 			SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
@@ -889,3 +895,31 @@ func GetGameInfoByGameItemID(id primitive.ObjectID) (*model.GameInfo, error) {
 	}
 	return res[0], nil
 }
+
+func GetOutdatedGameInfos(maxNum int) ([]*model.GameInfo, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	opts := options.Find().SetLimit(int64(maxNum))
+	filter := bson.M{
+		"info_updated_at": bson.M{"$lt": time.Now().Add(-24 * time.Hour * 30)},
+	}
+	cursor, err := GameInfoCollection.Find(ctx, filter, opts)
+	if err != nil {
+		return nil, err
+	}
+	var res []*model.GameInfo
+	if err = cursor.All(ctx, &res); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func MergeGameInfo(oldInfo *model.GameInfo, newInfo *model.GameInfo) {
+	newInfo.ID = oldInfo.ID
+	newInfo.UpdatedAt = time.Now()
+	newInfo.GameIDs = oldInfo.GameIDs
+	newInfo.IGDBID = oldInfo.IGDBID
+	newInfo.SteamID = oldInfo.SteamID
+	newInfo.CreatedAt = oldInfo.CreatedAt
+	newInfo.InfoUpdatedAt = time.Now()
+}

model/game.go

@@ -19,9 +19,10 @@ type GameInfo struct {
 	Languages     []string             `json:"languages" bson:"languages"`
 	Screenshots   []string             `json:"screenshots" bson:"screenshots"`
 	GameIDs       []primitive.ObjectID `json:"game_ids" bson:"games"`
-	Games       []*GameItem          `json:"game_downloads" bson:"-"`
+	Games         []*GameItem          `json:"games" bson:"-"`
 	CreatedAt     time.Time            `json:"created_at" bson:"created_at"`
 	UpdatedAt     time.Time            `json:"updated_at" bson:"updated_at"`
+	InfoUpdatedAt time.Time            `json:"info_updated_at" bson:"info_updated_at"`
 }
 
 type GameItem struct {

server/handler/get_game_by_raw_name.go

@@ -17,7 +17,7 @@ type GetGameItemByRawNameRequest struct {
 type GetGameItemByRawNameResponse struct {
 	Status   string            `json:"status"`
 	Message  string            `json:"message,omitempty"`
-	GameItem []*model.GameItem `json:"game_downloads,omitempty"`
+	GameItem []*model.GameItem `json:"games,omitempty"`
 }
 
 // GetGameItemByRawName retrieves game download details by raw name.
@@ -26,7 +26,7 @@ type GetGameItemByRawNameResponse struct {
 // @Tags game
 // @Accept json
 // @Produce json
-// @Param name path string true "Game Download Raw Name"
+// @Param name path string true "Game Raw Name"
 // @Success 200 {object} GetGameItemByRawNameResponse
 // @Failure 400 {object} GetGameItemByRawNameResponse
 // @Failure 500 {object} GetGameItemByRawNameResponse

server/handler/get_games_by_author.go

@@ -19,10 +19,10 @@ type GetGameItemsByAuthorResponse struct {
 	Status    string            `json:"status"`
 	Message   string            `json:"message,omitempty"`
 	TotalPage int               `json:"total_page"`
-	GameItems []*model.GameItem `json:"game_downloads,omitempty"`
+	GameItems []*model.GameItem `json:"games,omitempty"`
 }
 
-// GetGameItemsByAuthorHandler returns all game downloads by author
+// GetGameItemsByAuthorHandler returns games by author
 // @Summary Get game downloads by author
 // @Description Get game downloads by author
 // @Tags game

server/handler/get_popular_game_infos.go

@@ -1,11 +1,12 @@
 package handler
 
 import (
-	"github.com/gin-gonic/gin"
 	"net/http"
 	"pcgamedb/crawler"
 	"pcgamedb/db"
 	"pcgamedb/model"
+
+	"github.com/gin-gonic/gin"
 )
 
 type GetPopularGamesResponse struct {
@@ -20,7 +21,7 @@ type GetPopularGamesResponse struct {
 // @Tags popular
 // @Accept json
 // @Produce json
-// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-best-of-the-year, steam-most-played)"
+// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-month-top, steam-best-of-the-year, steam-most-played)"
 // @Success 200 {object} GetPopularGamesResponse
 // @Failure 400 {object} GetPopularGamesResponse
 // @Failure 500 {object} GetPopularGamesResponse
@@ -52,6 +53,8 @@ func GetPopularGameInfosHandler(c *gin.Context) {
 		steam250Func = crawler.GetSteam250BestOfTheYear
 	case "steam-most-played":
 		steam250Func = crawler.GetSteam250MostPlayed
+	case "steam-month-top":
+		steam250Func = crawler.GetSteam250MonthTop50
 	default:
 		c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
 			Status:  "error",

server/handler/get_unorganized_games.go

@@ -17,16 +17,16 @@ type GetUnorganizedGameItemsResponse struct {
 	Status    string            `json:"status"`
 	Message   string            `json:"message,omitempty"`
 	Size      int               `json:"size,omitempty"`
-	GameItems []*model.GameItem `json:"game_downloads,omitempty"`
+	GameItems []*model.GameItem `json:"games,omitempty"`
 }
 
-// GetUnorganizedGameItems retrieves a list of unorganized game downloads.
+// GetUnorganizedGameItems retrieves a list of unorganized games.
-// @Summary List unorganized game downloads
+// @Summary List unorganized games
-// @Description Retrieves game downloads that have not been organized
+// @Description Retrieves games that have not been organized
 // @Tags game
 // @Accept json
 // @Produce json
-// @Param num query int false "Number of game downloads to retrieve"
+// @Param num query int false "Number of games to retrieve"
 // @Success 200 {object} GetUnorganizedGameItemsResponse
 // @Failure 400 {object} GetUnorganizedGameItemsResponse
 // @Failure 500 {object} GetUnorganizedGameItemsResponse

server/handler/healthcheck.go

@@ -16,15 +16,14 @@ import (
 type HealthCheckResponse struct {
 	Version            string `json:"version"`
 	Status             string `json:"status"`
-	Message            string `json:"message,omitempty"`
 	Date               string `json:"date"`
 	Uptime             string `json:"uptime"`
 	Alloc              string `json:"alloc"`
 	AutoCrawl          bool   `json:"auto_crawl"`
-	AutoCrawlCron      string `json:"auto_crawl_cron,omitempty"`
+	AutoCrawlCron      string `json:"auto_crawl_cron"`
-	GameItem           int64  `json:"game_download,omitempty"`
+	GameItem           int64  `json:"game_num"`
-	GameInfo           int64  `json:"game_info,omitempty"`
+	GameInfo           int64  `json:"game_info_num"`
-	Unorganized        int64  `json:"unorganized,omitempty"`
+	Unorganized        int64  `json:"unorganized_game_num"`
 	RedisAvaliable     bool   `json:"redis_avaliable"`
 	OnlineFixAvaliable bool   `json:"online_fix_avaliable"`
 	MegaAvaliable      bool   `json:"mega_avaliable"`

server/handler/update_game_info.go

@@ -1,100 +0,0 @@
-package handler
-
-import (
-	"net/http"
-	"strings"
-
-	"pcgamedb/crawler"
-	"pcgamedb/db"
-	"pcgamedb/model"
-
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/bson/primitive"
-)
-
-type UpdateGameInfoRequest struct {
-	GameID     string `json:"game_id" binding:"required"`
-	Platform   string `json:"platform" binding:"required"`
-	PlatformID int    `json:"platform_id" binding:"required"`
-}
-
-type UpdateGameInfoResponse struct {
-	Status   string          `json:"status"`
-	Message  string          `json:"message"`
-	GameInfo *model.GameInfo `json:"game_info,omitempty"`
-}
-
-// UpdateGameInfoHandler updates game information.
-// @Summary Update game info
-// @Description Updates details of a game
-// @Tags game
-// @Accept json
-// @Produce json
-// @Param Authorization header string true "Authorization: Bearer <api_key>"
-// @Param body body handler.UpdateGameInfoRequest true "Update Game Info Request"
-// @Success 200 {object} handler.UpdateGameInfoResponse
-// @Failure 400 {object} handler.UpdateGameInfoResponse
-// @Failure 401 {object} handler.UpdateGameInfoResponse
-// @Failure 500 {object} handler.UpdateGameInfoResponse
-// @Router /game/update [post]
-func UpdateGameInfoHandler(c *gin.Context) {
-	var req UpdateGameInfoRequest
-	if err := c.ShouldBindJSON(&req); err != nil {
-		c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	req.Platform = strings.ToLower(req.Platform)
-	platformMap := map[string]bool{
-		"steam": true,
-		"igdb":  true,
-	}
-	if _, ok := platformMap[req.Platform]; !ok {
-		c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
-			Status:  "error",
-			Message: "Invalid platform",
-		})
-		return
-	}
-	objID, err := primitive.ObjectIDFromHex(req.GameID)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	info, err := db.GetGameInfoByID(objID)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	newInfo, err := crawler.GenerateGameInfo(req.Platform, req.PlatformID)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	newInfo.ID = objID
-	newInfo.GameIDs = info.GameIDs
-	err = db.SaveGameInfo(newInfo)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	c.JSON(http.StatusOK, UpdateGameInfoResponse{
-		Status:   "ok",
-		Message:  "Game info updated successfully",
-		GameInfo: newInfo,
-	})
-}

server/route.go

@@ -31,7 +31,6 @@ func initRoute(app *gin.Engine) {
 	GameInfoGroup.GET("/name/:name", handler.GetGameInfosByNameHandler)
 	GameInfoGroup.GET("/platform/:platform_type/:platform_id", handler.GetGameInfoByPlatformIDHandler)
 	GameInfoGroup.GET("/id/:id", handler.GetGameInfoByIDHandler)
-	GameInfoGroup.PUT("/update", middleware.Auth(), handler.UpdateGameInfoHandler)
 	GameInfoGroup.DELETE("/id/:id", middleware.Auth(), handler.DeleteGameInfoHandler)
 
 	app.GET("/popular/:type", handler.GetPopularGameInfosHandler)

server/server.go

@@ -30,6 +30,8 @@ func Run() {
 	app.Use(middleware.Recovery())
 	initRoute(app)
 	log.Logger.Info("Server running", zap.String("port", config.Config.Server.Port))
+
+	// Start auto-crawl task
 	if config.Config.Server.AutoCrawl {
 		go func() {
 			c := cron.New()
@@ -40,6 +42,17 @@ func Run() {
 			c.Start()
 		}()
 	}
+
+	// Start auto-update task
+	go func() {
+		c := cron.New()
+		_, err := c.AddFunc("0 */3 * * *", func() { task.UpdateOutdatedGameInfos(log.TaskLogger) })
+		if err != nil {
+			log.Logger.Error("Error adding cron job", zap.Error(err))
+		}
+		c.Start()
+	}()
+
 	err := app.Run(":" + config.Config.Server.Port)
 	if err != nil {
 		log.Logger.Panic("Failed to run server", zap.Error(err))

task/update_game_info.go

@@ -0,0 +1,27 @@
+package task
+
+import (
+	"pcgamedb/crawler"
+
+	"go.uber.org/zap"
+)
+
+func UpdateOutdatedGameInfos(logger *zap.Logger) {
+	channel, err := crawler.UpdateGameInfo(10)
+	count := 0
+	if err != nil {
+		logger.Error("Failed to update game info", zap.Error(err))
+		return
+	}
+	for info := range channel {
+		logger.Info("Updated game info",
+			zap.String("id", info.ID.String()),
+			zap.String("name", info.Name),
+		)
+		count++
+		if count == 10 {
+			break
+		}
+	}
+	logger.Info("Updated game info count", zap.Int("count", count))
+}