add export cmd

add supplement cmd
rename GameDown to GameItem

cmd/export.go

@@ -0,0 +1,54 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/nitezs/pcgamedb/db"
+	"github.com/nitezs/pcgamedb/log"
+	"go.uber.org/zap"
+
+	"github.com/spf13/cobra"
+)
+
+type exportCommandConfig struct {
+	output string
+}
+
+var exportCmdCfg exportCommandConfig
+
+var exportCmd = &cobra.Command{
+	Use:   "export",
+	Long:  "Export data to json files",
+	Short: "Export data to json files",
+	Run: func(cmd *cobra.Command, args []string) {
+		infoJson, gameJson, err := db.Export()
+		if err != nil {
+			log.Logger.Error("Error exporting data", zap.Error(err))
+			return
+		}
+		infoFilePath := filepath.Join(exportCmdCfg.output, "game_infos.json")
+		gameFilePath := filepath.Join(exportCmdCfg.output, "games.json")
+
+		err = os.WriteFile(infoFilePath, infoJson, 0644)
+		if err != nil {
+			if err != nil {
+				log.Logger.Error("Error exporting data", zap.Error(err))
+				return
+			}
+		}
+		err = os.WriteFile(gameFilePath, gameJson, 0644)
+		if err != nil {
+			if err != nil {
+				log.Logger.Error("Error exporting data", zap.Error(err))
+				return
+			}
+		}
+		log.Logger.Info("Data exported successfully")
+	},
+}
+
+func init() {
+	exportCmd.Flags().StringVarP(&exportCmdCfg.output, "output", "o", "", "Output directory for json files")
+	RootCmd.AddCommand(exportCmd)
+}

cmd/format.go

@@ -33,7 +33,7 @@ func formatRun(cmd *cobra.Command, args []string) {
 	formatCmdCfg.Source = strings.ToLower(formatCmdCfg.Source)
 	switch formatCmdCfg.Source {
 	case "dodi":
-		items, err := db.GetDODIGameDownloads()
+		items, err := db.GetDODIGameItems()
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 			return
@@ -43,14 +43,14 @@ func formatRun(cmd *cobra.Command, args []string) {
 			item.Name = crawler.DODIFormatter(item.RawName)
 			if oldName != item.Name {
 				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameDownload(item)
+				err := db.SaveGameItem(item)
 				if err != nil {
 					log.Logger.Error("Failed to update item", zap.Error(err))
 				}
 			}
 		}
 	case "kaoskrew":
-		items, err := db.GetKaOsKrewGameDownloads()
+		items, err := db.GetKaOsKrewGameItems()
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 			return
@@ -60,14 +60,14 @@ func formatRun(cmd *cobra.Command, args []string) {
 			item.Name = crawler.KaOsKrewFormatter(item.RawName)
 			if oldName != item.Name {
 				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameDownload(item)
+				err := db.SaveGameItem(item)
 				if err != nil {
 					log.Logger.Error("Failed to update item", zap.Error(err))
 				}
 			}
 		}
 	case "freegog":
-		items, err := db.GetFreeGOGGameDownloads()
+		items, err := db.GetFreeGOGGameItems()
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 			return
@@ -77,14 +77,14 @@ func formatRun(cmd *cobra.Command, args []string) {
 			item.Name = crawler.FreeGOGFormatter(item.RawName)
 			if oldName != item.Name {
 				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameDownload(item)
+				err := db.SaveGameItem(item)
 				if err != nil {
 					log.Logger.Error("Failed to update item", zap.Error(err))
 				}
 			}
 		}
 	case "xatab":
-		items, err := db.GetXatabGameDownloads()
+		items, err := db.GetXatabGameItems()
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 			return
@@ -94,14 +94,14 @@ func formatRun(cmd *cobra.Command, args []string) {
 			item.Name = crawler.XatabFormatter(item.RawName)
 			if oldName != item.Name {
 				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameDownload(item)
+				err := db.SaveGameItem(item)
 				if err != nil {
 					log.Logger.Error("Failed to update item", zap.Error(err))
 				}
 			}
 		}
 	case "onlinefix":
-		items, err := db.GetOnlineFixGameDownloads()
+		items, err := db.GetOnlineFixGameItems()
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 			return
@@ -111,14 +111,14 @@ func formatRun(cmd *cobra.Command, args []string) {
 			item.Name = crawler.OnlineFixFormatter(item.RawName)
 			if oldName != item.Name {
 				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameDownload(item)
+				err := db.SaveGameItem(item)
 				if err != nil {
 					log.Logger.Error("Failed to update item", zap.Error(err))
 				}
 			}
 		}
 	case "armgddn":
-		items, err := db.GetARMGDDNGameDownloads()
+		items, err := db.GetARMGDDNGameItems()
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 			return
@@ -128,7 +128,7 @@ func formatRun(cmd *cobra.Command, args []string) {
 			item.Name = crawler.ARMGDDNFormatter(item.RawName)
 			if oldName != item.Name {
 				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameDownload(item)
+				err := db.SaveGameItem(item)
 				if err != nil {
 					log.Logger.Error("Failed to update item", zap.Error(err))
 				}

cmd/list.go

@@ -28,7 +28,7 @@ func init() {
 
 func listRun(cmd *cobra.Command, args []string) {
 	if listCmdCfg.Unid {
-		games, err := db.GetUnorganizedGameDownloads(-1)
+		games, err := db.GetUnorganizedGameItems(-1)
 		if err != nil {
 			log.Logger.Error("Failed to get games", zap.Error(err))
 		}

cmd/organize.go

@@ -28,12 +28,12 @@ func init() {
 }
 
 func organizeRun(cmd *cobra.Command, args []string) {
-	games, err := db.GetUnorganizedGameDownloads(organizeCmdCfg.Num)
+	games, err := db.GetUnorganizedGameItems(organizeCmdCfg.Num)
 	if err != nil {
 		log.Logger.Error("Failed to get games", zap.Error(err))
 	}
 	for _, game := range games {
-		gameInfo, err := crawler.OrganizeGameDownload(game)
+		gameInfo, err := crawler.OrganizeGameItem(game)
 		if err == nil {
 			err = db.SaveGameInfo(gameInfo)
 			if err != nil {

cmd/organize_manually.go

@@ -58,7 +58,7 @@ func addRun(cmd *cobra.Command, args []string) {
 			log.Logger.Error("Failed to parse game id", zap.Error(err))
 			continue
 		}
-		info, err := crawler.OrganizeGameDownloadManually(objID, v.Platform, v.PlatformID)
+		info, err := crawler.OrganizeGameItemManually(objID, v.Platform, v.PlatformID)
 		if err != nil {
 			log.Logger.Error("Failed to add game info", zap.Error(err))
 			continue

cmd/supplement.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"github.com/nitezs/pcgamedb/crawler"
+	"github.com/nitezs/pcgamedb/log"
+	"go.uber.org/zap"
+
+	"github.com/spf13/cobra"
+)
+
+var supplementCmd = &cobra.Command{
+	Use:   "supplement",
+	Long:  "Supplement platform id to game info",
+	Short: "Supplement platform id to game info",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := crawler.SupplementPlatformIDToGameInfo(log.Logger)
+		if err != nil {
+			log.Logger.Error("Error supplementing platform id to game info", zap.Error(err))
+		}
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(supplementCmd)
+}

cmd/task.go

@@ -22,7 +22,7 @@ var taskCmd = &cobra.Command{
 		if taskCmdCfg.Crawl {
 			task.Crawl(log.Logger)
 			c := cron.New()
-			_, err := c.AddFunc("0 0 * * *", func() { task.Crawl(log.Logger) })
+			_, err := c.AddFunc("0 */3 * * *", func() { task.Crawl(log.Logger) })
 			if err != nil {
 				log.Logger.Error("Failed to add task", zap.Error(err))
 			}

crawler/1337x.go

@@ -33,7 +33,7 @@ func New1337xCrawler(source string, formatter Formatter, logger *zap.Logger) *s1
 	}
 }
 
-func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *s1337xCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	var resp *utils.FetchResponse
 	var doc *goquery.Document
 	var err error
@@ -57,7 +57,7 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			urls = append(urls, url)
 		}
 	})
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for _, u := range urls {
 		u = fmt.Sprintf("%s%s", constant.C1337xBaseURL, u)
 		if db.IsGameCrawledByURL(u) {
@@ -69,13 +69,13 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
 			continue
 		}
-		err = db.SaveGameDownload(item)
+		err = db.SaveGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
 			continue
 		}
 		res = append(res, item)
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -89,14 +89,14 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
 	if err != nil {
 		return nil, err
 	}
-	var item = &model.GameDownload{}
+	var item = &model.GameItem{}
 	item.Url = url
 	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
 	if err != nil {
@@ -119,8 +119,8 @@ func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *s1337xCrawler) CrawlMulti(pages []int) (res []*model.GameDownload, err error) {
+func (c *s1337xCrawler) CrawlMulti(pages []int) (res []*model.GameItem, err error) {
-	var items []*model.GameDownload
+	var items []*model.GameItem
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err
@@ -138,12 +138,12 @@ func (c *s1337xCrawler) CrawlMulti(pages []int) (res []*model.GameDownload, err
 	return res, nil
 }
 
-func (c *s1337xCrawler) CrawlAll() (res []*model.GameDownload, err error) {
+func (c *s1337xCrawler) CrawlAll() (res []*model.GameItem, err error) {
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err
 	}
-	var items []*model.GameDownload
+	var items []*model.GameItem
 	for i := 1; i <= totalPageNum; i++ {
 		items, err = c.Crawl(i)
 		res = append(res, items...)

crawler/armgddn.go

@@ -75,9 +75,9 @@ func (c *ARMGDDNCrawler) fetchAndParseFTPData(filePath string) ([]GameData, erro
 	return data, nil
 }
 
-func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameDownload, error) {
+func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameItem, error) {
 	count := 0
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	modTimeMap := make(map[string]time.Time)
 	entries, err := c.conn.List(fmt.Sprintf("/%s", platform))
 	if err != nil {
@@ -116,7 +116,7 @@ func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) (
 				size += fileSize
 			}
 		}
-		item, err := db.GetGameDownloadByUrl(u)
+		item, err := db.GetGameItemByUrl(u)
 		if err != nil {
 			continue
 		}
@@ -127,7 +127,7 @@ func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) (
 		item.RawName = v.FolderName
 		item.Author = "ARMGDDN"
 		item.Download = fmt.Sprintf("ftpes://%s:%s@%s/%s/%s", ftpUsername, ftpPassword, ftpAddress, platform, url.QueryEscape(v.FolderName))
-		if err := db.SaveGameDownload(item); err != nil {
+		if err := db.SaveGameItem(item); err != nil {
 			continue
 		}
 		res = append(res, item)
@@ -140,12 +140,12 @@ func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) (
 				c.logger.Warn("strconv error", zap.Error(err))
 				continue
 			}
-			info, err = OrganizeGameDownloadWithSteam(id, item)
+			info, err = OrganizeGameItemWithSteam(id, item)
 			if err != nil {
 				continue
 			}
 		} else {
-			info, err = OrganizeGameDownload(item)
+			info, err = OrganizeGameItem(item)
 			if err != nil {
 				continue
 			}
@@ -168,15 +168,15 @@ func ARMGDDNFormatter(name string) string {
 	return strings.TrimSpace(cleanedName[:matchIndex[0]])
 }
 
-func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameDownload, error) {
+func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameItem, error) {
 	return c.crawlPlatform("/PC/currentserverPC-FTP.json", "PC", num)
 }
 
-func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameDownload, error) {
+func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameItem, error) {
 	return c.crawlPlatform("/PCVR/currentserverPCVR-FTP.json", "PCVR", num)
 }
 
-func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameDownload, error) {
+func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameItem, error) {
 	num1 := num / 2
 	num2 := num - num1
 	if num == -1 {
@@ -194,11 +194,11 @@ func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 	return append(res1, res2...), nil
 }
 
-func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameItem, error) {
 	return c.Crawl(-1)
 }
 
-func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameDownload, error) {
+func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameItem, error) {
 	err := c.connectFTP()
 	if err != nil {
 		return nil, err

crawler/chovka.go

@@ -30,7 +30,7 @@ func (c *ChovkaCrawler) Name() string {
 	return "ChovkaCrawler"
 }
 
-func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
@@ -41,7 +41,7 @@ func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	if err != nil {
 		return nil, err
 	}
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -70,7 +70,7 @@ func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: fmt.Sprintf(constant.RepackInfoURL, page),
 	})
@@ -91,7 +91,7 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 		urls = append(urls, u)
 		updateFlags = append(updateFlags, s.Find(".entry__title.h2 a").Text())
 	})
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i, u := range urls {
 		if db.IsChovkaCrawled(updateFlags[i]) {
 			continue
@@ -102,12 +102,12 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
 			continue
 		}
-		if err := db.SaveGameDownload(item); err != nil {
+		if err := db.SaveGameItem(item); err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
 			continue
 		}
 		res = append(res, item)
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -120,8 +120,8 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for _, page := range pages {
 		items, err := c.Crawl(page)
 		if err != nil {
@@ -132,12 +132,12 @@ func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *ChovkaCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *ChovkaCrawler) CrawlAll() ([]*model.GameItem, error) {
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err
 	}
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i := 1; i <= totalPageNum; i++ {
 		items, err := c.Crawl(i)
 		if err != nil {

crawler/crawler.go

@@ -8,8 +8,8 @@ import (
 
 type Crawler interface {
 	Name() string
-	Crawl(int) ([]*model.GameDownload, error)
+	Crawl(int) ([]*model.GameItem, error)
-	CrawlAll() ([]*model.GameDownload, error)
+	CrawlAll() ([]*model.GameItem, error)
 }
 
 type SimpleCrawler interface {
@@ -18,7 +18,7 @@ type SimpleCrawler interface {
 
 type PagedCrawler interface {
 	Crawler
-	CrawlMulti([]int) ([]*model.GameDownload, error)
+	CrawlMulti([]int) ([]*model.GameItem, error)
 	GetTotalPageNum() (int, error)
 }
 

crawler/dodi.go

@@ -32,19 +32,19 @@ func (c *DODICrawler) Name() string {
 	return "DODICrawler"
 }
 
-func (c *DODICrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *DODICrawler) Crawl(page int) ([]*model.GameItem, error) {
 	return c.crawler.Crawl(page)
 }
 
-func (c *DODICrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *DODICrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	return c.crawler.CrawlByUrl(url)
 }
 
-func (c *DODICrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *DODICrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
 	return c.crawler.CrawlMulti(pages)
 }
 
-func (c *DODICrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *DODICrawler) CrawlAll() ([]*model.GameItem, error) {
 	return c.crawler.CrawlAll()
 }
 

crawler/fitgirl.go

@@ -31,7 +31,7 @@ func (c *FitGirlCrawler) Name() string {
 	return "FitGirlCrawler"
 }
 
-func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
@@ -61,7 +61,7 @@ func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 		return nil, errors.New("Failed to find magnet")
 	}
 	magnet := magnetRegexRes[0]
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -74,7 +74,7 @@ func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: fmt.Sprintf(constant.FitGirlURL, page),
 	})
@@ -97,7 +97,7 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			updateFlags = append(updateFlags, fmt.Sprintf("%s%s", u, d))
 		}
 	})
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i, u := range urls {
 		if db.IsFitgirlCrawled(updateFlags[i]) {
 			continue
@@ -109,13 +109,13 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			continue
 		}
 		item.UpdateFlag = updateFlags[i]
-		err = db.SaveGameDownload(item)
+		err = db.SaveGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err))
 			continue
 		}
 		res = append(res, item)
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -129,8 +129,8 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *FitGirlCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *FitGirlCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for _, page := range pages {
 		items, err := c.Crawl(page)
 		if err != nil {
@@ -141,8 +141,8 @@ func (c *FitGirlCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error)
 	return res, nil
 }
 
-func (c *FitGirlCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *FitGirlCrawler) CrawlAll() ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err

crawler/freegog.go

@@ -27,7 +27,7 @@ func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
 	}
 }
 
-func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
+func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
 	count := 0
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: constant.FreeGOGListURL,
@@ -49,7 +49,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 		updateFlags = append(updateFlags, s.Text()+s.AttrOr("href", ""))
 	})
 
-	res := []*model.GameDownload{}
+	res := []*model.GameItem{}
 	for i, u := range urls {
 		if count == num {
 			break
@@ -64,14 +64,14 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 			continue
 		}
 		item.UpdateFlag = updateFlags[i]
-		err = db.SaveGameDownload(item)
+		err = db.SaveGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err))
 			continue
 		}
 		res = append(res, item)
 		count++
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -85,14 +85,14 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
 	if err != nil {
 		return nil, err
 	}
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +127,7 @@ func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *FreeGOGCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *FreeGOGCrawler) CrawlAll() ([]*model.GameItem, error) {
 	return c.Crawl(-1)
 }
 

crawler/game.go

@@ -9,6 +9,7 @@ import (
 	"github.com/nitezs/pcgamedb/db"
 	"github.com/nitezs/pcgamedb/model"
 	"github.com/nitezs/pcgamedb/utils"
+	"go.uber.org/zap"
 
 	"go.mongodb.org/mongo-driver/bson/primitive"
 	"go.mongodb.org/mongo-driver/mongo"
@@ -25,10 +26,11 @@ func GenerateGameInfo(platform string, id int) (*model.GameInfo, error) {
 	}
 }
 
-func OrganizeGameDownload(game *model.GameDownload) (*model.GameInfo, error) {
+func OrganizeGameItem(game *model.GameItem) (*model.GameInfo, error) {
-	item, err := OrganizeGameDownloadWithIGDB(0, game)
+	item, err := OrganizeGameItemWithIGDB(0, game)
 	if err == nil {
 		if item.SteamID == 0 {
+			// get steam id from igdb
 			steamID, err := GetSteamIDByIGDBIDCache(item.IGDBID)
 			if err == nil {
 				item.SteamID = steamID
@@ -36,7 +38,7 @@ func OrganizeGameDownload(game *model.GameDownload) (*model.GameInfo, error) {
 			return item, nil
 		}
 	}
-	item, err = OrganizeGameDownloadWithSteam(0, game)
+	item, err = OrganizeGameItemWithSteam(0, game)
 	if err == nil {
 		if item.IGDBID == 0 {
 			igdbID, err := GetIGDBIDBySteamIDCache(item.SteamID)
@@ -59,7 +61,7 @@ func AddGameInfoManually(gameID primitive.ObjectID, platform string, plateformID
 	return info, db.SaveGameInfo(info)
 }
 
-func OrganizeGameDownloadManually(gameID primitive.ObjectID, platform string, platformID int) (*model.GameInfo, error) {
+func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platformID int) (*model.GameInfo, error) {
 	info, err := db.GetGameInfoByPlatformID(platform, platformID)
 	if err != nil {
 		if err == mongo.ErrNoDocuments {
@@ -105,38 +107,7 @@ func FormatName(name string) string {
 	return name
 }
 
-func TransformSteamIDToIGDBID() {
+func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
-	gameInfos, err := db.GetGameInfoWithSteamID()
-	if err != nil {
-		return
-	}
-	for _, info := range gameInfos {
-		id, err := GetIGDBIDBySteamIDCache(info.SteamID)
-		if err != nil {
-			continue
-		}
-		existedInfo, err := db.GetGameInfoByPlatformID("igdb", id)
-		if err == nil {
-			existedInfo.GameIDs = append(existedInfo.GameIDs, info.GameIDs...)
-			existedInfo.GameIDs = utils.Unique(existedInfo.GameIDs)
-			_ = db.SaveGameInfo(existedInfo)
-			_ = db.DeleteGameInfoByID(info.ID)
-		} else {
-			if err == mongo.ErrNoDocuments {
-				newInfo, err := GenerateIGDBGameInfo(id)
-				if err != nil {
-					continue
-				}
-				newInfo.ID = info.ID
-				newInfo.CreatedAt = info.CreatedAt
-				newInfo.GameIDs = info.GameIDs
-				_ = db.SaveGameInfo(newInfo)
-			}
-		}
-	}
-}
-
-func SupplementGameInfoPlatformID() error {
 	infos, err := db.GetAllGameInfos()
 	if err != nil {
 		return err
@@ -162,6 +133,7 @@ func SupplementGameInfoPlatformID() error {
 			changed = true
 		}
 		if changed {
+			logger.Info("Supplemented platform id for game info", zap.String("name", info.Name), zap.Int("igdb", int(info.IGDBID)), zap.Int("steam", int(info.SteamID)))
 			_ = db.SaveGameInfo(info)
 		}
 	}

crawler/gnarly.go

@@ -24,8 +24,8 @@ func NewGnarlyCrawler(logger *zap.Logger) *GnarlyCrawler {
 	}
 }
 
-func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
+func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	count := 0
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: constant.GnarlyURL,
@@ -55,7 +55,7 @@ func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 						if db.IsGnarlyCrawled(lines[i-1]) {
 							continue
 						}
-						item, err := db.GetGameDownloadByUrl(lines[i])
+						item, err := db.GetGameItemByUrl(lines[i])
 						if err != nil {
 							continue
 						}
@@ -76,7 +76,7 @@ func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 						item.UpdateFlag = item.RawName
 						res = append(res, item)
 						count++
-						info, err := OrganizeGameDownload(item)
+						info, err := OrganizeGameItem(item)
 						if err != nil {
 							continue
 						}
@@ -93,7 +93,7 @@ func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *GnarlyCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *GnarlyCrawler) CrawlAll() ([]*model.GameItem, error) {
 	return c.Crawl(-1)
 }
 

crawler/goggames.go

@@ -31,7 +31,7 @@ func (c *GOGGamesCrawler) Name() string {
 	return "GOGGamesCrawler"
 }
 
-func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
@@ -56,7 +56,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	if err != nil {
 		return nil, err
 	}
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -69,7 +69,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: fmt.Sprintf(constant.GOGGamesURL, page),
 	})
@@ -88,7 +88,7 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 		}
 		urls = append(urls, fmt.Sprintf("%s%s", constant.GOGGamesBaseURL, u))
 	})
-	res := make([]*model.GameDownload, 0)
+	res := make([]*model.GameItem, 0)
 	for _, u := range urls {
 		c.logger.Info("Crawling", zap.String("URL", u))
 		item, err := c.CrawlByUrl(u)
@@ -96,12 +96,12 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
 			continue
 		}
-		if err := db.SaveGameDownload(item); err != nil {
+		if err := db.SaveGameItem(item); err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
 			continue
 		}
 		res = append(res, item)
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -114,8 +114,8 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
-	res := make([]*model.GameDownload, 0)
+	res := make([]*model.GameItem, 0)
 	for _, page := range pages {
 		items, err := c.Crawl(page)
 		if err != nil {
@@ -126,12 +126,12 @@ func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error)
 	return res, nil
 }
 
-func (c *GOGGamesCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *GOGGamesCrawler) CrawlAll() ([]*model.GameItem, error) {
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err
 	}
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i := 1; i <= totalPageNum; i++ {
 		items, err := c.Crawl(i)
 		if err != nil {

crawler/igdb.go

@@ -311,7 +311,8 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
 	return item, nil
 }
 
-func OrganizeGameDownloadWithIGDB(id int, game *model.GameDownload) (*model.GameInfo, error) {
+// id=0, means search id by name
+func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
 	var err error
 	if id == 0 {
 		id, err = GetIGDBIDCache(game.Name)
@@ -390,7 +391,7 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
 	}
 }
 
-func GetIGDBIDBySteamIDs(ids []int) (map[int]int, error) {
+func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
 	var err error
 	if TwitchToken == "" {
 		TwitchToken, err = LoginTwitch()
@@ -446,7 +447,7 @@ func GetIGDBIDBySteamIDs(ids []int) (map[int]int, error) {
 	return ret, nil
 }
 
-func GetIGDBIDBySteamIDsCache(ids []int) (map[int]int, error) {
+func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
 	res := make(map[int]int)
 	notExistIDs := make([]int, 0)
 	if config.Config.RedisAvaliable {
@@ -463,7 +464,7 @@ func GetIGDBIDBySteamIDsCache(ids []int) (map[int]int, error) {
 		if len(res) == len(ids) {
 			return res, nil
 		}
-		idMap, err := GetIGDBIDBySteamIDs(notExistIDs)
+		idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
 		if err != nil {
 			return nil, err
 		}
@@ -475,6 +476,6 @@ func GetIGDBIDBySteamIDsCache(ids []int) (map[int]int, error) {
 		}
 		return res, nil
 	} else {
-		return GetIGDBIDBySteamIDs(ids)
+		return GetIGDBIDsBySteamIDs(ids)
 	}
 }

crawler/kaoskrew.go

@@ -31,19 +31,19 @@ func (c *KaOsKrewCrawler) Name() string {
 	return "KaOsKrewCrawler"
 }
 
-func (c *KaOsKrewCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *KaOsKrewCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	return c.crawler.Crawl(page)
 }
 
-func (c *KaOsKrewCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *KaOsKrewCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	return c.crawler.CrawlByUrl(url)
 }
 
-func (c *KaOsKrewCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *KaOsKrewCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
 	return c.crawler.CrawlMulti(pages)
 }
 
-func (c *KaOsKrewCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *KaOsKrewCrawler) CrawlAll() ([]*model.GameItem, error) {
 	return c.crawler.CrawlAll()
 }
 

crawler/onlinefix.go

@@ -37,7 +37,7 @@ func (c *OnlineFixCrawler) Name() string {
 	return "OnlineFixCrawler"
 }
 
-func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	if !config.Config.OnlineFixAvaliable {
 		c.logger.Error("Need Online Fix account")
 		return nil, errors.New("Online Fix is not available")
@@ -77,7 +77,7 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 		)
 	})
 
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i, u := range urls {
 		if db.IsOnlineFixCrawled(updateFlags[i]) {
 			continue
@@ -89,13 +89,13 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			continue
 		}
 		item.UpdateFlag = updateFlags[i]
-		err = db.SaveGameDownload(item)
+		err = db.SaveGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err))
 			continue
 		}
 		res = append(res, item)
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -109,7 +109,7 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	if len(c.cookies) == 0 {
 		err := c.login()
 		if err != nil {
@@ -137,7 +137,7 @@ func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	if len(downloadRegexRes) == 0 {
 		return nil, errors.New("Failed to find download button")
 	}
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -215,8 +215,8 @@ func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for _, page := range pages {
 		items, err := c.Crawl(page)
 		if err != nil {
@@ -227,8 +227,8 @@ func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error
 	return res, nil
 }
 
-func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err

crawler/steam.go

@@ -176,7 +176,7 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
 	return item, nil
 }
 
-func OrganizeGameDownloadWithSteam(id int, game *model.GameDownload) (*model.GameInfo, error) {
+func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, error) {
 	var err error
 	if id == 0 {
 		id, err = GetSteamIDCache(game.Name)

crawler/steam250.go

@@ -45,29 +45,13 @@ func GetSteam250(url string) ([]*model.GameInfo, error) {
 	})
 	var res []*model.GameInfo
 	count := 0
-	idMap, err := GetIGDBIDBySteamIDsCache(steamIDs)
+	for _, steamID := range steamIDs {
-	if err != nil {
+		info, err := db.GetGameInfoByPlatformID("steam", steamID)
-		return nil, err
-	}
-	for _, item := range rank {
-		if count == 10 {
-			break
-		}
-		if idMap[item.SteamID] != 0 {
-			info, err := db.GetGameInfoByPlatformID("igdb", idMap[item.SteamID])
 		if err == nil {
 			res = append(res, info)
 			count++
 			continue
 		}
-		} else {
-			info, err := db.GetGameInfoByPlatformID("steam", item.SteamID)
-			if err == nil {
-				res = append(res, info)
-				count++
-				continue
-			}
-		}
 	}
 	return res, nil
 }
@@ -124,7 +108,7 @@ func GetSteam250Cache(k string, f func() ([]*model.GameInfo, error)) ([]*model.G
 			if err != nil {
 				return data, nil
 			}
-			err = cache.AddWithExpire(key, dataBytes, 24*time.Hour)
+			err = cache.AddWithExpire(key, dataBytes, 12*time.Hour)
 			if err != nil {
 				return data, nil
 			}

crawler/steamrip.go

@@ -30,7 +30,7 @@ func (c *SteamRIPCrawler) Name() string {
 	return "SteamRIPCrawler"
 }
 
-func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
@@ -41,7 +41,7 @@ func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	if err != nil {
 		return nil, err
 	}
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -82,7 +82,7 @@ func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
+func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
 	count := 0
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: constant.SteamRIPGameListURL,
@@ -94,7 +94,7 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 	if err != nil {
 		return nil, err
 	}
-	var items []*model.GameDownload
+	var items []*model.GameItem
 	urls := []string{}
 	updateFlags := []string{} // title
 	doc.Find(".az-list-item>a").Each(func(i int, s *goquery.Selection) {
@@ -119,13 +119,13 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 			continue
 		}
 		item.UpdateFlag = updateFlags[i]
-		if err := db.SaveGameDownload(item); err != nil {
+		if err := db.SaveGameItem(item); err != nil {
 			c.logger.Error("Failed to save item", zap.Error(err))
 			continue
 		}
 		items = append(items, item)
 		count++
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -139,7 +139,7 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
 	return items, nil
 }
 
-func (c *SteamRIPCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *SteamRIPCrawler) CrawlAll() ([]*model.GameItem, error) {
 	return c.Crawl(-1)
 }
 

crawler/xatab.go

@@ -31,7 +31,7 @@ func (c *XatabCrawler) Name() string {
 	return "XatabCrawler"
 }
 
-func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
+func (c *XatabCrawler) Crawl(page int) ([]*model.GameItem, error) {
 	requestURL := fmt.Sprintf("%s/page/%v", constant.XatabBaseURL, page)
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: requestURL,
@@ -55,7 +55,7 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 		urls = append(urls, u)
 		updateFlags = append(updateFlags, s.Find(".entry__title.h2 a").Text())
 	})
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i, u := range urls {
 		if db.IsXatabCrawled(updateFlags[i]) {
 			continue
@@ -66,13 +66,13 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 			c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
 			continue
 		}
-		err = db.SaveGameDownload(item)
+		err = db.SaveGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to save", zap.Error(err))
 			continue
 		}
 		res = append(res, item)
-		info, err := OrganizeGameDownload(item)
+		info, err := OrganizeGameItem(item)
 		if err != nil {
 			c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
 			continue
@@ -86,7 +86,7 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
+func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
 	resp, err := utils.Fetch(utils.FetchConfig{
 		Url: url,
 	})
@@ -97,7 +97,7 @@ func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	if err != nil {
 		return nil, err
 	}
-	item, err := db.GetGameDownloadByUrl(url)
+	item, err := db.GetGameItemByUrl(url)
 	if err != nil {
 		return nil, err
 	}
@@ -126,12 +126,12 @@ func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
 	return item, nil
 }
 
-func (c *XatabCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
+func (c *XatabCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err
 	}
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for _, page := range pages {
 		if page > totalPageNum {
 			continue
@@ -145,12 +145,12 @@ func (c *XatabCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
 	return res, nil
 }
 
-func (c *XatabCrawler) CrawlAll() ([]*model.GameDownload, error) {
+func (c *XatabCrawler) CrawlAll() ([]*model.GameItem, error) {
 	totalPageNum, err := c.GetTotalPageNum()
 	if err != nil {
 		return nil, err
 	}
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	for i := 1; i <= totalPageNum; i++ {
 		items, err := c.Crawl(i)
 		if err != nil {

db/1337x.go

@@ -4,10 +4,10 @@ import (
 	"github.com/nitezs/pcgamedb/model"
 )
 
-func GetDODIGameDownloads() ([]*model.GameDownload, error) {
+func GetDODIGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("dodi")
+	return GetGameItemsByAuthor("dodi")
 }
 
-func GetKaOsKrewGameDownloads() ([]*model.GameDownload, error) {
+func GetKaOsKrewGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("kaoskrew")
+	return GetGameItemsByAuthor("kaoskrew")
 }

db/armgddn.go

@@ -6,6 +6,6 @@ func IsARMGDDNCrawled(flag string) bool {
 	return IsGameCrawled(flag, "armgddn")
 }
 
-func GetARMGDDNGameDownloads() ([]*model.GameDownload, error) {
+func GetARMGDDNGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("armgddn")
+	return GetGameItemsByAuthor("armgddn")
 }

db/db.go

@@ -16,14 +16,14 @@ import (
 )
 
 const (
-	gameDownloadCollectionName = "game_downloads"
+	gameDownloadCollectionName = "games"
 	gameInfoCollectionName     = "game_infos"
 )
 
 var (
 	mongoDB            *mongo.Client
 	mutx               = &sync.RWMutex{}
-	GameDownloadCollection = &CustomCollection{
+	GameItemCollection = &CustomCollection{
 		collName: gameDownloadCollectionName,
 	}
 	GameInfoCollection = &CustomCollection{

db/export.go

@@ -0,0 +1,42 @@
+package db
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/nitezs/pcgamedb/model"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func Export() ([]byte, []byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	infos := []model.GameInfo{}
+	games := []model.GameItem{}
+	cursor, err := GameInfoCollection.Find(ctx, bson.M{})
+	if err != nil {
+		return nil, nil, err
+	}
+	defer cursor.Close(ctx)
+	if err = cursor.All(ctx, &infos); err != nil {
+		return nil, nil, err
+	}
+	cursor, err = GameItemCollection.Find(ctx, bson.M{})
+	if err != nil {
+		return nil, nil, err
+	}
+	defer cursor.Close(ctx)
+	if err = cursor.All(ctx, &games); err != nil {
+		return nil, nil, err
+	}
+	infoJson, err := json.Marshal(infos)
+	if err != nil {
+		return nil, nil, err
+	}
+	gameJson, err := json.Marshal(games)
+	if err != nil {
+		return nil, nil, err
+	}
+	return infoJson, gameJson, nil
+}

db/fitgirl.go

@@ -2,8 +2,8 @@ package db
 
 import "github.com/nitezs/pcgamedb/model"
 
-func GetFitgirlAllGameDownloads() ([]*model.GameDownload, error) {
+func GetFitgirlAllGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("fitgirl")
+	return GetGameItemsByAuthor("fitgirl")
 }
 
 func IsFitgirlCrawled(flag string) bool {

db/freegog.go

@@ -4,8 +4,8 @@ import (
 	"github.com/nitezs/pcgamedb/model"
 )
 
-func GetFreeGOGGameDownloads() ([]*model.GameDownload, error) {
+func GetFreeGOGGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("freegog")
+	return GetGameItemsByAuthor("freegog")
 }
 func IsFreeGOGCrawled(flag string) bool {
 	return IsGameCrawled(flag, "freegog")

db/game.go

@@ -25,12 +25,12 @@ var (
 	removeRepeatingSpacesRegex = regexp.MustCompile(`\s+`)
 )
 
-func GetGameDownloadsByAuthor(regex string) ([]*model.GameDownload, error) {
+func GetGameItemsByAuthor(regex string) ([]*model.GameItem, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	filter := bson.D{{Key: "author", Value: primitive.Regex{Pattern: regex, Options: "i"}}}
-	cursor, err := GameDownloadCollection.Find(ctx, filter)
+	cursor, err := GameItemCollection.Find(ctx, filter)
 	if err != nil {
 		return nil, err
 	}
@@ -44,20 +44,20 @@ func GetGameDownloadsByAuthor(regex string) ([]*model.GameDownload, error) {
 	return res, err
 }
 
-func GetGameDownloadsByAuthorPagination(regex string, page int, pageSize int) ([]*model.GameDownload, int, error) {
+func GetGameItemsByAuthorPagination(regex string, page int, pageSize int) ([]*model.GameItem, int, error) {
-	var res []*model.GameDownload
+	var res []*model.GameItem
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	filter := bson.D{{Key: "author", Value: primitive.Regex{Pattern: regex, Options: "i"}}}
 	opts := options.Find()
 	opts.SetSkip(int64((page - 1) * pageSize))
 	opts.SetLimit(int64(pageSize))
-	totalCount, err := GameDownloadCollection.CountDocuments(ctx, filter)
+	totalCount, err := GameItemCollection.CountDocuments(ctx, filter)
 	if err != nil {
 		return nil, 0, err
 	}
 	totalPage := (totalCount + int64(pageSize) - 1) / int64(pageSize)
-	cursor, err := GameDownloadCollection.Find(ctx, filter, opts)
+	cursor, err := GameItemCollection.Find(ctx, filter, opts)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -78,8 +78,8 @@ func IsGameCrawled(flag string, author string) bool {
 		{Key: "author", Value: primitive.Regex{Pattern: author, Options: "i"}},
 		{Key: "update_flag", Value: flag},
 	}
-	var game model.GameDownload
+	var game model.GameItem
-	err := GameDownloadCollection.FindOne(ctx, filter).Decode(&game)
+	err := GameItemCollection.FindOne(ctx, filter).Decode(&game)
 	if err != nil {
 		if errors.Is(err, mongo.ErrNoDocuments) {
 			return false
@@ -95,8 +95,8 @@ func IsGameCrawledByURL(url string) bool {
 	filter := bson.D{
 		{Key: "url", Value: url},
 	}
-	var game model.GameDownload
+	var game model.GameItem
-	err := GameDownloadCollection.FindOne(ctx, filter).Decode(&game)
+	err := GameItemCollection.FindOne(ctx, filter).Decode(&game)
 	if err != nil {
 		if errors.Is(err, mongo.ErrNoDocuments) {
 			return false
@@ -106,7 +106,7 @@ func IsGameCrawledByURL(url string) bool {
 	return true
 }
 
-func SaveGameDownload(item *model.GameDownload) error {
+func SaveGameItem(item *model.GameItem) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	if item.ID.IsZero() {
@@ -121,7 +121,7 @@ func SaveGameDownload(item *model.GameDownload) error {
 	filter := bson.M{"_id": item.ID}
 	update := bson.M{"$set": item}
 	opts := options.Update().SetUpsert(true)
-	_, err := GameDownloadCollection.UpdateOne(ctx, filter, update, opts)
+	_, err := GameItemCollection.UpdateOne(ctx, filter, update, opts)
 	if err != nil {
 		return err
 	}
@@ -148,17 +148,17 @@ func SaveGameInfo(item *model.GameInfo) error {
 	return nil
 }
 
-func GetAllGameDownloads() ([]*model.GameDownload, error) {
+func GetAllGameItems() ([]*model.GameItem, error) {
-	var items []*model.GameDownload
+	var items []*model.GameItem
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cursor, err := GameDownloadCollection.Find(ctx, bson.D{})
+	cursor, err := GameItemCollection.Find(ctx, bson.D{})
 	if err != nil {
 		return nil, err
 	}
 	defer cursor.Close(ctx)
 	for cursor.Next(ctx) {
-		var game model.GameDownload
+		var game model.GameItem
 		if err = cursor.Decode(&game); err != nil {
 			return nil, err
 		}
@@ -170,44 +170,44 @@ func GetAllGameDownloads() ([]*model.GameDownload, error) {
 	return items, err
 }
 
-func GetGameDownloadByUrl(url string) (*model.GameDownload, error) {
+func GetGameItemByUrl(url string) (*model.GameItem, error) {
-	var item model.GameDownload
+	var item model.GameItem
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	filter := bson.M{"url": url}
-	err := GameDownloadCollection.FindOne(ctx, filter).Decode(&item)
+	err := GameItemCollection.FindOne(ctx, filter).Decode(&item)
 	if err != nil {
 		if errors.Is(err, mongo.ErrNoDocuments) {
-			return &model.GameDownload{}, nil
+			return &model.GameItem{}, nil
 		}
 		return nil, err
 	}
 	return &item, nil
 }
 
-func GetGameDownloadByID(id primitive.ObjectID) (*model.GameDownload, error) {
+func GetGameItemByID(id primitive.ObjectID) (*model.GameItem, error) {
-	var item model.GameDownload
+	var item model.GameItem
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	filter := bson.M{"_id": id}
-	err := GameDownloadCollection.FindOne(ctx, filter).Decode(&item)
+	err := GameItemCollection.FindOne(ctx, filter).Decode(&item)
 	if err != nil {
 		return nil, err
 	}
 	return &item, nil
 }
 
-func GetGameDownloadsByIDs(ids []primitive.ObjectID) ([]*model.GameDownload, error) {
+func GetGameItemsByIDs(ids []primitive.ObjectID) ([]*model.GameItem, error) {
-	var items []*model.GameDownload
+	var items []*model.GameItem
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	cursor, err := GameDownloadCollection.Find(ctx, bson.M{"_id": bson.M{"$in": ids}})
+	cursor, err := GameItemCollection.Find(ctx, bson.M{"_id": bson.M{"$in": ids}})
 	if err != nil {
 		return nil, err
 	}
 	defer cursor.Close(ctx)
 	for cursor.Next(ctx) {
-		var game model.GameDownload
+		var game model.GameItem
 		if err = cursor.Decode(&game); err != nil {
 			return nil, err
 		}
@@ -250,7 +250,7 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
 		if err = cursor.Decode(&game); err != nil {
 			return nil, 0, err
 		}
-		game.Games, err = GetGameDownloadsByIDs(game.GameIDs)
+		game.Games, err = GetGameItemsByIDs(game.GameIDs)
 		if err != nil {
 			return nil, 0, err
 		}
@@ -287,7 +287,7 @@ func SearchGameInfosCache(name string, page int, pageSize int) ([]*model.GameInf
 			if err != nil {
 				return nil, 0, err
 			}
-			_ = cache.AddWithExpire(key, string(dataBytes), 12*time.Hour)
+			_ = cache.AddWithExpire(key, string(dataBytes), 5*time.Minute)
 			return data, totalPage, nil
 		}
 	} else {
@@ -315,10 +315,10 @@ func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {
 	return &game, nil
 }
 
-func GetUnorganizedGameDownloads(num int) ([]*model.GameDownload, error) {
+func GetUnorganizedGameItems(num int) ([]*model.GameItem, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	var gamesNotInDetails []*model.GameDownload
+	var gamesNotInDetails []*model.GameItem
 	pipeline := mongo.Pipeline{
 		bson.D{{Key: "$lookup", Value: bson.D{
 			{Key: "from", Value: "game_infos"},
@@ -337,14 +337,14 @@ func GetUnorganizedGameDownloads(num int) ([]*model.GameDownload, error) {
 		bson.D{{Key: "$sort", Value: bson.D{{Key: "name", Value: 1}}}},
 	)
 
-	cursor, err := GameDownloadCollection.Aggregate(ctx, pipeline)
+	cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
 	if err != nil {
 		return nil, err
 	}
 	defer cursor.Close(ctx)
 
 	for cursor.Next(ctx) {
-		var game model.GameDownload
+		var game model.GameItem
 		if err := cursor.Decode(&game); err != nil {
 			return nil, err
 		}
@@ -392,7 +392,7 @@ func DeduplicateGames() ([]primitive.ObjectID, error) {
 			{Key: "total", Value: bson.D{{Key: "$gt", Value: 1}}},
 		}}},
 	}
-	cursor, err := GameDownloadCollection.Aggregate(ctx, pipeline)
+	cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
 	if err != nil {
 		return nil, err
 	}
@@ -402,7 +402,7 @@ func DeduplicateGames() ([]primitive.ObjectID, error) {
 	for _, item := range qres {
 		idsToDelete := item.IDs[:len(item.IDs)-1]
 		res = append(res, idsToDelete...)
-		_, err = GameDownloadCollection.DeleteMany(ctx, bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToDelete}}}})
+		_, err = GameItemCollection.DeleteMany(ctx, bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToDelete}}}})
 		if err != nil {
 			return nil, err
 		}
@@ -522,17 +522,17 @@ func GetGameInfosByName(name string) ([]*model.GameInfo, error) {
 	return games, nil
 }
 
-func GetGameDownloadByRawName(name string) ([]*model.GameDownload, error) {
+func GetGameItemByRawName(name string) ([]*model.GameItem, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 	name = strings.TrimSpace(name)
 	name = fmt.Sprintf("^%s$", name)
 	filter := bson.M{"raw_name": bson.M{"$regex": primitive.Regex{Pattern: name, Options: "i"}}}
-	cursor, err := GameDownloadCollection.Find(ctx, filter)
+	cursor, err := GameItemCollection.Find(ctx, filter)
 	if err != nil {
 		return nil, err
 	}
-	var game []*model.GameDownload
+	var game []*model.GameItem
 	if err = cursor.All(ctx, &game); err != nil {
 		return nil, err
 	}
@@ -622,10 +622,10 @@ func GetGameInfoCount() (int64, error) {
 	return count, nil
 }
 
-func GetGameDownloadCount() (int64, error) {
+func GetGameItemCount() (int64, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	count, err := GameDownloadCollection.CountDocuments(ctx, bson.M{})
+	count, err := GameItemCollection.CountDocuments(ctx, bson.M{})
 	if err != nil {
 		return 0, err
 	}
@@ -661,10 +661,10 @@ func DeleteGameInfoByID(id primitive.ObjectID) error {
 	return nil
 }
 
-func DeleteGameDownloadByID(id primitive.ObjectID) error {
+func DeleteGameItemByID(id primitive.ObjectID) error {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
-	_, err := GameDownloadCollection.DeleteOne(ctx, bson.M{"_id": id})
+	_, err := GameItemCollection.DeleteOne(ctx, bson.M{"_id": id})
 	if err != nil {
 		return err
 	}
@@ -702,7 +702,7 @@ func GetAllAuthors() ([]string, error) {
 		}}},
 	}
 
-	cursor, err := GameDownloadCollection.Aggregate(ctx, pipeline)
+	cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
 	if err != nil {
 		return nil, err
 	}

db/onlinefix.go

@@ -4,8 +4,8 @@ import (
 	"github.com/nitezs/pcgamedb/model"
 )
 
-func GetOnlineFixGameDownloads() ([]*model.GameDownload, error) {
+func GetOnlineFixGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("onlinefix")
+	return GetGameItemsByAuthor("onlinefix")
 }
 
 func IsOnlineFixCrawled(flag string) bool {

db/xatab.go

@@ -4,8 +4,8 @@ import (
 	"github.com/nitezs/pcgamedb/model"
 )
 
-func GetXatabGameDownloads() ([]*model.GameDownload, error) {
+func GetXatabGameItems() ([]*model.GameItem, error) {
-	return GetGameDownloadsByAuthor("xatab")
+	return GetGameItemsByAuthor("xatab")
 }
 
 func IsXatabCrawled(flag string) bool {

model/game.go

@@ -20,12 +20,12 @@ type GameInfo struct {
 	Languages   []string             `json:"languages" bson:"languages"`
 	Screenshots []string             `json:"screenshots" bson:"screenshots"`
 	GameIDs     []primitive.ObjectID `json:"game_ids" bson:"games"`
-	Games       []*GameDownload      `json:"game_downloads" bson:"-"`
+	Games       []*GameItem          `json:"game_downloads" bson:"-"`
 	CreatedAt   time.Time            `json:"created_at" bson:"created_at"`
 	UpdatedAt   time.Time            `json:"updated_at" bson:"updated_at"`
 }
 
-type GameDownload struct {
+type GameItem struct {
 	ID         primitive.ObjectID `json:"id" bson:"_id"`
 	Name       string             `json:"speculative_name" bson:"name"`
 	RawName    string             `json:"raw_name,omitempty" bson:"raw_name"`

server/handler/get_game_by_id.go

@@ -11,31 +11,31 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
-type GetGameDownloadByIDRequest struct {
+type GetGameItemByIDRequest struct {
 	ID string `uri:"id" binding:"required"`
 }
 
-type GetGameDownloadByIDResponse struct {
+type GetGameItemByIDResponse struct {
 	Status  string          `json:"status"`
 	Message string          `json:"message,omitempty"`
-	Game    *model.GameDownload `json:"game,omitempty"`
+	Game    *model.GameItem `json:"game,omitempty"`
 }
 
-// GetGameDownloadByID retrieves game download details by ID.
+// GetGameItemByID retrieves game download details by ID.
 // @Summary Retrieve game download by ID
 // @Description Retrieves details of a game download by game ID
 // @Tags game
 // @Accept json
 // @Produce json
 // @Param id path string true "Game Download ID"
-// @Success 200 {object} GetGameDownloadByIDResponse
+// @Success 200 {object} GetGameItemByIDResponse
-// @Failure 400 {object} GetGameDownloadByIDResponse
+// @Failure 400 {object} GetGameItemByIDResponse
-// @Failure 500 {object} GetGameDownloadByIDResponse
+// @Failure 500 {object} GetGameItemByIDResponse
 // @Router /game/raw/id/{id} [get]
-func GetGameDownloadByIDHanlder(c *gin.Context) {
+func GetGameItemByIDHanlder(c *gin.Context) {
-	var req GetGameDownloadByIDRequest
+	var req GetGameItemByIDRequest
 	if err := c.ShouldBindUri(&req); err != nil {
-		c.JSON(http.StatusBadRequest, GetGameDownloadByIDResponse{
+		c.JSON(http.StatusBadRequest, GetGameItemByIDResponse{
 			Status:  "error",
 			Message: err.Error(),
 		})
@@ -43,28 +43,28 @@ func GetGameDownloadByIDHanlder(c *gin.Context) {
 	}
 	id, err := primitive.ObjectIDFromHex(req.ID)
 	if err != nil {
-		c.JSON(http.StatusBadRequest, GetGameDownloadByIDResponse{
+		c.JSON(http.StatusBadRequest, GetGameItemByIDResponse{
 			Status:  "error",
 			Message: err.Error(),
 		})
 		return
 	}
-	game, err := db.GetGameDownloadByID(id)
+	game, err := db.GetGameItemByID(id)
 	if err != nil {
 		if err == mongo.ErrNoDocuments {
-			c.JSON(http.StatusOK, GetGameDownloadByIDResponse{
+			c.JSON(http.StatusOK, GetGameItemByIDResponse{
 				Status:  "ok",
 				Message: "No results found",
 			})
 			return
 		}
-		c.JSON(http.StatusInternalServerError, GetGameDownloadByIDResponse{
+		c.JSON(http.StatusInternalServerError, GetGameItemByIDResponse{
 			Status:  "error",
 			Message: err.Error(),
 		})
 		return
 	}
-	c.JSON(http.StatusOK, GetGameDownloadByIDResponse{
+	c.JSON(http.StatusOK, GetGameItemByIDResponse{
 		Status: "ok",
 		Game:   game,
 	})

server/handler/get_game_by_raw_name.go

@@ -0,0 +1,69 @@
+package handler
+
+import (
+	"net/http"
+
+	"github.com/nitezs/pcgamedb/db"
+	"github.com/nitezs/pcgamedb/model"
+
+	"github.com/gin-gonic/gin"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type GetGameItemByRawNameRequest struct {
+	Name string `uri:"name" binding:"required"`
+}
+
+type GetGameItemByRawNameResponse struct {
+	Status   string            `json:"status"`
+	Message  string            `json:"message,omitempty"`
+	GameItem []*model.GameItem `json:"game_downloads,omitempty"`
+}
+
+// GetGameItemByRawName retrieves game download details by raw name.
+// @Summary Retrieve game download by raw name
+// @Description Retrieves details of a game download by its raw name
+// @Tags game
+// @Accept json
+// @Produce json
+// @Param name path string true "Game Download Raw Name"
+// @Success 200 {object} GetGameItemByRawNameResponse
+// @Failure 400 {object} GetGameItemByRawNameResponse
+// @Failure 500 {object} GetGameItemByRawNameResponse
+// @Router /game/raw/name/{name} [get]
+func GetGameItemByRawNameHandler(c *gin.Context) {
+	var req GetGameItemByRawNameRequest
+	if err := c.ShouldBindUri(&req); err != nil {
+		c.JSON(http.StatusBadRequest, GetGameItemByRawNameResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	gameDownload, err := db.GetGameItemByRawName(req.Name)
+	if err != nil {
+		if err == mongo.ErrNoDocuments {
+			c.JSON(http.StatusOK, GetGameItemByRawNameResponse{
+				Status:  "ok",
+				Message: "No results found",
+			})
+			return
+		}
+		c.JSON(http.StatusInternalServerError, GetGameItemByRawNameResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	if gameDownload == nil {
+		c.JSON(http.StatusOK, GetGameItemByRawNameResponse{
+			Status:  "ok",
+			Message: "No results found",
+		})
+		return
+	}
+	c.JSON(http.StatusOK, GetGameItemByRawNameResponse{
+		Status:   "ok",
+		GameItem: gameDownload,
+	})
+}

server/handler/get_game_download_by_raw_name.go

@@ -1,69 +0,0 @@
-package handler
-
-import (
-	"net/http"
-
-	"github.com/nitezs/pcgamedb/db"
-	"github.com/nitezs/pcgamedb/model"
-
-	"github.com/gin-gonic/gin"
-	"go.mongodb.org/mongo-driver/mongo"
-)
-
-type GetGameDownloadByRawNameRequest struct {
-	Name string `uri:"name" binding:"required"`
-}
-
-type GetGameDownloadByRawNameResponse struct {
-	Status       string                `json:"status"`
-	Message      string                `json:"message,omitempty"`
-	GameDownload []*model.GameDownload `json:"game_downloads,omitempty"`
-}
-
-// GetGameDownloadByRawName retrieves game download details by raw name.
-// @Summary Retrieve game download by raw name
-// @Description Retrieves details of a game download by its raw name
-// @Tags game
-// @Accept json
-// @Produce json
-// @Param name path string true "Game Download Raw Name"
-// @Success 200 {object} GetGameDownloadByRawNameResponse
-// @Failure 400 {object} GetGameDownloadByRawNameResponse
-// @Failure 500 {object} GetGameDownloadByRawNameResponse
-// @Router /game/raw/name/{name} [get]
-func GetGameDownloadByRawNameHandler(c *gin.Context) {
-	var req GetGameDownloadByRawNameRequest
-	if err := c.ShouldBindUri(&req); err != nil {
-		c.JSON(http.StatusBadRequest, GetGameDownloadByRawNameResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	gameDownload, err := db.GetGameDownloadByRawName(req.Name)
-	if err != nil {
-		if err == mongo.ErrNoDocuments {
-			c.JSON(http.StatusOK, GetGameDownloadByRawNameResponse{
-				Status:  "ok",
-				Message: "No results found",
-			})
-			return
-		}
-		c.JSON(http.StatusInternalServerError, GetGameDownloadByRawNameResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	if gameDownload == nil {
-		c.JSON(http.StatusOK, GetGameDownloadByRawNameResponse{
-			Status:  "ok",
-			Message: "No results found",
-		})
-		return
-	}
-	c.JSON(http.StatusOK, GetGameDownloadByRawNameResponse{
-		Status:       "ok",
-		GameDownload: gameDownload,
-	})
-}

server/handler/get_game_downloads_by_author.go

@@ -1,83 +0,0 @@
-package handler
-
-import (
-	"net/http"
-
-	"github.com/nitezs/pcgamedb/db"
-	"github.com/nitezs/pcgamedb/model"
-
-	"github.com/gin-gonic/gin"
-)
-
-type GetGameDownloadsByAuthorRequest struct {
-	Author   string `uri:"author" binding:"required"`
-	Page     int    `form:"page" json:"page"`
-	PageSize int    `form:"page_size" json:"page_size"`
-}
-
-type GetGameDownloadsByAuthorResponse struct {
-	Status        string                `json:"status"`
-	Message       string                `json:"message,omitempty"`
-	TotalPage     int                   `json:"total_page"`
-	GameDownloads []*model.GameDownload `json:"game_downloads,omitempty"`
-}
-
-// GetGameDownloadsByAuthorHandler returns all game downloads by author
-// @Summary Get game downloads by author
-// @Description Get game downloads by author
-// @Tags game
-// @Accept json
-// @Produce json
-// @Param author path string true "Author"
-// @Param page query int false "Page"
-// @Param page_size query int false "Page Size"
-// @Success 200 {object} GetGameDownloadsByAuthorResponse
-// @Failure 400 {object} GetGameDownloadsByAuthorResponse
-// @Failure 500 {object} GetGameDownloadsByAuthorResponse
-// @Router /game/raw/author/{author} [get]
-func GetGameDownloadsByAuthorHandler(ctx *gin.Context) {
-	var req GetGameDownloadsByAuthorRequest
-	if err := ctx.ShouldBindUri(&req); err != nil {
-		ctx.JSON(http.StatusBadRequest, GetGameDownloadsByAuthorResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	if err := ctx.ShouldBind(&req); err != nil {
-		ctx.JSON(http.StatusBadRequest, GetGameDownloadsByAuthorResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	if req.Page == 0 || req.Page < 0 {
-		req.Page = 1
-	}
-	if req.PageSize == 0 || req.PageSize < 0 {
-		req.PageSize = 10
-	}
-	if req.PageSize > 10 {
-		req.PageSize = 10
-	}
-	downloads, totalPage, err := db.GetGameDownloadsByAuthorPagination(req.Author, req.Page, req.PageSize)
-	if err != nil {
-		ctx.JSON(http.StatusInternalServerError, GetGameDownloadsByAuthorResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	if len(downloads) == 0 {
-		ctx.JSON(http.StatusOK, GetGameDownloadsByAuthorResponse{
-			Status:  "ok",
-			Message: "No results found",
-		})
-		return
-	}
-	ctx.JSON(http.StatusOK, GetGameDownloadsByAuthorResponse{
-		Status:        "ok",
-		TotalPage:     totalPage,
-		GameDownloads: downloads,
-	})
-}

server/handler/get_game_info_by_id.go

@@ -33,7 +33,7 @@ type GetGameInfoByIDResponse struct {
 // @Failure 500 {object} GetGameInfoByIDResponse
 // @Router /game/id/{id} [get]
 func GetGameInfoByIDHandler(c *gin.Context) {
-	var req GetGameDownloadByIDRequest
+	var req GetGameItemByIDRequest
 	if err := c.ShouldBindUri(&req); err != nil {
 		c.JSON(http.StatusBadRequest, GetGameInfoByIDResponse{
 			Status:  "error",
@@ -64,7 +64,7 @@ func GetGameInfoByIDHandler(c *gin.Context) {
 		})
 		return
 	}
-	gameInfo.Games, err = db.GetGameDownloadsByIDs(gameInfo.GameIDs)
+	gameInfo.Games, err = db.GetGameItemsByIDs(gameInfo.GameIDs)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, GetGameInfoByIDResponse{
 			Status:  "error",

server/handler/get_games_by_author.go

@@ -0,0 +1,83 @@
+package handler
+
+import (
+	"net/http"
+
+	"github.com/nitezs/pcgamedb/db"
+	"github.com/nitezs/pcgamedb/model"
+
+	"github.com/gin-gonic/gin"
+)
+
+type GetGameItemsByAuthorRequest struct {
+	Author   string `uri:"author" binding:"required"`
+	Page     int    `form:"page" json:"page"`
+	PageSize int    `form:"page_size" json:"page_size"`
+}
+
+type GetGameItemsByAuthorResponse struct {
+	Status    string            `json:"status"`
+	Message   string            `json:"message,omitempty"`
+	TotalPage int               `json:"total_page"`
+	GameItems []*model.GameItem `json:"game_downloads,omitempty"`
+}
+
+// GetGameItemsByAuthorHandler returns all game downloads by author
+// @Summary Get game downloads by author
+// @Description Get game downloads by author
+// @Tags game
+// @Accept json
+// @Produce json
+// @Param author path string true "Author"
+// @Param page query int false "Page"
+// @Param page_size query int false "Page Size"
+// @Success 200 {object} GetGameItemsByAuthorResponse
+// @Failure 400 {object} GetGameItemsByAuthorResponse
+// @Failure 500 {object} GetGameItemsByAuthorResponse
+// @Router /game/raw/author/{author} [get]
+func GetGameItemsByAuthorHandler(ctx *gin.Context) {
+	var req GetGameItemsByAuthorRequest
+	if err := ctx.ShouldBindUri(&req); err != nil {
+		ctx.JSON(http.StatusBadRequest, GetGameItemsByAuthorResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	if err := ctx.ShouldBind(&req); err != nil {
+		ctx.JSON(http.StatusBadRequest, GetGameItemsByAuthorResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	if req.Page == 0 || req.Page < 0 {
+		req.Page = 1
+	}
+	if req.PageSize == 0 || req.PageSize < 0 {
+		req.PageSize = 10
+	}
+	if req.PageSize > 10 {
+		req.PageSize = 10
+	}
+	downloads, totalPage, err := db.GetGameItemsByAuthorPagination(req.Author, req.Page, req.PageSize)
+	if err != nil {
+		ctx.JSON(http.StatusInternalServerError, GetGameItemsByAuthorResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	if len(downloads) == 0 {
+		ctx.JSON(http.StatusOK, GetGameItemsByAuthorResponse{
+			Status:  "ok",
+			Message: "No results found",
+		})
+		return
+	}
+	ctx.JSON(http.StatusOK, GetGameItemsByAuthorResponse{
+		Status:    "ok",
+		TotalPage: totalPage,
+		GameItems: downloads,
+	})
+}

server/handler/get_unorganized_game_downloads.go

@@ -1,66 +0,0 @@
-package handler
-
-import (
-	"net/http"
-
-	"github.com/nitezs/pcgamedb/db"
-	"github.com/nitezs/pcgamedb/model"
-
-	"github.com/gin-gonic/gin"
-)
-
-type GetUnorganizedGameDownloadsRequest struct {
-	Num int `json:"num" form:"num"`
-}
-
-type GetUnorganizedGameDownloadsResponse struct {
-	Status        string                `json:"status"`
-	Message       string                `json:"message,omitempty"`
-	Size          int                   `json:"size,omitempty"`
-	GameDownloads []*model.GameDownload `json:"game_downloads,omitempty"`
-}
-
-// GetUnorganizedGameDownloads retrieves a list of unorganized game downloads.
-// @Summary List unorganized game downloads
-// @Description Retrieves game downloads that have not been organized
-// @Tags game
-// @Accept json
-// @Produce json
-// @Param num query int false "Number of game downloads to retrieve"
-// @Success 200 {object} GetUnorganizedGameDownloadsResponse
-// @Failure 400 {object} GetUnorganizedGameDownloadsResponse
-// @Failure 500 {object} GetUnorganizedGameDownloadsResponse
-// @Router /game/raw/unorganized [get]
-func GetUnorganizedGameDownloadsHandler(c *gin.Context) {
-	var req GetUnorganizedGameDownloadsRequest
-	if err := c.ShouldBind(&req); err != nil {
-		c.JSON(http.StatusBadRequest, GetUnorganizedGameDownloadsResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	if req.Num == 0 || req.Num < 0 {
-		req.Num = -1
-	}
-	gameDownloads, err := db.GetUnorganizedGameDownloads(req.Num)
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, GetUnorganizedGameDownloadsResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	if len(gameDownloads) == 0 {
-		c.JSON(http.StatusOK, GetUnorganizedGameDownloadsResponse{
-			Status:  "ok",
-			Message: "No unorganized game downloads found",
-		})
-		return
-	}
-	c.JSON(http.StatusOK, GetUnorganizedGameDownloadsResponse{
-		Status:        "ok",
-		GameDownloads: gameDownloads,
-		Size:          len(gameDownloads),
-	})
-}

server/handler/get_unorganized_games.go

@@ -0,0 +1,66 @@
+package handler
+
+import (
+	"net/http"
+
+	"github.com/nitezs/pcgamedb/db"
+	"github.com/nitezs/pcgamedb/model"
+
+	"github.com/gin-gonic/gin"
+)
+
+type GetUnorganizedGameItemsRequest struct {
+	Num int `json:"num" form:"num"`
+}
+
+type GetUnorganizedGameItemsResponse struct {
+	Status    string            `json:"status"`
+	Message   string            `json:"message,omitempty"`
+	Size      int               `json:"size,omitempty"`
+	GameItems []*model.GameItem `json:"game_downloads,omitempty"`
+}
+
+// GetUnorganizedGameItems retrieves a list of unorganized game downloads.
+// @Summary List unorganized game downloads
+// @Description Retrieves game downloads that have not been organized
+// @Tags game
+// @Accept json
+// @Produce json
+// @Param num query int false "Number of game downloads to retrieve"
+// @Success 200 {object} GetUnorganizedGameItemsResponse
+// @Failure 400 {object} GetUnorganizedGameItemsResponse
+// @Failure 500 {object} GetUnorganizedGameItemsResponse
+// @Router /game/raw/unorganized [get]
+func GetUnorganizedGameItemsHandler(c *gin.Context) {
+	var req GetUnorganizedGameItemsRequest
+	if err := c.ShouldBind(&req); err != nil {
+		c.JSON(http.StatusBadRequest, GetUnorganizedGameItemsResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	if req.Num == 0 || req.Num < 0 {
+		req.Num = -1
+	}
+	gameDownloads, err := db.GetUnorganizedGameItems(req.Num)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, GetUnorganizedGameItemsResponse{
+			Status:  "error",
+			Message: err.Error(),
+		})
+		return
+	}
+	if len(gameDownloads) == 0 {
+		c.JSON(http.StatusOK, GetUnorganizedGameItemsResponse{
+			Status:  "ok",
+			Message: "No unorganized game downloads found",
+		})
+		return
+	}
+	c.JSON(http.StatusOK, GetUnorganizedGameItemsResponse{
+		Status:    "ok",
+		GameItems: gameDownloads,
+		Size:      len(gameDownloads),
+	})
+}

server/handler/healthcheck.go

@@ -21,7 +21,7 @@ type HealthCheckResponse struct {
 	Uptime             string `json:"uptime"`
 	Alloc              string `json:"alloc"`
 	AutoCrawl          bool   `json:"auto_crawl"`
-	GameDownload       int64  `json:"game_download,omitempty"`
+	GameItem           int64  `json:"game_download,omitempty"`
 	GameInfo           int64  `json:"game_info,omitempty"`
 	Unorganized        int64  `json:"unorganized,omitempty"`
 	RedisAvaliable     bool   `json:"redis_avaliable"`
@@ -41,9 +41,9 @@ type HealthCheckResponse struct {
 func HealthCheckHandler(c *gin.Context) {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
-	downloadCount, _ := db.GetGameDownloadCount()
+	downloadCount, _ := db.GetGameItemCount()
 	infoCount, _ := db.GetGameInfoCount()
-	unorganized, err := db.GetUnorganizedGameDownloads(-1)
+	unorganized, err := db.GetUnorganizedGameItems(-1)
 	unorganizedCount := int64(0)
 	if err == nil {
 		unorganizedCount = int64(len(unorganized))
@@ -55,7 +55,7 @@ func HealthCheckHandler(c *gin.Context) {
 		Uptime:             time.Since(config.Runtime.ServerStartTime).String(),
 		AutoCrawl:          config.Config.Server.AutoCrawl,
 		Alloc:              fmt.Sprintf("%.2f MB", float64(m.Alloc)/1024.0/1024.0),
-		GameDownload:       downloadCount,
+		GameItem:           downloadCount,
 		GameInfo:           infoCount,
 		Unorganized:        unorganizedCount,
 		RedisAvaliable:     config.Config.RedisAvaliable,

server/handler/organize_game_info.go

@@ -10,36 +10,36 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-type OrganizeGameDownloadRequest struct {
+type OrganizeGameItemRequest struct {
 	Platform   string `form:"platform" json:"platform" binding:"required"`
 	GameID     string `form:"game_id" json:"game_id" binding:"required"`
 	PlatformID int    `form:"platform_id" json:"platform_id" binding:"required"`
 }
 
-type OrganizeGameDownloadResponse struct {
+type OrganizeGameItemResponse struct {
 	Status   string          `json:"status"`
 	Message  string          `json:"message,omitempty"`
 	GameInfo *model.GameInfo `json:"game_info,omitempty"`
 }
 
-// OrganizeGameDownload organizes a specific game download.
+// OrganizeGameItem organizes a specific game download.
 // @Summary Organize a game download
 // @Description Organizes a game download based on platform and game ID
 // @Tags game
 // @Accept json
 // @Produce json
 // @Param Authorization header string true "Authorization: Bearer <api_key>"
-// @Param body body OrganizeGameDownloadRequest true "Organize Game Download Request"
+// @Param body body OrganizeGameItemRequest true "Organize Game Download Request"
-// @Success 200 {object} OrganizeGameDownloadResponse
+// @Success 200 {object} OrganizeGameItemResponse
-// @Failure 400 {object} OrganizeGameDownloadResponse
+// @Failure 400 {object} OrganizeGameItemResponse
-// @Failure 401 {object} OrganizeGameDownloadResponse
+// @Failure 401 {object} OrganizeGameItemResponse
-// @Failure 500 {object} OrganizeGameDownloadResponse
+// @Failure 500 {object} OrganizeGameItemResponse
 // @Security BearerAuth
 // @Router /game/raw/organize [post]
-func OrganizeGameDownloadHandler(c *gin.Context) {
+func OrganizeGameItemHandler(c *gin.Context) {
-	var req OrganizeGameDownloadRequest
+	var req OrganizeGameItemRequest
 	if err := c.ShouldBind(&req); err != nil {
-		c.JSON(http.StatusBadRequest, OrganizeGameDownloadResponse{
+		c.JSON(http.StatusBadRequest, OrganizeGameItemResponse{
 			Status:  "error",
 			Message: err.Error(),
 		})
@@ -47,21 +47,21 @@ func OrganizeGameDownloadHandler(c *gin.Context) {
 	}
 	objID, err := primitive.ObjectIDFromHex(req.GameID)
 	if err != nil {
-		c.JSON(http.StatusBadRequest, OrganizeGameDownloadResponse{
+		c.JSON(http.StatusBadRequest, OrganizeGameItemResponse{
 			Status:  "error",
 			Message: err.Error(),
 		})
 		return
 	}
-	info, err := crawler.OrganizeGameDownloadManually(objID, req.Platform, req.PlatformID)
+	info, err := crawler.OrganizeGameItemManually(objID, req.Platform, req.PlatformID)
 	if err != nil {
-		c.JSON(http.StatusInternalServerError, OrganizeGameDownloadResponse{
+		c.JSON(http.StatusInternalServerError, OrganizeGameItemResponse{
 			Status:  "error",
 			Message: err.Error(),
 		})
 		return
 	}
-	c.JSON(http.StatusOK, OrganizeGameDownloadResponse{
+	c.JSON(http.StatusOK, OrganizeGameItemResponse{
 		Status:   "ok",
 		GameInfo: info,
 	})

server/route.go

@@ -19,13 +19,13 @@ func initRoute(app *gin.Engine) {
 	}))
 
 	GameInfoGroup := app.Group("/game")
-	GameDownloadGroup := GameInfoGroup.Group("/raw")
+	GameItemGroup := GameInfoGroup.Group("/raw")
 
-	GameDownloadGroup.GET("/unorganized", handler.GetUnorganizedGameDownloadsHandler)
+	GameItemGroup.GET("/unorganized", handler.GetUnorganizedGameItemsHandler)
-	GameDownloadGroup.POST("/organize", middleware.Auth(), handler.OrganizeGameDownloadHandler)
+	GameItemGroup.POST("/organize", middleware.Auth(), handler.OrganizeGameItemHandler)
-	GameDownloadGroup.GET("/id/:id", handler.GetGameDownloadByIDHanlder)
+	GameItemGroup.GET("/id/:id", handler.GetGameItemByIDHanlder)
-	GameDownloadGroup.GET("/name/:name", handler.GetGameDownloadByRawNameHandler)
+	GameItemGroup.GET("/name/:name", handler.GetGameItemByRawNameHandler)
-	GameDownloadGroup.GET("/author/:author", handler.GetGameDownloadsByAuthorHandler)
+	GameItemGroup.GET("/author/:author", handler.GetGameItemsByAuthorHandler)
 
 	GameInfoGroup.GET("/search", handler.SearchGamesHandler)
 	GameInfoGroup.GET("/name/:name", handler.GetGameInfosByNameHandler)

task/crawl.go

@@ -13,7 +13,7 @@ import (
 )
 
 func Crawl(logger *zap.Logger) {
-	var games []*model.GameDownload
+	var games []*model.GameItem
 	var crawlerMap = crawler.BuildCrawlerMap(logger)
 	for _, item := range crawlerMap {
 		logger.Info("Crawling", zap.String("crawler", item.Name()))