nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

add export cmd

Browse Source 
add supplement cmd
rename GameDown to GameItem
This commit is contained in:
Nite07 2024-11-16 13:48:48 +08:00
parent f458a3d1a5
commit 356a76f5b4
48 changed files with 590 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
cmd/export.go Normal file
View File

@ -0,0 +1,54 @@
package cmd
import (
"os"
"path/filepath"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/log"
"go.uber.org/zap"
"github.com/spf13/cobra"
)
type exportCommandConfig struct {
output string
}
var exportCmdCfg exportCommandConfig
var exportCmd = &cobra.Command{
Use: "export",
Long: "Export data to json files",
Short: "Export data to json files",
Run: func(cmd *cobra.Command, args []string) {
infoJson, gameJson, err := db.Export()
if err != nil {
log.Logger.Error("Error exporting data", zap.Error(err))
return
}
infoFilePath := filepath.Join(exportCmdCfg.output, "game_infos.json")
gameFilePath := filepath.Join(exportCmdCfg.output, "games.json")
err = os.WriteFile(infoFilePath, infoJson, 0644)
if err != nil {
if err != nil {
log.Logger.Error("Error exporting data", zap.Error(err))
return
}
}
err = os.WriteFile(gameFilePath, gameJson, 0644)
if err != nil {
if err != nil {
log.Logger.Error("Error exporting data", zap.Error(err))
return
}
}
log.Logger.Info("Data exported successfully")
},
}
func init() {
exportCmd.Flags().StringVarP(&exportCmdCfg.output, "output", "o", "", "Output directory for json files")
RootCmd.AddCommand(exportCmd)
}

24
cmd/format.go
View File

@ -33,7 +33,7 @@ func formatRun(cmd *cobra.Command, args []string) {
formatCmdCfg.Source = strings.ToLower(formatCmdCfg.Source)
switch formatCmdCfg.Source {
case "dodi":
items, err := db.GetDODIGameDownloads()
items, err := db.GetDODIGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
@ -43,14 +43,14 @@ func formatRun(cmd *cobra.Command, args []string) {
item.Name = crawler.DODIFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameDownload(item)
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "kaoskrew":
items, err := db.GetKaOsKrewGameDownloads()
items, err := db.GetKaOsKrewGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
@ -60,14 +60,14 @@ func formatRun(cmd *cobra.Command, args []string) {
item.Name = crawler.KaOsKrewFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameDownload(item)
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "freegog":
items, err := db.GetFreeGOGGameDownloads()
items, err := db.GetFreeGOGGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
@ -77,14 +77,14 @@ func formatRun(cmd *cobra.Command, args []string) {
item.Name = crawler.FreeGOGFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameDownload(item)
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "xatab":
items, err := db.GetXatabGameDownloads()
items, err := db.GetXatabGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
@ -94,14 +94,14 @@ func formatRun(cmd *cobra.Command, args []string) {
item.Name = crawler.XatabFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameDownload(item)
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "onlinefix":
items, err := db.GetOnlineFixGameDownloads()
items, err := db.GetOnlineFixGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
@ -111,14 +111,14 @@ func formatRun(cmd *cobra.Command, args []string) {
item.Name = crawler.OnlineFixFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameDownload(item)
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "armgddn":
items, err := db.GetARMGDDNGameDownloads()
items, err := db.GetARMGDDNGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
@ -128,7 +128,7 @@ func formatRun(cmd *cobra.Command, args []string) {
item.Name = crawler.ARMGDDNFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameDownload(item)
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}

2
cmd/list.go
View File

@ -28,7 +28,7 @@ func init() {
func listRun(cmd *cobra.Command, args []string) {
if listCmdCfg.Unid {
games, err := db.GetUnorganizedGameDownloads(-1)
games, err := db.GetUnorganizedGameItems(-1)
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
}

4
cmd/organize.go
View File

@ -28,12 +28,12 @@ func init() {
}
func organizeRun(cmd *cobra.Command, args []string) {
games, err := db.GetUnorganizedGameDownloads(organizeCmdCfg.Num)
games, err := db.GetUnorganizedGameItems(organizeCmdCfg.Num)
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
}
for _, game := range games {
gameInfo, err := crawler.OrganizeGameDownload(game)
gameInfo, err := crawler.OrganizeGameItem(game)
if err == nil {
err = db.SaveGameInfo(gameInfo)
if err != nil {

2
cmd/organize_manually.go
View File

@ -58,7 +58,7 @@ func addRun(cmd *cobra.Command, args []string) {
log.Logger.Error("Failed to parse game id", zap.Error(err))
continue
}
info, err := crawler.OrganizeGameDownloadManually(objID, v.Platform, v.PlatformID)
info, err := crawler.OrganizeGameItemManually(objID, v.Platform, v.PlatformID)
if err != nil {
log.Logger.Error("Failed to add game info", zap.Error(err))
continue

25
cmd/supplement.go Normal file
View File

@ -0,0 +1,25 @@
package cmd
import (
"github.com/nitezs/pcgamedb/crawler"
"github.com/nitezs/pcgamedb/log"
"go.uber.org/zap"
"github.com/spf13/cobra"
)
var supplementCmd = &cobra.Command{
Use: "supplement",
Long: "Supplement platform id to game info",
Short: "Supplement platform id to game info",
Run: func(cmd *cobra.Command, args []string) {
err := crawler.SupplementPlatformIDToGameInfo(log.Logger)
if err != nil {
log.Logger.Error("Error supplementing platform id to game info", zap.Error(err))
}
},
}
func init() {
RootCmd.AddCommand(supplementCmd)
}

2
cmd/task.go
View File

@ -22,7 +22,7 @@ var taskCmd = &cobra.Command{
if taskCmdCfg.Crawl {
task.Crawl(log.Logger)
c := cron.New()
_, err := c.AddFunc("0 0 * * *", func() { task.Crawl(log.Logger) })
_, err := c.AddFunc("0 */3 * * *", func() { task.Crawl(log.Logger) })
if err != nil {
log.Logger.Error("Failed to add task", zap.Error(err))
}

20
crawler/1337x.go
View File

@ -33,7 +33,7 @@ func New1337xCrawler(source string, formatter Formatter, logger *zap.Logger) *s1
}
}
func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *s1337xCrawler) Crawl(page int) ([]*model.GameItem, error) {
var resp *utils.FetchResponse
var doc *goquery.Document
var err error
@ -57,7 +57,7 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
urls = append(urls, url)
}
})
var res []*model.GameDownload
var res []*model.GameItem
for _, u := range urls {
u = fmt.Sprintf("%s%s", constant.C1337xBaseURL, u)
if db.IsGameCrawledByURL(u) {
@ -69,13 +69,13 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
continue
}
err = db.SaveGameDownload(item)
err = db.SaveGameItem(item)
if err != nil {
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
continue
}
res = append(res, item)
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -89,14 +89,14 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
if err != nil {
return nil, err
}
var item = &model.GameDownload{}
var item = &model.GameItem{}
item.Url = url
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
if err != nil {
@ -119,8 +119,8 @@ func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *s1337xCrawler) CrawlMulti(pages []int) (res []*model.GameDownload, err error) {
var items []*model.GameDownload
func (c *s1337xCrawler) CrawlMulti(pages []int) (res []*model.GameItem, err error) {
var items []*model.GameItem
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err
@ -138,12 +138,12 @@ func (c *s1337xCrawler) CrawlMulti(pages []int) (res []*model.GameDownload, err
return res, nil
}
func (c *s1337xCrawler) CrawlAll() (res []*model.GameDownload, err error) {
func (c *s1337xCrawler) CrawlAll() (res []*model.GameItem, err error) {
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err
}
var items []*model.GameDownload
var items []*model.GameItem
for i := 1; i <= totalPageNum; i++ {
items, err = c.Crawl(i)
res = append(res, items...)

22
crawler/armgddn.go
View File

@ -75,9 +75,9 @@ func (c *ARMGDDNCrawler) fetchAndParseFTPData(filePath string) ([]GameData, erro
return data, nil
}
func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameDownload, error) {
func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameItem, error) {
count := 0
var res []*model.GameDownload
var res []*model.GameItem
modTimeMap := make(map[string]time.Time)
entries, err := c.conn.List(fmt.Sprintf("/%s", platform))
if err != nil {
@ -116,7 +116,7 @@ func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) (
size += fileSize
}
}
item, err := db.GetGameDownloadByUrl(u)
item, err := db.GetGameItemByUrl(u)
if err != nil {
continue
}
@ -127,7 +127,7 @@ func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) (
item.RawName = v.FolderName
item.Author = "ARMGDDN"
item.Download = fmt.Sprintf("ftpes://%s:%s@%s/%s/%s", ftpUsername, ftpPassword, ftpAddress, platform, url.QueryEscape(v.FolderName))
if err := db.SaveGameDownload(item); err != nil {
if err := db.SaveGameItem(item); err != nil {
continue
}
res = append(res, item)
@ -140,12 +140,12 @@ func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) (
c.logger.Warn("strconv error", zap.Error(err))
continue
}
info, err = OrganizeGameDownloadWithSteam(id, item)
info, err = OrganizeGameItemWithSteam(id, item)
if err != nil {
continue
}
} else {
info, err = OrganizeGameDownload(item)
info, err = OrganizeGameItem(item)
if err != nil {
continue
}
@ -168,15 +168,15 @@ func ARMGDDNFormatter(name string) string {
return strings.TrimSpace(cleanedName[:matchIndex[0]])
}
func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameDownload, error) {
func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameItem, error) {
return c.crawlPlatform("/PC/currentserverPC-FTP.json", "PC", num)
}
func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameDownload, error) {
func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameItem, error) {
return c.crawlPlatform("/PCVR/currentserverPCVR-FTP.json", "PCVR", num)
}
func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameDownload, error) {
func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameItem, error) {
num1 := num / 2
num2 := num - num1
if num == -1 {
@ -194,11 +194,11 @@ func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameDownload, error) {
return append(res1, res2...), nil
}
func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameItem, error) {
return c.Crawl(-1)
}
func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameDownload, error) {
func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameItem, error) {
err := c.connectFTP()
if err != nil {
return nil, err

20
crawler/chovka.go
View File

@ -30,7 +30,7 @@ func (c *ChovkaCrawler) Name() string {
return "ChovkaCrawler"
}
func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
@ -41,7 +41,7 @@ func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
if err != nil {
return nil, err
}
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -70,7 +70,7 @@ func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: fmt.Sprintf(constant.RepackInfoURL, page),
})
@ -91,7 +91,7 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
urls = append(urls, u)
updateFlags = append(updateFlags, s.Find(".entry__title.h2 a").Text())
})
var res []*model.GameDownload
var res []*model.GameItem
for i, u := range urls {
if db.IsChovkaCrawled(updateFlags[i]) {
continue
@ -102,12 +102,12 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
continue
}
if err := db.SaveGameDownload(item); err != nil {
if err := db.SaveGameItem(item); err != nil {
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
continue
}
res = append(res, item)
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -120,8 +120,8 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
var res []*model.GameDownload
func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
var res []*model.GameItem
for _, page := range pages {
items, err := c.Crawl(page)
if err != nil {
@ -132,12 +132,12 @@ func (c *ChovkaCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *ChovkaCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *ChovkaCrawler) CrawlAll() ([]*model.GameItem, error) {
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err
}
var res []*model.GameDownload
var res []*model.GameItem
for i := 1; i <= totalPageNum; i++ {
items, err := c.Crawl(i)
if err != nil {

6
crawler/crawler.go
View File

@ -8,8 +8,8 @@ import (
type Crawler interface {
Name() string
Crawl(int) ([]*model.GameDownload, error)
CrawlAll() ([]*model.GameDownload, error)
Crawl(int) ([]*model.GameItem, error)
CrawlAll() ([]*model.GameItem, error)
}
type SimpleCrawler interface {
@ -18,7 +18,7 @@ type SimpleCrawler interface {
type PagedCrawler interface {
Crawler
CrawlMulti([]int) ([]*model.GameDownload, error)
CrawlMulti([]int) ([]*model.GameItem, error)
GetTotalPageNum() (int, error)
}

8
crawler/dodi.go
View File

@ -32,19 +32,19 @@ func (c *DODICrawler) Name() string {
return "DODICrawler"
}
func (c *DODICrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *DODICrawler) Crawl(page int) ([]*model.GameItem, error) {
return c.crawler.Crawl(page)
}
func (c *DODICrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *DODICrawler) CrawlByUrl(url string) (*model.GameItem, error) {
return c.crawler.CrawlByUrl(url)
}
func (c *DODICrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
func (c *DODICrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
return c.crawler.CrawlMulti(pages)
}
func (c *DODICrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *DODICrawler) CrawlAll() ([]*model.GameItem, error) {
return c.crawler.CrawlAll()
}

20
crawler/fitgirl.go
View File

@ -31,7 +31,7 @@ func (c *FitGirlCrawler) Name() string {
return "FitGirlCrawler"
}
func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
@ -61,7 +61,7 @@ func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return nil, errors.New("Failed to find magnet")
}
magnet := magnetRegexRes[0]
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -74,7 +74,7 @@ func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: fmt.Sprintf(constant.FitGirlURL, page),
})
@ -97,7 +97,7 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
updateFlags = append(updateFlags, fmt.Sprintf("%s%s", u, d))
}
})
var res []*model.GameDownload
var res []*model.GameItem
for i, u := range urls {
if db.IsFitgirlCrawled(updateFlags[i]) {
continue
@ -109,13 +109,13 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
continue
}
item.UpdateFlag = updateFlags[i]
err = db.SaveGameDownload(item)
err = db.SaveGameItem(item)
if err != nil {
c.logger.Warn("Failed to save", zap.Error(err))
continue
}
res = append(res, item)
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -129,8 +129,8 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *FitGirlCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
var res []*model.GameDownload
func (c *FitGirlCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
var res []*model.GameItem
for _, page := range pages {
items, err := c.Crawl(page)
if err != nil {
@ -141,8 +141,8 @@ func (c *FitGirlCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error)
return res, nil
}
func (c *FitGirlCrawler) CrawlAll() ([]*model.GameDownload, error) {
var res []*model.GameDownload
func (c *FitGirlCrawler) CrawlAll() ([]*model.GameItem, error) {
var res []*model.GameItem
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err

14
crawler/freegog.go
View File

@ -27,7 +27,7 @@ func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
}
}
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
count := 0
resp, err := utils.Fetch(utils.FetchConfig{
Url: constant.FreeGOGListURL,
@ -49,7 +49,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
updateFlags = append(updateFlags, s.Text()+s.AttrOr("href", ""))
})
res := []*model.GameDownload{}
res := []*model.GameItem{}
for i, u := range urls {
if count == num {
break
@ -64,14 +64,14 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
continue
}
item.UpdateFlag = updateFlags[i]
err = db.SaveGameDownload(item)
err = db.SaveGameItem(item)
if err != nil {
c.logger.Warn("Failed to save", zap.Error(err))
continue
}
res = append(res, item)
count++
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -85,14 +85,14 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
if err != nil {
return nil, err
}
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -127,7 +127,7 @@ func (c *FreeGOGCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *FreeGOGCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *FreeGOGCrawler) CrawlAll() ([]*model.GameItem, error) {
return c.Crawl(-1)
}

44
crawler/game.go
View File

@ -9,6 +9,7 @@ import (
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/nitezs/pcgamedb/utils"
"go.uber.org/zap"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
@ -25,10 +26,11 @@ func GenerateGameInfo(platform string, id int) (*model.GameInfo, error) {
}
}
func OrganizeGameDownload(game *model.GameDownload) (*model.GameInfo, error) {
item, err := OrganizeGameDownloadWithIGDB(0, game)
func OrganizeGameItem(game *model.GameItem) (*model.GameInfo, error) {
item, err := OrganizeGameItemWithIGDB(0, game)
if err == nil {
if item.SteamID == 0 {
// get steam id from igdb
steamID, err := GetSteamIDByIGDBIDCache(item.IGDBID)
if err == nil {
item.SteamID = steamID
@ -36,7 +38,7 @@ func OrganizeGameDownload(game *model.GameDownload) (*model.GameInfo, error) {
return item, nil
}
}
item, err = OrganizeGameDownloadWithSteam(0, game)
item, err = OrganizeGameItemWithSteam(0, game)
if err == nil {
if item.IGDBID == 0 {
igdbID, err := GetIGDBIDBySteamIDCache(item.SteamID)
@ -59,7 +61,7 @@ func AddGameInfoManually(gameID primitive.ObjectID, platform string, plateformID
return info, db.SaveGameInfo(info)
}
func OrganizeGameDownloadManually(gameID primitive.ObjectID, platform string, platformID int) (*model.GameInfo, error) {
func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platformID int) (*model.GameInfo, error) {
info, err := db.GetGameInfoByPlatformID(platform, platformID)
if err != nil {
if err == mongo.ErrNoDocuments {
@ -105,38 +107,7 @@ func FormatName(name string) string {
return name
}
func TransformSteamIDToIGDBID() {
gameInfos, err := db.GetGameInfoWithSteamID()
if err != nil {
return
}
for _, info := range gameInfos {
id, err := GetIGDBIDBySteamIDCache(info.SteamID)
if err != nil {
continue
}
existedInfo, err := db.GetGameInfoByPlatformID("igdb", id)
if err == nil {
existedInfo.GameIDs = append(existedInfo.GameIDs, info.GameIDs...)
existedInfo.GameIDs = utils.Unique(existedInfo.GameIDs)
_ = db.SaveGameInfo(existedInfo)
_ = db.DeleteGameInfoByID(info.ID)
} else {
if err == mongo.ErrNoDocuments {
newInfo, err := GenerateIGDBGameInfo(id)
if err != nil {
continue
}
newInfo.ID = info.ID
newInfo.CreatedAt = info.CreatedAt
newInfo.GameIDs = info.GameIDs
_ = db.SaveGameInfo(newInfo)
}
}
}
}
func SupplementGameInfoPlatformID() error {
func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
infos, err := db.GetAllGameInfos()
if err != nil {
return err
@ -162,6 +133,7 @@ func SupplementGameInfoPlatformID() error {
changed = true
}
if changed {
logger.Info("Supplemented platform id for game info", zap.String("name", info.Name), zap.Int("igdb", int(info.IGDBID)), zap.Int("steam", int(info.SteamID)))
_ = db.SaveGameInfo(info)
}
}

10
crawler/gnarly.go
View File

@ -24,8 +24,8 @@ func NewGnarlyCrawler(logger *zap.Logger) *GnarlyCrawler {
}
}
func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
var res []*model.GameDownload
func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameItem, error) {
var res []*model.GameItem
count := 0
resp, err := utils.Fetch(utils.FetchConfig{
Url: constant.GnarlyURL,
@ -55,7 +55,7 @@ func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
if db.IsGnarlyCrawled(lines[i-1]) {
continue
}
item, err := db.GetGameDownloadByUrl(lines[i])
item, err := db.GetGameItemByUrl(lines[i])
if err != nil {
continue
}
@ -76,7 +76,7 @@ func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
item.UpdateFlag = item.RawName
res = append(res, item)
count++
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
continue
}
@ -93,7 +93,7 @@ func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *GnarlyCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *GnarlyCrawler) CrawlAll() ([]*model.GameItem, error) {
return c.Crawl(-1)
}

20
crawler/goggames.go
View File

@ -31,7 +31,7 @@ func (c *GOGGamesCrawler) Name() string {
return "GOGGamesCrawler"
}
func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
@ -56,7 +56,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
if err != nil {
return nil, err
}
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -69,7 +69,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: fmt.Sprintf(constant.GOGGamesURL, page),
})
@ -88,7 +88,7 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
}
urls = append(urls, fmt.Sprintf("%s%s", constant.GOGGamesBaseURL, u))
})
res := make([]*model.GameDownload, 0)
res := make([]*model.GameItem, 0)
for _, u := range urls {
c.logger.Info("Crawling", zap.String("URL", u))
item, err := c.CrawlByUrl(u)
@ -96,12 +96,12 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
continue
}
if err := db.SaveGameDownload(item); err != nil {
if err := db.SaveGameItem(item); err != nil {
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
continue
}
res = append(res, item)
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -114,8 +114,8 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
res := make([]*model.GameDownload, 0)
func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
res := make([]*model.GameItem, 0)
for _, page := range pages {
items, err := c.Crawl(page)
if err != nil {
@ -126,12 +126,12 @@ func (c *GOGGamesCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error)
return res, nil
}
func (c *GOGGamesCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *GOGGamesCrawler) CrawlAll() ([]*model.GameItem, error) {
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err
}
var res []*model.GameDownload
var res []*model.GameItem
for i := 1; i <= totalPageNum; i++ {
items, err := c.Crawl(i)
if err != nil {

11
crawler/igdb.go
View File

@ -311,7 +311,8 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
return item, nil
}
func OrganizeGameDownloadWithIGDB(id int, game *model.GameDownload) (*model.GameInfo, error) {
// id=0, means search id by name
func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
var err error
if id == 0 {
id, err = GetIGDBIDCache(game.Name)
@ -390,7 +391,7 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
}
}
func GetIGDBIDBySteamIDs(ids []int) (map[int]int, error) {
func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
var err error
if TwitchToken == "" {
TwitchToken, err = LoginTwitch()
@ -446,7 +447,7 @@ func GetIGDBIDBySteamIDs(ids []int) (map[int]int, error) {
return ret, nil
}
func GetIGDBIDBySteamIDsCache(ids []int) (map[int]int, error) {
func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
res := make(map[int]int)
notExistIDs := make([]int, 0)
if config.Config.RedisAvaliable {
@ -463,7 +464,7 @@ func GetIGDBIDBySteamIDsCache(ids []int) (map[int]int, error) {
if len(res) == len(ids) {
return res, nil
}
idMap, err := GetIGDBIDBySteamIDs(notExistIDs)
idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
if err != nil {
return nil, err
}
@ -475,6 +476,6 @@ func GetIGDBIDBySteamIDsCache(ids []int) (map[int]int, error) {
}
return res, nil
} else {
return GetIGDBIDBySteamIDs(ids)
return GetIGDBIDsBySteamIDs(ids)
}
}

8
crawler/kaoskrew.go
View File

@ -31,19 +31,19 @@ func (c *KaOsKrewCrawler) Name() string {
return "KaOsKrewCrawler"
}
func (c *KaOsKrewCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *KaOsKrewCrawler) Crawl(page int) ([]*model.GameItem, error) {
return c.crawler.Crawl(page)
}
func (c *KaOsKrewCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *KaOsKrewCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
return c.crawler.CrawlByUrl(url)
}
func (c *KaOsKrewCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
func (c *KaOsKrewCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
return c.crawler.CrawlMulti(pages)
}
func (c *KaOsKrewCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *KaOsKrewCrawler) CrawlAll() ([]*model.GameItem, error) {
return c.crawler.CrawlAll()
}

20
crawler/onlinefix.go
View File

@ -37,7 +37,7 @@ func (c *OnlineFixCrawler) Name() string {
return "OnlineFixCrawler"
}
func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
if !config.Config.OnlineFixAvaliable {
c.logger.Error("Need Online Fix account")
return nil, errors.New("Online Fix is not available")
@ -77,7 +77,7 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
)
})
var res []*model.GameDownload
var res []*model.GameItem
for i, u := range urls {
if db.IsOnlineFixCrawled(updateFlags[i]) {
continue
@ -89,13 +89,13 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
continue
}
item.UpdateFlag = updateFlags[i]
err = db.SaveGameDownload(item)
err = db.SaveGameItem(item)
if err != nil {
c.logger.Warn("Failed to save", zap.Error(err))
continue
}
res = append(res, item)
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -109,7 +109,7 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
if len(c.cookies) == 0 {
err := c.login()
if err != nil {
@ -137,7 +137,7 @@ func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
if len(downloadRegexRes) == 0 {
return nil, errors.New("Failed to find download button")
}
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -215,8 +215,8 @@ func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
var res []*model.GameDownload
func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
var res []*model.GameItem
for _, page := range pages {
items, err := c.Crawl(page)
if err != nil {
@ -227,8 +227,8 @@ func (c *OnlineFixCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error
return res, nil
}
func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameDownload, error) {
var res []*model.GameDownload
func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameItem, error) {
var res []*model.GameItem
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err

2
crawler/steam.go
View File

@ -176,7 +176,7 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
return item, nil
}
func OrganizeGameDownloadWithSteam(id int, game *model.GameDownload) (*model.GameInfo, error) {
func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, error) {
var err error
if id == 0 {
id, err = GetSteamIDCache(game.Name)

22
crawler/steam250.go
View File

@ -45,29 +45,13 @@ func GetSteam250(url string) ([]*model.GameInfo, error) {
})
var res []*model.GameInfo
count := 0
idMap, err := GetIGDBIDBySteamIDsCache(steamIDs)
if err != nil {
return nil, err
}
for _, item := range rank {
if count == 10 {
break
}
if idMap[item.SteamID] != 0 {
info, err := db.GetGameInfoByPlatformID("igdb", idMap[item.SteamID])
for _, steamID := range steamIDs {
info, err := db.GetGameInfoByPlatformID("steam", steamID)
if err == nil {
res = append(res, info)
count++
continue
}
} else {
info, err := db.GetGameInfoByPlatformID("steam", item.SteamID)
if err == nil {
res = append(res, info)
count++
continue
}
}
}
return res, nil
}
@ -124,7 +108,7 @@ func GetSteam250Cache(k string, f func() ([]*model.GameInfo, error)) ([]*model.G
if err != nil {
return data, nil
}
err = cache.AddWithExpire(key, dataBytes, 24*time.Hour)
err = cache.AddWithExpire(key, dataBytes, 12*time.Hour)
if err != nil {
return data, nil
}

14
crawler/steamrip.go
View File

@ -30,7 +30,7 @@ func (c *SteamRIPCrawler) Name() string {
return "SteamRIPCrawler"
}
func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
@ -41,7 +41,7 @@ func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
if err != nil {
return nil, err
}
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -82,7 +82,7 @@ func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
count := 0
resp, err := utils.Fetch(utils.FetchConfig{
Url: constant.SteamRIPGameListURL,
@ -94,7 +94,7 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
if err != nil {
return nil, err
}
var items []*model.GameDownload
var items []*model.GameItem
urls := []string{}
updateFlags := []string{} // title
doc.Find(".az-list-item>a").Each(func(i int, s *goquery.Selection) {
@ -119,13 +119,13 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
continue
}
item.UpdateFlag = updateFlags[i]
if err := db.SaveGameDownload(item); err != nil {
if err := db.SaveGameItem(item); err != nil {
c.logger.Error("Failed to save item", zap.Error(err))
continue
}
items = append(items, item)
count++
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -139,7 +139,7 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameDownload, error) {
return items, nil
}
func (c *SteamRIPCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *SteamRIPCrawler) CrawlAll() ([]*model.GameItem, error) {
return c.Crawl(-1)
}

20
crawler/xatab.go
View File

@ -31,7 +31,7 @@ func (c *XatabCrawler) Name() string {
return "XatabCrawler"
}
func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
func (c *XatabCrawler) Crawl(page int) ([]*model.GameItem, error) {
requestURL := fmt.Sprintf("%s/page/%v", constant.XatabBaseURL, page)
resp, err := utils.Fetch(utils.FetchConfig{
Url: requestURL,
@ -55,7 +55,7 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
urls = append(urls, u)
updateFlags = append(updateFlags, s.Find(".entry__title.h2 a").Text())
})
var res []*model.GameDownload
var res []*model.GameItem
for i, u := range urls {
if db.IsXatabCrawled(updateFlags[i]) {
continue
@ -66,13 +66,13 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
continue
}
err = db.SaveGameDownload(item)
err = db.SaveGameItem(item)
if err != nil {
c.logger.Warn("Failed to save", zap.Error(err))
continue
}
res = append(res, item)
info, err := OrganizeGameDownload(item)
info, err := OrganizeGameItem(item)
if err != nil {
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
continue
@ -86,7 +86,7 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
resp, err := utils.Fetch(utils.FetchConfig{
Url: url,
})
@ -97,7 +97,7 @@ func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
if err != nil {
return nil, err
}
item, err := db.GetGameDownloadByUrl(url)
item, err := db.GetGameItemByUrl(url)
if err != nil {
return nil, err
}
@ -126,12 +126,12 @@ func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameDownload, error) {
return item, nil
}
func (c *XatabCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
func (c *XatabCrawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err
}
var res []*model.GameDownload
var res []*model.GameItem
for _, page := range pages {
if page > totalPageNum {
continue
@ -145,12 +145,12 @@ func (c *XatabCrawler) CrawlMulti(pages []int) ([]*model.GameDownload, error) {
return res, nil
}
func (c *XatabCrawler) CrawlAll() ([]*model.GameDownload, error) {
func (c *XatabCrawler) CrawlAll() ([]*model.GameItem, error) {
totalPageNum, err := c.GetTotalPageNum()
if err != nil {
return nil, err
}
var res []*model.GameDownload
var res []*model.GameItem
for i := 1; i <= totalPageNum; i++ {
items, err := c.Crawl(i)
if err != nil {

8
db/1337x.go
View File

@ -4,10 +4,10 @@ import (
"github.com/nitezs/pcgamedb/model"
)
func GetDODIGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("dodi")
func GetDODIGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("dodi")
}
func GetKaOsKrewGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("kaoskrew")
func GetKaOsKrewGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("kaoskrew")
}

4
db/armgddn.go
View File

@ -6,6 +6,6 @@ func IsARMGDDNCrawled(flag string) bool {
return IsGameCrawled(flag, "armgddn")
}
func GetARMGDDNGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("armgddn")
func GetARMGDDNGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("armgddn")
}

4
db/db.go
View File

@ -16,14 +16,14 @@ import (
)
const (
gameDownloadCollectionName = "game_downloads"
gameDownloadCollectionName = "games"
gameInfoCollectionName = "game_infos"
)
var (
mongoDB *mongo.Client
mutx = &sync.RWMutex{}
GameDownloadCollection = &CustomCollection{
GameItemCollection = &CustomCollection{
collName: gameDownloadCollectionName,
}
GameInfoCollection = &CustomCollection{

42
db/export.go Normal file
View File

@ -0,0 +1,42 @@
package db
import (
"context"
"encoding/json"
"time"
"github.com/nitezs/pcgamedb/model"
"go.mongodb.org/mongo-driver/bson"
)
func Export() ([]byte, []byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
infos := []model.GameInfo{}
games := []model.GameItem{}
cursor, err := GameInfoCollection.Find(ctx, bson.M{})
if err != nil {
return nil, nil, err
}
defer cursor.Close(ctx)
if err = cursor.All(ctx, &infos); err != nil {
return nil, nil, err
}
cursor, err = GameItemCollection.Find(ctx, bson.M{})
if err != nil {
return nil, nil, err
}
defer cursor.Close(ctx)
if err = cursor.All(ctx, &games); err != nil {
return nil, nil, err
}
infoJson, err := json.Marshal(infos)
if err != nil {
return nil, nil, err
}
gameJson, err := json.Marshal(games)
if err != nil {
return nil, nil, err
}
return infoJson, gameJson, nil
}

4
db/fitgirl.go
View File

@ -2,8 +2,8 @@ package db
import "github.com/nitezs/pcgamedb/model"
func GetFitgirlAllGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("fitgirl")
func GetFitgirlAllGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("fitgirl")
}
func IsFitgirlCrawled(flag string) bool {

4
db/freegog.go
View File

@ -4,8 +4,8 @@ import (
"github.com/nitezs/pcgamedb/model"
)
func GetFreeGOGGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("freegog")
func GetFreeGOGGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("freegog")
}
func IsFreeGOGCrawled(flag string) bool {
return IsGameCrawled(flag, "freegog")

88
db/game.go
View File

@ -25,12 +25,12 @@ var (
removeRepeatingSpacesRegex = regexp.MustCompile(`\s+`)
)
func GetGameDownloadsByAuthor(regex string) ([]*model.GameDownload, error) {
var res []*model.GameDownload
func GetGameItemsByAuthor(regex string) ([]*model.GameItem, error) {
var res []*model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
filter := bson.D{{Key: "author", Value: primitive.Regex{Pattern: regex, Options: "i"}}}
cursor, err := GameDownloadCollection.Find(ctx, filter)
cursor, err := GameItemCollection.Find(ctx, filter)
if err != nil {
return nil, err
}
@ -44,20 +44,20 @@ func GetGameDownloadsByAuthor(regex string) ([]*model.GameDownload, error) {
return res, err
}
func GetGameDownloadsByAuthorPagination(regex string, page int, pageSize int) ([]*model.GameDownload, int, error) {
var res []*model.GameDownload
func GetGameItemsByAuthorPagination(regex string, page int, pageSize int) ([]*model.GameItem, int, error) {
var res []*model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
filter := bson.D{{Key: "author", Value: primitive.Regex{Pattern: regex, Options: "i"}}}
opts := options.Find()
opts.SetSkip(int64((page - 1) * pageSize))
opts.SetLimit(int64(pageSize))
totalCount, err := GameDownloadCollection.CountDocuments(ctx, filter)
totalCount, err := GameItemCollection.CountDocuments(ctx, filter)
if err != nil {
return nil, 0, err
}
totalPage := (totalCount + int64(pageSize) - 1) / int64(pageSize)
cursor, err := GameDownloadCollection.Find(ctx, filter, opts)
cursor, err := GameItemCollection.Find(ctx, filter, opts)
if err != nil {
return nil, 0, err
}
@ -78,8 +78,8 @@ func IsGameCrawled(flag string, author string) bool {
{Key: "author", Value: primitive.Regex{Pattern: author, Options: "i"}},
{Key: "update_flag", Value: flag},
}
var game model.GameDownload
err := GameDownloadCollection.FindOne(ctx, filter).Decode(&game)
var game model.GameItem
err := GameItemCollection.FindOne(ctx, filter).Decode(&game)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return false
@ -95,8 +95,8 @@ func IsGameCrawledByURL(url string) bool {
filter := bson.D{
{Key: "url", Value: url},
}
var game model.GameDownload
err := GameDownloadCollection.FindOne(ctx, filter).Decode(&game)
var game model.GameItem
err := GameItemCollection.FindOne(ctx, filter).Decode(&game)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return false
@ -106,7 +106,7 @@ func IsGameCrawledByURL(url string) bool {
return true
}
func SaveGameDownload(item *model.GameDownload) error {
func SaveGameItem(item *model.GameItem) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if item.ID.IsZero() {
@ -121,7 +121,7 @@ func SaveGameDownload(item *model.GameDownload) error {
filter := bson.M{"_id": item.ID}
update := bson.M{"$set": item}
opts := options.Update().SetUpsert(true)
_, err := GameDownloadCollection.UpdateOne(ctx, filter, update, opts)
_, err := GameItemCollection.UpdateOne(ctx, filter, update, opts)
if err != nil {
return err
}
@ -148,17 +148,17 @@ func SaveGameInfo(item *model.GameInfo) error {
return nil
}
func GetAllGameDownloads() ([]*model.GameDownload, error) {
var items []*model.GameDownload
func GetAllGameItems() ([]*model.GameItem, error) {
var items []*model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cursor, err := GameDownloadCollection.Find(ctx, bson.D{})
cursor, err := GameItemCollection.Find(ctx, bson.D{})
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var game model.GameDownload
var game model.GameItem
if err = cursor.Decode(&game); err != nil {
return nil, err
}
@ -170,44 +170,44 @@ func GetAllGameDownloads() ([]*model.GameDownload, error) {
return items, err
}
func GetGameDownloadByUrl(url string) (*model.GameDownload, error) {
var item model.GameDownload
func GetGameItemByUrl(url string) (*model.GameItem, error) {
var item model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
filter := bson.M{"url": url}
err := GameDownloadCollection.FindOne(ctx, filter).Decode(&item)
err := GameItemCollection.FindOne(ctx, filter).Decode(&item)
if err != nil {
if errors.Is(err, mongo.ErrNoDocuments) {
return &model.GameDownload{}, nil
return &model.GameItem{}, nil
}
return nil, err
}
return &item, nil
}
func GetGameDownloadByID(id primitive.ObjectID) (*model.GameDownload, error) {
var item model.GameDownload
func GetGameItemByID(id primitive.ObjectID) (*model.GameItem, error) {
var item model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
filter := bson.M{"_id": id}
err := GameDownloadCollection.FindOne(ctx, filter).Decode(&item)
err := GameItemCollection.FindOne(ctx, filter).Decode(&item)
if err != nil {
return nil, err
}
return &item, nil
}
func GetGameDownloadsByIDs(ids []primitive.ObjectID) ([]*model.GameDownload, error) {
var items []*model.GameDownload
func GetGameItemsByIDs(ids []primitive.ObjectID) ([]*model.GameItem, error) {
var items []*model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
cursor, err := GameDownloadCollection.Find(ctx, bson.M{"_id": bson.M{"$in": ids}})
cursor, err := GameItemCollection.Find(ctx, bson.M{"_id": bson.M{"$in": ids}})
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var game model.GameDownload
var game model.GameItem
if err = cursor.Decode(&game); err != nil {
return nil, err
}
@ -250,7 +250,7 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
if err = cursor.Decode(&game); err != nil {
return nil, 0, err
}
game.Games, err = GetGameDownloadsByIDs(game.GameIDs)
game.Games, err = GetGameItemsByIDs(game.GameIDs)
if err != nil {
return nil, 0, err
}
@ -287,7 +287,7 @@ func SearchGameInfosCache(name string, page int, pageSize int) ([]*model.GameInf
if err != nil {
return nil, 0, err
}
_ = cache.AddWithExpire(key, string(dataBytes), 12*time.Hour)
_ = cache.AddWithExpire(key, string(dataBytes), 5*time.Minute)
return data, totalPage, nil
}
} else {
@ -315,10 +315,10 @@ func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {
return &game, nil
}
func GetUnorganizedGameDownloads(num int) ([]*model.GameDownload, error) {
func GetUnorganizedGameItems(num int) ([]*model.GameItem, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
var gamesNotInDetails []*model.GameDownload
var gamesNotInDetails []*model.GameItem
pipeline := mongo.Pipeline{
bson.D{{Key: "$lookup", Value: bson.D{
{Key: "from", Value: "game_infos"},
@ -337,14 +337,14 @@ func GetUnorganizedGameDownloads(num int) ([]*model.GameDownload, error) {
bson.D{{Key: "$sort", Value: bson.D{{Key: "name", Value: 1}}}},
)
cursor, err := GameDownloadCollection.Aggregate(ctx, pipeline)
cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var game model.GameDownload
var game model.GameItem
if err := cursor.Decode(&game); err != nil {
return nil, err
}
@ -392,7 +392,7 @@ func DeduplicateGames() ([]primitive.ObjectID, error) {
{Key: "total", Value: bson.D{{Key: "$gt", Value: 1}}},
}}},
}
cursor, err := GameDownloadCollection.Aggregate(ctx, pipeline)
cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
if err != nil {
return nil, err
}
@ -402,7 +402,7 @@ func DeduplicateGames() ([]primitive.ObjectID, error) {
for _, item := range qres {
idsToDelete := item.IDs[:len(item.IDs)-1]
res = append(res, idsToDelete...)
_, err = GameDownloadCollection.DeleteMany(ctx, bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToDelete}}}})
_, err = GameItemCollection.DeleteMany(ctx, bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToDelete}}}})
if err != nil {
return nil, err
}
@ -522,17 +522,17 @@ func GetGameInfosByName(name string) ([]*model.GameInfo, error) {
return games, nil
}
func GetGameDownloadByRawName(name string) ([]*model.GameDownload, error) {
func GetGameItemByRawName(name string) ([]*model.GameItem, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
name = strings.TrimSpace(name)
name = fmt.Sprintf("^%s$", name)
filter := bson.M{"raw_name": bson.M{"$regex": primitive.Regex{Pattern: name, Options: "i"}}}
cursor, err := GameDownloadCollection.Find(ctx, filter)
cursor, err := GameItemCollection.Find(ctx, filter)
if err != nil {
return nil, err
}
var game []*model.GameDownload
var game []*model.GameItem
if err = cursor.All(ctx, &game); err != nil {
return nil, err
}
@ -622,10 +622,10 @@ func GetGameInfoCount() (int64, error) {
return count, nil
}
func GetGameDownloadCount() (int64, error) {
func GetGameItemCount() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
count, err := GameDownloadCollection.CountDocuments(ctx, bson.M{})
count, err := GameItemCollection.CountDocuments(ctx, bson.M{})
if err != nil {
return 0, err
}
@ -661,10 +661,10 @@ func DeleteGameInfoByID(id primitive.ObjectID) error {
return nil
}
func DeleteGameDownloadByID(id primitive.ObjectID) error {
func DeleteGameItemByID(id primitive.ObjectID) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err := GameDownloadCollection.DeleteOne(ctx, bson.M{"_id": id})
_, err := GameItemCollection.DeleteOne(ctx, bson.M{"_id": id})
if err != nil {
return err
}
@ -702,7 +702,7 @@ func GetAllAuthors() ([]string, error) {
}}},
}
cursor, err := GameDownloadCollection.Aggregate(ctx, pipeline)
cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
if err != nil {
return nil, err
}

4
db/onlinefix.go
View File

@ -4,8 +4,8 @@ import (
"github.com/nitezs/pcgamedb/model"
)
func GetOnlineFixGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("onlinefix")
func GetOnlineFixGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("onlinefix")
}
func IsOnlineFixCrawled(flag string) bool {

4
db/xatab.go
View File

@ -4,8 +4,8 @@ import (
"github.com/nitezs/pcgamedb/model"
)
func GetXatabGameDownloads() ([]*model.GameDownload, error) {
return GetGameDownloadsByAuthor("xatab")
func GetXatabGameItems() ([]*model.GameItem, error) {
return GetGameItemsByAuthor("xatab")
}
func IsXatabCrawled(flag string) bool {

1
game_infos.json Normal file
View File

File diff suppressed because one or more lines are too long

1
games.json Normal file
View File

File diff suppressed because one or more lines are too long

4
model/game.go
View File

@ -20,12 +20,12 @@ type GameInfo struct {
Languages []string `json:"languages" bson:"languages"`
Screenshots []string `json:"screenshots" bson:"screenshots"`
GameIDs []primitive.ObjectID `json:"game_ids" bson:"games"`
Games []*GameDownload `json:"game_downloads" bson:"-"`
Games []*GameItem `json:"game_downloads" bson:"-"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
}
type GameDownload struct {
type GameItem struct {
ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"speculative_name" bson:"name"`
RawName string `json:"raw_name,omitempty" bson:"raw_name"`

30
server/handler/get_game_download_by_id.go → server/handler/get_game_by_id.go
View File

@ -11,31 +11,31 @@ import (
"go.mongodb.org/mongo-driver/mongo"
)
type GetGameDownloadByIDRequest struct {
type GetGameItemByIDRequest struct {
ID string `uri:"id" binding:"required"`
}
type GetGameDownloadByIDResponse struct {
type GetGameItemByIDResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
Game *model.GameDownload `json:"game,omitempty"`
Game *model.GameItem `json:"game,omitempty"`
}
// GetGameDownloadByID retrieves game download details by ID.
// GetGameItemByID retrieves game download details by ID.
// @Summary Retrieve game download by ID
// @Description Retrieves details of a game download by game ID
// @Tags game
// @Accept json
// @Produce json
// @Param id path string true "Game Download ID"
// @Success 200 {object} GetGameDownloadByIDResponse
// @Failure 400 {object} GetGameDownloadByIDResponse
// @Failure 500 {object} GetGameDownloadByIDResponse
// @Success 200 {object} GetGameItemByIDResponse
// @Failure 400 {object} GetGameItemByIDResponse
// @Failure 500 {object} GetGameItemByIDResponse
// @Router /game/raw/id/{id} [get]
func GetGameDownloadByIDHanlder(c *gin.Context) {
var req GetGameDownloadByIDRequest
func GetGameItemByIDHanlder(c *gin.Context) {
var req GetGameItemByIDRequest
if err := c.ShouldBindUri(&req); err != nil {
c.JSON(http.StatusBadRequest, GetGameDownloadByIDResponse{
c.JSON(http.StatusBadRequest, GetGameItemByIDResponse{
Status: "error",
Message: err.Error(),
})
@ -43,28 +43,28 @@ func GetGameDownloadByIDHanlder(c *gin.Context) {
}
id, err := primitive.ObjectIDFromHex(req.ID)
if err != nil {
c.JSON(http.StatusBadRequest, GetGameDownloadByIDResponse{
c.JSON(http.StatusBadRequest, GetGameItemByIDResponse{
Status: "error",
Message: err.Error(),
})
return
}
game, err := db.GetGameDownloadByID(id)
game, err := db.GetGameItemByID(id)
if err != nil {
if err == mongo.ErrNoDocuments {
c.JSON(http.StatusOK, GetGameDownloadByIDResponse{
c.JSON(http.StatusOK, GetGameItemByIDResponse{
Status: "ok",
Message: "No results found",
})
return
}
c.JSON(http.StatusInternalServerError, GetGameDownloadByIDResponse{
c.JSON(http.StatusInternalServerError, GetGameItemByIDResponse{
Status: "error",
Message: err.Error(),
})
return
}
c.JSON(http.StatusOK, GetGameDownloadByIDResponse{
c.JSON(http.StatusOK, GetGameItemByIDResponse{
Status: "ok",
Game: game,
})

69
server/handler/get_game_by_raw_name.go Normal file
View File

@ -0,0 +1,69 @@
package handler
import (
"net/http"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/mongo"
)
type GetGameItemByRawNameRequest struct {
Name string `uri:"name" binding:"required"`
}
type GetGameItemByRawNameResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
GameItem []*model.GameItem `json:"game_downloads,omitempty"`
}
// GetGameItemByRawName retrieves game download details by raw name.
// @Summary Retrieve game download by raw name
// @Description Retrieves details of a game download by its raw name
// @Tags game
// @Accept json
// @Produce json
// @Param name path string true "Game Download Raw Name"
// @Success 200 {object} GetGameItemByRawNameResponse
// @Failure 400 {object} GetGameItemByRawNameResponse
// @Failure 500 {object} GetGameItemByRawNameResponse
// @Router /game/raw/name/{name} [get]
func GetGameItemByRawNameHandler(c *gin.Context) {
var req GetGameItemByRawNameRequest
if err := c.ShouldBindUri(&req); err != nil {
c.JSON(http.StatusBadRequest, GetGameItemByRawNameResponse{
Status: "error",
Message: err.Error(),
})
return
}
gameDownload, err := db.GetGameItemByRawName(req.Name)
if err != nil {
if err == mongo.ErrNoDocuments {
c.JSON(http.StatusOK, GetGameItemByRawNameResponse{
Status: "ok",
Message: "No results found",
})
return
}
c.JSON(http.StatusInternalServerError, GetGameItemByRawNameResponse{
Status: "error",
Message: err.Error(),
})
return
}
if gameDownload == nil {
c.JSON(http.StatusOK, GetGameItemByRawNameResponse{
Status: "ok",
Message: "No results found",
})
return
}
c.JSON(http.StatusOK, GetGameItemByRawNameResponse{
Status: "ok",
GameItem: gameDownload,
})
}

69
server/handler/get_game_download_by_raw_name.go
View File

@ -1,69 +0,0 @@
package handler
import (
"net/http"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/mongo"
)
type GetGameDownloadByRawNameRequest struct {
Name string `uri:"name" binding:"required"`
}
type GetGameDownloadByRawNameResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
GameDownload []*model.GameDownload `json:"game_downloads,omitempty"`
}
// GetGameDownloadByRawName retrieves game download details by raw name.
// @Summary Retrieve game download by raw name
// @Description Retrieves details of a game download by its raw name
// @Tags game
// @Accept json
// @Produce json
// @Param name path string true "Game Download Raw Name"
// @Success 200 {object} GetGameDownloadByRawNameResponse
// @Failure 400 {object} GetGameDownloadByRawNameResponse
// @Failure 500 {object} GetGameDownloadByRawNameResponse
// @Router /game/raw/name/{name} [get]
func GetGameDownloadByRawNameHandler(c *gin.Context) {
var req GetGameDownloadByRawNameRequest
if err := c.ShouldBindUri(&req); err != nil {
c.JSON(http.StatusBadRequest, GetGameDownloadByRawNameResponse{
Status: "error",
Message: err.Error(),
})
return
}
gameDownload, err := db.GetGameDownloadByRawName(req.Name)
if err != nil {
if err == mongo.ErrNoDocuments {
c.JSON(http.StatusOK, GetGameDownloadByRawNameResponse{
Status: "ok",
Message: "No results found",
})
return
}
c.JSON(http.StatusInternalServerError, GetGameDownloadByRawNameResponse{
Status: "error",
Message: err.Error(),
})
return
}
if gameDownload == nil {
c.JSON(http.StatusOK, GetGameDownloadByRawNameResponse{
Status: "ok",
Message: "No results found",
})
return
}
c.JSON(http.StatusOK, GetGameDownloadByRawNameResponse{
Status: "ok",
GameDownload: gameDownload,
})
}

83
server/handler/get_game_downloads_by_author.go
View File

@ -1,83 +0,0 @@
package handler
import (
"net/http"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/gin-gonic/gin"
)
type GetGameDownloadsByAuthorRequest struct {
Author string `uri:"author" binding:"required"`
Page int `form:"page" json:"page"`
PageSize int `form:"page_size" json:"page_size"`
}
type GetGameDownloadsByAuthorResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
TotalPage int `json:"total_page"`
GameDownloads []*model.GameDownload `json:"game_downloads,omitempty"`
}
// GetGameDownloadsByAuthorHandler returns all game downloads by author
// @Summary Get game downloads by author
// @Description Get game downloads by author
// @Tags game
// @Accept json
// @Produce json
// @Param author path string true "Author"
// @Param page query int false "Page"
// @Param page_size query int false "Page Size"
// @Success 200 {object} GetGameDownloadsByAuthorResponse
// @Failure 400 {object} GetGameDownloadsByAuthorResponse
// @Failure 500 {object} GetGameDownloadsByAuthorResponse
// @Router /game/raw/author/{author} [get]
func GetGameDownloadsByAuthorHandler(ctx *gin.Context) {
var req GetGameDownloadsByAuthorRequest
if err := ctx.ShouldBindUri(&req); err != nil {
ctx.JSON(http.StatusBadRequest, GetGameDownloadsByAuthorResponse{
Status: "error",
Message: err.Error(),
})
return
}
if err := ctx.ShouldBind(&req); err != nil {
ctx.JSON(http.StatusBadRequest, GetGameDownloadsByAuthorResponse{
Status: "error",
Message: err.Error(),
})
return
}
if req.Page == 0 || req.Page < 0 {
req.Page = 1
}
if req.PageSize == 0 || req.PageSize < 0 {
req.PageSize = 10
}
if req.PageSize > 10 {
req.PageSize = 10
}
downloads, totalPage, err := db.GetGameDownloadsByAuthorPagination(req.Author, req.Page, req.PageSize)
if err != nil {
ctx.JSON(http.StatusInternalServerError, GetGameDownloadsByAuthorResponse{
Status: "error",
Message: err.Error(),
})
return
}
if len(downloads) == 0 {
ctx.JSON(http.StatusOK, GetGameDownloadsByAuthorResponse{
Status: "ok",
Message: "No results found",
})
return
}
ctx.JSON(http.StatusOK, GetGameDownloadsByAuthorResponse{
Status: "ok",
TotalPage: totalPage,
GameDownloads: downloads,
})
}

4
server/handler/get_game_info_by_id.go
View File

@ -33,7 +33,7 @@ type GetGameInfoByIDResponse struct {
// @Failure 500 {object} GetGameInfoByIDResponse
// @Router /game/id/{id} [get]
func GetGameInfoByIDHandler(c *gin.Context) {
var req GetGameDownloadByIDRequest
var req GetGameItemByIDRequest
if err := c.ShouldBindUri(&req); err != nil {
c.JSON(http.StatusBadRequest, GetGameInfoByIDResponse{
Status: "error",
@ -64,7 +64,7 @@ func GetGameInfoByIDHandler(c *gin.Context) {
})
return
}
gameInfo.Games, err = db.GetGameDownloadsByIDs(gameInfo.GameIDs)
gameInfo.Games, err = db.GetGameItemsByIDs(gameInfo.GameIDs)
if err != nil {
c.JSON(http.StatusInternalServerError, GetGameInfoByIDResponse{
Status: "error",

83
server/handler/get_games_by_author.go Normal file
View File

@ -0,0 +1,83 @@
package handler
import (
"net/http"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/gin-gonic/gin"
)
type GetGameItemsByAuthorRequest struct {
Author string `uri:"author" binding:"required"`
Page int `form:"page" json:"page"`
PageSize int `form:"page_size" json:"page_size"`
}
type GetGameItemsByAuthorResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
TotalPage int `json:"total_page"`
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
}
// GetGameItemsByAuthorHandler returns all game downloads by author
// @Summary Get game downloads by author
// @Description Get game downloads by author
// @Tags game
// @Accept json
// @Produce json
// @Param author path string true "Author"
// @Param page query int false "Page"
// @Param page_size query int false "Page Size"
// @Success 200 {object} GetGameItemsByAuthorResponse
// @Failure 400 {object} GetGameItemsByAuthorResponse
// @Failure 500 {object} GetGameItemsByAuthorResponse
// @Router /game/raw/author/{author} [get]
func GetGameItemsByAuthorHandler(ctx *gin.Context) {
var req GetGameItemsByAuthorRequest
if err := ctx.ShouldBindUri(&req); err != nil {
ctx.JSON(http.StatusBadRequest, GetGameItemsByAuthorResponse{
Status: "error",
Message: err.Error(),
})
return
}
if err := ctx.ShouldBind(&req); err != nil {
ctx.JSON(http.StatusBadRequest, GetGameItemsByAuthorResponse{
Status: "error",
Message: err.Error(),
})
return
}
if req.Page == 0 || req.Page < 0 {
req.Page = 1
}
if req.PageSize == 0 || req.PageSize < 0 {
req.PageSize = 10
}
if req.PageSize > 10 {
req.PageSize = 10
}
downloads, totalPage, err := db.GetGameItemsByAuthorPagination(req.Author, req.Page, req.PageSize)
if err != nil {
ctx.JSON(http.StatusInternalServerError, GetGameItemsByAuthorResponse{
Status: "error",
Message: err.Error(),
})
return
}
if len(downloads) == 0 {
ctx.JSON(http.StatusOK, GetGameItemsByAuthorResponse{
Status: "ok",
Message: "No results found",
})
return
}
ctx.JSON(http.StatusOK, GetGameItemsByAuthorResponse{
Status: "ok",
TotalPage: totalPage,
GameItems: downloads,
})
}

66
server/handler/get_unorganized_game_downloads.go
View File

@ -1,66 +0,0 @@
package handler
import (
"net/http"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/gin-gonic/gin"
)
type GetUnorganizedGameDownloadsRequest struct {
Num int `json:"num" form:"num"`
}
type GetUnorganizedGameDownloadsResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
Size int `json:"size,omitempty"`
GameDownloads []*model.GameDownload `json:"game_downloads,omitempty"`
}
// GetUnorganizedGameDownloads retrieves a list of unorganized game downloads.
// @Summary List unorganized game downloads
// @Description Retrieves game downloads that have not been organized
// @Tags game
// @Accept json
// @Produce json
// @Param num query int false "Number of game downloads to retrieve"
// @Success 200 {object} GetUnorganizedGameDownloadsResponse
// @Failure 400 {object} GetUnorganizedGameDownloadsResponse
// @Failure 500 {object} GetUnorganizedGameDownloadsResponse
// @Router /game/raw/unorganized [get]
func GetUnorganizedGameDownloadsHandler(c *gin.Context) {
var req GetUnorganizedGameDownloadsRequest
if err := c.ShouldBind(&req); err != nil {
c.JSON(http.StatusBadRequest, GetUnorganizedGameDownloadsResponse{
Status: "error",
Message: err.Error(),
})
return
}
if req.Num == 0 || req.Num < 0 {
req.Num = -1
}
gameDownloads, err := db.GetUnorganizedGameDownloads(req.Num)
if err != nil {
c.JSON(http.StatusInternalServerError, GetUnorganizedGameDownloadsResponse{
Status: "error",
Message: err.Error(),
})
return
}
if len(gameDownloads) == 0 {
c.JSON(http.StatusOK, GetUnorganizedGameDownloadsResponse{
Status: "ok",
Message: "No unorganized game downloads found",
})
return
}
c.JSON(http.StatusOK, GetUnorganizedGameDownloadsResponse{
Status: "ok",
GameDownloads: gameDownloads,
Size: len(gameDownloads),
})
}

66
server/handler/get_unorganized_games.go Normal file
View File

@ -0,0 +1,66 @@
package handler
import (
"net/http"
"github.com/nitezs/pcgamedb/db"
"github.com/nitezs/pcgamedb/model"
"github.com/gin-gonic/gin"
)
type GetUnorganizedGameItemsRequest struct {
Num int `json:"num" form:"num"`
}
type GetUnorganizedGameItemsResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
Size int `json:"size,omitempty"`
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
}
// GetUnorganizedGameItems retrieves a list of unorganized game downloads.
// @Summary List unorganized game downloads
// @Description Retrieves game downloads that have not been organized
// @Tags game
// @Accept json
// @Produce json
// @Param num query int false "Number of game downloads to retrieve"
// @Success 200 {object} GetUnorganizedGameItemsResponse
// @Failure 400 {object} GetUnorganizedGameItemsResponse
// @Failure 500 {object} GetUnorganizedGameItemsResponse
// @Router /game/raw/unorganized [get]
func GetUnorganizedGameItemsHandler(c *gin.Context) {
var req GetUnorganizedGameItemsRequest
if err := c.ShouldBind(&req); err != nil {
c.JSON(http.StatusBadRequest, GetUnorganizedGameItemsResponse{
Status: "error",
Message: err.Error(),
})
return
}
if req.Num == 0 || req.Num < 0 {
req.Num = -1
}
gameDownloads, err := db.GetUnorganizedGameItems(req.Num)
if err != nil {
c.JSON(http.StatusInternalServerError, GetUnorganizedGameItemsResponse{
Status: "error",
Message: err.Error(),
})
return
}
if len(gameDownloads) == 0 {
c.JSON(http.StatusOK, GetUnorganizedGameItemsResponse{
Status: "ok",
Message: "No unorganized game downloads found",
})
return
}
c.JSON(http.StatusOK, GetUnorganizedGameItemsResponse{
Status: "ok",
GameItems: gameDownloads,
Size: len(gameDownloads),
})
}

8
server/handler/healthcheck.go
View File

@ -21,7 +21,7 @@ type HealthCheckResponse struct {
Uptime string `json:"uptime"`
Alloc string `json:"alloc"`
AutoCrawl bool `json:"auto_crawl"`
GameDownload int64 `json:"game_download,omitempty"`
GameItem int64 `json:"game_download,omitempty"`
GameInfo int64 `json:"game_info,omitempty"`
Unorganized int64 `json:"unorganized,omitempty"`
RedisAvaliable bool `json:"redis_avaliable"`
@ -41,9 +41,9 @@ type HealthCheckResponse struct {
func HealthCheckHandler(c *gin.Context) {
var m runtime.MemStats
runtime.ReadMemStats(&m)
downloadCount, _ := db.GetGameDownloadCount()
downloadCount, _ := db.GetGameItemCount()
infoCount, _ := db.GetGameInfoCount()
unorganized, err := db.GetUnorganizedGameDownloads(-1)
unorganized, err := db.GetUnorganizedGameItems(-1)
unorganizedCount := int64(0)
if err == nil {
unorganizedCount = int64(len(unorganized))
@ -55,7 +55,7 @@ func HealthCheckHandler(c *gin.Context) {
Uptime: time.Since(config.Runtime.ServerStartTime).String(),
AutoCrawl: config.Config.Server.AutoCrawl,
Alloc: fmt.Sprintf("%.2f MB", float64(m.Alloc)/1024.0/1024.0),
GameDownload: downloadCount,
GameItem: downloadCount,
GameInfo: infoCount,
Unorganized: unorganizedCount,
RedisAvaliable: config.Config.RedisAvaliable,

30
server/handler/organize_game_info.go
View File

@ -10,36 +10,36 @@ import (
"go.mongodb.org/mongo-driver/bson/primitive"
)
type OrganizeGameDownloadRequest struct {
type OrganizeGameItemRequest struct {
Platform string `form:"platform" json:"platform" binding:"required"`
GameID string `form:"game_id" json:"game_id" binding:"required"`
PlatformID int `form:"platform_id" json:"platform_id" binding:"required"`
}
type OrganizeGameDownloadResponse struct {
type OrganizeGameItemResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
GameInfo *model.GameInfo `json:"game_info,omitempty"`
}
// OrganizeGameDownload organizes a specific game download.
// OrganizeGameItem organizes a specific game download.
// @Summary Organize a game download
// @Description Organizes a game download based on platform and game ID
// @Tags game
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization: Bearer <api_key>"
// @Param body body OrganizeGameDownloadRequest true "Organize Game Download Request"
// @Success 200 {object} OrganizeGameDownloadResponse
// @Failure 400 {object} OrganizeGameDownloadResponse
// @Failure 401 {object} OrganizeGameDownloadResponse
// @Failure 500 {object} OrganizeGameDownloadResponse
// @Param body body OrganizeGameItemRequest true "Organize Game Download Request"
// @Success 200 {object} OrganizeGameItemResponse
// @Failure 400 {object} OrganizeGameItemResponse
// @Failure 401 {object} OrganizeGameItemResponse
// @Failure 500 {object} OrganizeGameItemResponse
// @Security BearerAuth
// @Router /game/raw/organize [post]
func OrganizeGameDownloadHandler(c *gin.Context) {
var req OrganizeGameDownloadRequest
func OrganizeGameItemHandler(c *gin.Context) {
var req OrganizeGameItemRequest
if err := c.ShouldBind(&req); err != nil {
c.JSON(http.StatusBadRequest, OrganizeGameDownloadResponse{
c.JSON(http.StatusBadRequest, OrganizeGameItemResponse{
Status: "error",
Message: err.Error(),
})
@ -47,21 +47,21 @@ func OrganizeGameDownloadHandler(c *gin.Context) {
}
objID, err := primitive.ObjectIDFromHex(req.GameID)
if err != nil {
c.JSON(http.StatusBadRequest, OrganizeGameDownloadResponse{
c.JSON(http.StatusBadRequest, OrganizeGameItemResponse{
Status: "error",
Message: err.Error(),
})
return
}
info, err := crawler.OrganizeGameDownloadManually(objID, req.Platform, req.PlatformID)
info, err := crawler.OrganizeGameItemManually(objID, req.Platform, req.PlatformID)
if err != nil {
c.JSON(http.StatusInternalServerError, OrganizeGameDownloadResponse{
c.JSON(http.StatusInternalServerError, OrganizeGameItemResponse{
Status: "error",
Message: err.Error(),
})
return
}
c.JSON(http.StatusOK, OrganizeGameDownloadResponse{
c.JSON(http.StatusOK, OrganizeGameItemResponse{
Status: "ok",
GameInfo: info,
})

12
server/route.go
View File

@ -19,13 +19,13 @@ func initRoute(app *gin.Engine) {
}))
GameInfoGroup := app.Group("/game")
GameDownloadGroup := GameInfoGroup.Group("/raw")
GameItemGroup := GameInfoGroup.Group("/raw")
GameDownloadGroup.GET("/unorganized", handler.GetUnorganizedGameDownloadsHandler)
GameDownloadGroup.POST("/organize", middleware.Auth(), handler.OrganizeGameDownloadHandler)
GameDownloadGroup.GET("/id/:id", handler.GetGameDownloadByIDHanlder)
GameDownloadGroup.GET("/name/:name", handler.GetGameDownloadByRawNameHandler)
GameDownloadGroup.GET("/author/:author", handler.GetGameDownloadsByAuthorHandler)
GameItemGroup.GET("/unorganized", handler.GetUnorganizedGameItemsHandler)
GameItemGroup.POST("/organize", middleware.Auth(), handler.OrganizeGameItemHandler)
GameItemGroup.GET("/id/:id", handler.GetGameItemByIDHanlder)
GameItemGroup.GET("/name/:name", handler.GetGameItemByRawNameHandler)
GameItemGroup.GET("/author/:author", handler.GetGameItemsByAuthorHandler)
GameInfoGroup.GET("/search", handler.SearchGamesHandler)
GameInfoGroup.GET("/name/:name", handler.GetGameInfosByNameHandler)

2
task/crawl.go
View File

@ -13,7 +13,7 @@ import (
)
func Crawl(logger *zap.Logger) {
var games []*model.GameDownload
var games []*model.GameItem
var crawlerMap = crawler.BuildCrawlerMap(logger)
for _, item := range crawlerMap {
logger.Info("Crawling", zap.String("crawler", item.Name()))