nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

fix: OrganizeGameItem

Browse Source 
fix: save twitchtoken
remove: getSteamID
remove: update cmd&api
add: getIGDBIDBySteamSearch
add: update game info task
This commit is contained in:
Nite07 2024-11-28 18:37:01 +08:00
parent 71a2ac545b
commit 8702d3e93f
22 changed files with 352 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
cmd/task.go
View File

@ -10,32 +10,63 @@ import (
)
type taskCommandConfig struct {
Crawl bool
CrawlCron string
Cron string
Now bool
}
var taskCmdCfg taskCommandConfig
var taskCommandCfg taskCommandConfig
var crawlTaskCmd = &cobra.Command{
Use: "crawl",
Long: "Start crawl task",
Short: "Start crawl task",
Run: func(cmd *cobra.Command, args []string) {
if taskCommandCfg.Now {
task.Crawl(log.Logger)
}
c := cron.New()
_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.Crawl(log.Logger) })
if err != nil {
log.Logger.Error("Failed to add task", zap.Error(err))
}
c.Start()
select {}
},
}
var updateTaskCmd = &cobra.Command{
Use: "update",
Long: "Start update outdated game infos task",
Short: "Start update outdated game infos task",
Run: func(cmd *cobra.Command, args []string) {
if taskCommandCfg.Now {
task.UpdateOutdatedGameInfos(log.Logger)
}
c := cron.New()
_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.UpdateOutdatedGameInfos(log.Logger) })
if err != nil {
log.Logger.Error("Failed to add task", zap.Error(err))
}
c.Start()
select {}
},
}
var taskCmd = &cobra.Command{
Use: "task",
Long: "Start task",
Short: "Start task",
Run: func(cmd *cobra.Command, args []string) {
if taskCmdCfg.Crawl {
task.Crawl(log.Logger)
c := cron.New()
_, err := c.AddFunc(taskCmdCfg.CrawlCron, func() { task.Crawl(log.Logger) })
if err != nil {
log.Logger.Error("Failed to add task", zap.Error(err))
}
c.Start()
select {}
}
},
}
func init() {
taskCmd.Flags().BoolVar(&taskCmdCfg.Crawl, "crawl", false, "enable auto crawl")
taskCmd.Flags().StringVar(&taskCmdCfg.CrawlCron, "crawl-cron", "0 */3 * * *", "crawl cron expression")
crawlTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
crawlTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
updateTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
updateTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
taskCmd.AddCommand(crawlTaskCmd)
taskCmd.AddCommand(updateTaskCmd)
RootCmd.AddCommand(taskCmd)
}

57
cmd/update.go
View File

@ -1,57 +0,0 @@
package cmd
import (
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/log"
"github.com/spf13/cobra"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.uber.org/zap"
)
var updateCmd = &cobra.Command{
Use: "update",
Long: "Update game info by game data platform",
Short: "Update game info by game data platform",
Run: updateRun,
}
type updateCommandConfig struct {
PlatformID int
Platform string
ID string
}
var updateCmdcfx updateCommandConfig
func init() {
updateCmd.Flags().IntVarP(&updateCmdcfx.PlatformID, "platform-id", "p", 0, "platform id")
updateCmd.Flags().StringVarP(&updateCmdcfx.Platform, "platform", "t", "", "platform")
updateCmd.Flags().StringVarP(&updateCmdcfx.ID, "game-id", "i", "", "game info id")
RootCmd.AddCommand(updateCmd)
}
func updateRun(cmd *cobra.Command, args []string) {
id, err := primitive.ObjectIDFromHex(updateCmdcfx.ID)
if err != nil {
log.Logger.Error("Failed to parse game info id", zap.Error(err))
return
}
oldInfo, err := db.GetGameInfoByID(id)
if err != nil {
log.Logger.Error("Failed to get game info", zap.Error(err))
return
}
newInfo, err := crawler.GenerateGameInfo(updateCmdcfx.Platform, updateCmdcfx.PlatformID)
if err != nil {
log.Logger.Error("Failed to generate game info", zap.Error(err))
return
}
newInfo.ID = id
newInfo.GameIDs = oldInfo.GameIDs
err = db.SaveGameInfo(newInfo)
if err != nil {
log.Logger.Error("Failed to save game info", zap.Error(err))
}
}

1
constant/url.go
View File

@ -24,6 +24,7 @@ const (
Steam250Top250URL = "https://steam250.com/top250"
Steam250BestOfTheYearURL = "https://steam250.com/%v"
Steam250WeekTop50URL = "https://steam250.com/7day"
Steam250MonthTop50URL = "https://steam250.com/30day"
Steam250MostPlayedURL = "https://steam250.com/most_played"
FitGirlURL = "https://fitgirl-repacks.site/page/%v/"
SteamRIPBaseURL = "https://steamrip.com"

56
crawler/game.go
View File

@ -6,11 +6,12 @@ import (
"strings"
"time"
"go.uber.org/zap"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"go.uber.org/zap"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
)
@ -26,14 +27,14 @@ func GenerateGameInfo(platform string, id int) (*model.GameInfo, error) {
}
}
// OrganizeGameItem Organize and save GameInfo
// OrganizeGameItem Organize game item and save game info to database
func OrganizeGameItem(game *model.GameItem) error {
hasOriganized, _ := db.HasGameItemOrganized(game.ID)
if hasOriganized {
return nil
}
item, err := OrganizeGameItemWithIGDB(0, game)
item, err := OrganizeGameItemWithIGDB(game)
if err == nil {
if item.SteamID == 0 {
// get steam id from igdb
@ -41,20 +42,6 @@ func OrganizeGameItem(game *model.GameItem) error {
if err == nil {
item.SteamID = steamID
}
err = db.SaveGameInfo(item)
if err != nil {
return err
}
return nil
}
}
item, err = OrganizeGameItemWithSteam(0, game)
if err == nil {
if item.IGDBID == 0 {
igdbID, err := GetIGDBIDBySteamIDCache(item.SteamID)
if err == nil {
item.IGDBID = igdbID
}
}
err = db.SaveGameInfo(item)
if err != nil {
@ -100,7 +87,7 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo
}
}
if platform == "steam" {
igdbID, err := GetIGDBIDBySteamIDCache(platformID)
igdbID, err := GetIGDBIDBySteamAppIDCache(platformID)
if err == nil {
info.IGDBID = igdbID
}
@ -138,7 +125,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
changed = true
}
if info.SteamID != 0 && info.IGDBID == 0 {
igdbID, err := GetIGDBIDBySteamIDCache(info.SteamID)
igdbID, err := GetIGDBIDBySteamAppIDCache(info.SteamID)
time.Sleep(time.Millisecond * 100)
if err != nil {
continue
@ -147,9 +134,38 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
changed = true
}
if changed {
logger.Info("Supplemented platform id for game info", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
logger.Info("supp", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
_ = db.SaveGameInfo(info)
} else {
logger.Info("skip", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
}
}
return nil
}
func UpdateGameInfo(num int) (chan *model.GameInfo, error) {
infos, err := db.GetOutdatedGameInfos(num)
if err != nil {
return nil, err
}
updateChan := make(chan *model.GameInfo)
go func() {
for _, info := range infos {
if info.IGDBID != 0 {
newInfo, err := GenerateIGDBGameInfo(info.IGDBID)
if err != nil {
continue
}
db.MergeGameInfo(info, newInfo)
err = db.SaveGameInfo(newInfo)
if err != nil {
continue
}
updateChan <- newInfo
}
}
}()
return updateChan, nil
}

173
crawler/igdb.go
View File

@ -1,6 +1,7 @@
package crawler
import (
"bytes"
"encoding/json"
"errors"
"fmt"
@ -18,14 +19,18 @@ import (
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"github.com/PuerkitoBio/goquery"
)
type twitchToken struct {
token string
expires time.Time
Token string `json:"token"`
Expires time.Time `json:"expires"`
once sync.Once
}
var token = twitchToken{}
func (t *twitchToken) getToken() (string, error) {
t.once.Do(func() {
if config.Config.RedisAvaliable {
@ -34,19 +39,19 @@ func (t *twitchToken) getToken() (string, error) {
}
}
})
if t.token == "" || time.Now().After(t.expires) {
if t.Token == "" || time.Now().After(t.Expires) {
token, expires, err := loginTwitch()
if err != nil {
return "", fmt.Errorf("failed to login twitch: %w", err)
}
t.token = token
t.expires = expires
t.Token = token
t.Expires = expires
j, err := json.Marshal(t)
if err == nil {
_ = cache.Add("twitch_token", j)
}
}
return t.token, nil
return t.Token, nil
}
func loginTwitch() (string, time.Time, error) {
@ -100,8 +105,6 @@ func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) {
return resp, nil
}
var token = twitchToken{}
func getIGDBID(name string) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
@ -110,6 +113,9 @@ func getIGDBID(name string) (int, error) {
}
if string(resp.Data) == "[]" {
resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
if err != nil {
return 0, err
}
}
var data model.IGDBSearches
@ -150,6 +156,83 @@ func getIGDBID(name string) (int, error) {
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
func getIGDBIDBySteamSearch(name string) (int, error) {
baseURL, _ := url.Parse(constant.SteamSearchURL)
params := url.Values{}
params.Add("term", name)
baseURL.RawQuery = params.Encode()
resp, err := utils.Fetch(utils.FetchConfig{
Url: baseURL.String(),
})
if err != nil {
return 0, err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
if err != nil {
return 0, err
}
type searchResult struct {
ID int
Type string
Name string
}
var items []searchResult
doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
if strings.HasPrefix(itemKey, "App_") {
id, err := strconv.Atoi(itemKey[4:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "App",
Name: name,
})
}
if strings.HasPrefix(itemKey, "Bundle_") {
id, err := strconv.Atoi(itemKey[7:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "Bundle",
Name: name,
})
}
}
})
maxSim := 0.0
var maxSimItem searchResult
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
maxSimItem = item
break
} else {
sim := utils.Similarity(item.Name, name)
if sim >= 0.8 && sim > maxSim {
maxSim = sim
maxSimItem = item
}
}
}
if maxSim != 0 {
if maxSimItem.Type == "App" {
return GetIGDBIDBySteamAppIDCache(maxSimItem.ID)
}
if maxSimItem.Type == "Bundle" {
return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID)
}
}
return 0, fmt.Errorf("steam ID not found: %s", name)
}
// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
detail, err := GetIGDBAppDetailCache(id)
@ -226,7 +309,7 @@ func GetIGDBAppParentCache(id int) (int, error) {
return GetIGDBAppParent(id)
}
// GetIGDBID returns the IGDB ID of the game, try raw name first then formated names
// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
func GetIGDBID(name string) (int, error) {
name1 := name
name2 := FormatName(name)
@ -240,6 +323,12 @@ func GetIGDBID(name string) (int, error) {
return id, nil
}
}
for _, name := range names {
id, err := getIGDBIDBySteamSearch(name)
if err == nil {
return id, nil
}
}
return 0, errors.New("IGDB ID not found")
}
@ -305,7 +394,7 @@ func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
if err != nil {
return nil, err
}
_ = cache.Add(key, dataBytes)
_ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour)
return data, nil
}
} else {
@ -361,6 +450,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
item.Name = detail.Name
item.Description = detail.Summary
item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)
item.InfoUpdatedAt = time.Now()
for _, lang := range detail.LanguageSupports {
if lang.LanguageSupportType == 3 {
@ -399,13 +489,10 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
}
// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
var err error
if id == 0 {
id, err = GetIGDBIDCache(game.Name)
if err != nil {
return nil, err
}
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
id, err := GetIGDBIDCache(game.Name)
if err != nil {
return nil, err
}
d, err := db.GetGameInfoByPlatformID("igdb", id)
if err == nil {
@ -422,7 +509,7 @@ func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, er
return info, nil
}
func GetIGDBIDBySteamID(id int) (int, error) {
func GetIGDBIDBySteamAppID(id int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
@ -438,19 +525,19 @@ func GetIGDBIDBySteamID(id int) (int, error) {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamID(id)
return GetIGDBIDBySteamAppID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamIDCache(id int) (int, error) {
func GetIGDBIDBySteamAppIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_id:%v", id)
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamID(id)
data, err := GetIGDBIDBySteamAppID(id)
if err != nil {
return 0, err
}
@ -458,7 +545,47 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
return data, nil
}
} else {
return GetIGDBIDBySteamID(id)
return GetIGDBIDBySteamAppID(id)
}
}
func GetIGDBIDBySteamBundleID(id int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
return 0, err
}
var data []struct {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Data, &data); err != nil {
return 0, err
}
if len(data) == 0 {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamBundleID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamBundleIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamBundleID(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, strconv.Itoa(data))
return data, nil
}
} else {
return GetIGDBIDBySteamBundleID(id)
}
}

111
crawler/steam.go
View File

@ -8,99 +8,15 @@ import (
"regexp"
"strconv"
"strings"
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/constant"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
)
func getSteamID(name string) (int, error) {
baseURL, _ := url.Parse(constant.SteamSearchURL)
params := url.Values{}
params.Add("term", name)
baseURL.RawQuery = params.Encode()
resp, err := utils.Fetch(utils.FetchConfig{
Url: baseURL.String(),
})
if err != nil {
return 0, err
}
idRegex := regexp.MustCompile(`data-ds-appid="(.*?)"`)
nameRegex := regexp.MustCompile(`<span class="title">(.*?)</span>`)
idRegexRes := idRegex.FindAllStringSubmatch(string(resp.Data), -1)
nameRegexRes := nameRegex.FindAllStringSubmatch(string(resp.Data), -1)
if len(idRegexRes) == 0 {
return 0, fmt.Errorf("steam ID not found: %s", name)
}
maxSim := 0.0
maxSimID := 0
for i, id := range idRegexRes {
idStr := id[1]
nameStr := nameRegexRes[i][1]
if index := strings.Index(idStr, ","); index != -1 {
idStr = idStr[:index]
}
if strings.EqualFold(strings.TrimSpace(nameStr), strings.TrimSpace(name)) {
return strconv.Atoi(idStr)
} else {
sim := utils.Similarity(nameStr, name)
if sim >= 0.8 && sim > maxSim {
maxSim = sim
maxSimID, _ = strconv.Atoi(idStr)
}
}
}
if maxSimID != 0 {
return maxSimID, nil
}
return 0, fmt.Errorf("steam ID not found: %s", name)
}
func GetSteamID(name string) (int, error) {
name1 := name
name2 := FormatName(name)
names := []string{name1}
if name1 != name2 {
names = append(names, name2)
}
for _, n := range names {
id, err := getSteamID(n)
if err == nil {
return id, nil
}
}
return 0, errors.New("steam ID not found")
}
func GetSteamIDCache(name string) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("steam_id:%s", name)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetSteamID(name)
if err != nil {
return 0, err
}
_ = cache.Add(key, id)
return id, nil
}
} else {
return GetSteamID(name)
}
}
func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
baseURL, _ := url.Parse(constant.SteamAppDetailURL)
params := url.Values{}
@ -168,6 +84,7 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
item.Cover = fmt.Sprintf("https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/%v/library_600x900_2x.jpg", id)
item.Developers = detail.Data.Developers
item.Publishers = detail.Data.Publishers
item.InfoUpdatedAt = time.Now()
var screenshots []string
for _, screenshot := range detail.Data.Screenshots {
screenshots = append(screenshots, screenshot.PathFull)
@ -176,30 +93,6 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
return item, nil
}
// OrganizeGameItemWithSteam Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, error) {
var err error
if id == 0 {
id, err = GetSteamIDCache(game.Name)
if err != nil {
return nil, err
}
}
d, err := db.GetGameInfoByPlatformID("steam", id)
if err == nil {
d.GameIDs = append(d.GameIDs, game.ID)
d.GameIDs = utils.Unique(d.GameIDs)
return d, nil
}
detail, err := GenerateGameInfo("steam", id)
if err != nil {
return nil, err
}
detail.GameIDs = append(detail.GameIDs, game.ID)
detail.GameIDs = utils.Unique(detail.GameIDs)
return detail, nil
}
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))

4
crawler/steam250.go
View File

@ -62,6 +62,10 @@ func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250WeekTop50URL)
}
func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250MonthTop50URL)
}
func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250MostPlayedURL)
}

36
db/game.go
View File

@ -141,6 +141,9 @@ func SaveGameInfo(item *model.GameInfo) error {
if item.CreatedAt.IsZero() {
item.CreatedAt = time.Now()
}
if item.InfoUpdatedAt.IsZero() {
item.InfoUpdatedAt = item.CreatedAt
}
item.UpdatedAt = time.Now()
filter := bson.M{"_id": item.ID}
update := bson.M{"$set": item}
@ -153,7 +156,7 @@ func SaveGameInfo(item *model.GameInfo) error {
}
func SaveGameInfos(items []*model.GameInfo) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
operations := make([]mongo.WriteModel, len(items))
@ -164,6 +167,9 @@ func SaveGameInfos(items []*model.GameInfo) error {
if item.CreatedAt.IsZero() {
item.CreatedAt = time.Now()
}
if item.InfoUpdatedAt.IsZero() {
item.InfoUpdatedAt = item.CreatedAt
}
item.UpdatedAt = time.Now()
operations[i] = mongo.NewUpdateOneModel().
SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
@ -889,3 +895,31 @@ func GetGameInfoByGameItemID(id primitive.ObjectID) (*model.GameInfo, error) {
}
return res[0], nil
}
func GetOutdatedGameInfos(maxNum int) ([]*model.GameInfo, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
opts := options.Find().SetLimit(int64(maxNum))
filter := bson.M{
"info_updated_at": bson.M{"$lt": time.Now().Add(-24 * time.Hour * 30)},
}
cursor, err := GameInfoCollection.Find(ctx, filter, opts)
if err != nil {
return nil, err
}
var res []*model.GameInfo
if err = cursor.All(ctx, &res); err != nil {
return nil, err
}
return res, nil
}
func MergeGameInfo(oldInfo *model.GameInfo, newInfo *model.GameInfo) {
newInfo.ID = oldInfo.ID
newInfo.UpdatedAt = time.Now()
newInfo.GameIDs = oldInfo.GameIDs
newInfo.IGDBID = oldInfo.IGDBID
newInfo.SteamID = oldInfo.SteamID
newInfo.CreatedAt = oldInfo.CreatedAt
newInfo.InfoUpdatedAt = time.Now()
}

1
game_infos-backup.json
View File

File diff suppressed because one or more lines are too long

2
game_infos.json
View File

File diff suppressed because one or more lines are too long

1
games-backup.json
View File

File diff suppressed because one or more lines are too long

2
games.json
View File

File diff suppressed because one or more lines are too long

31
model/game.go
View File

@ -7,21 +7,22 @@ import (
)
type GameInfo struct {
ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Aliases []string `json:"aliases" bson:"aliases"`
Developers []string `json:"developers" bson:"developers"`
Publishers []string `json:"publishers" bson:"publishers"`
IGDBID int `json:"igdb_id" bson:"igdb_id"`
SteamID int `json:"steam_id" bson:"steam_id"`
Cover string `json:"cover" bson:"cover"`
Languages []string `json:"languages" bson:"languages"`
Screenshots []string `json:"screenshots" bson:"screenshots"`
GameIDs []primitive.ObjectID `json:"game_ids" bson:"games"`
Games []*GameItem `json:"game_downloads" bson:"-"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"name" bson:"name"`
Description string `json:"description" bson:"description"`
Aliases []string `json:"aliases" bson:"aliases"`
Developers []string `json:"developers" bson:"developers"`
Publishers []string `json:"publishers" bson:"publishers"`
IGDBID int `json:"igdb_id" bson:"igdb_id"`
SteamID int `json:"steam_id" bson:"steam_id"`
Cover string `json:"cover" bson:"cover"`
Languages []string `json:"languages" bson:"languages"`
Screenshots []string `json:"screenshots" bson:"screenshots"`
GameIDs []primitive.ObjectID `json:"game_ids" bson:"games"`
Games []*GameItem `json:"games" bson:"-"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
InfoUpdatedAt time.Time `json:"info_updated_at" bson:"info_updated_at"`
}
type GameItem struct {

4
server/handler/get_game_by_raw_name.go
View File

@ -17,7 +17,7 @@ type GetGameItemByRawNameRequest struct {
type GetGameItemByRawNameResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
GameItem []*model.GameItem `json:"game_downloads,omitempty"`
GameItem []*model.GameItem `json:"games,omitempty"`
}
// GetGameItemByRawName retrieves game download details by raw name.
@ -26,7 +26,7 @@ type GetGameItemByRawNameResponse struct {
// @Tags game
// @Accept json
// @Produce json
// @Param name path string true "Game Download Raw Name"
// @Param name path string true "Game Raw Name"
// @Success 200 {object} GetGameItemByRawNameResponse
// @Failure 400 {object} GetGameItemByRawNameResponse
// @Failure 500 {object} GetGameItemByRawNameResponse

4
server/handler/get_games_by_author.go
View File

@ -19,10 +19,10 @@ type GetGameItemsByAuthorResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
TotalPage int `json:"total_page"`
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
GameItems []*model.GameItem `json:"games,omitempty"`
}
// GetGameItemsByAuthorHandler returns all game downloads by author
// GetGameItemsByAuthorHandler returns games by author
// @Summary Get game downloads by author
// @Description Get game downloads by author
// @Tags game

7
server/handler/get_popular_game_infos.go
View File

@ -1,11 +1,12 @@
package handler
import (
"github.com/gin-gonic/gin"
"net/http"
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/model"
"github.com/gin-gonic/gin"
)
type GetPopularGamesResponse struct {
@ -20,7 +21,7 @@ type GetPopularGamesResponse struct {
// @Tags popular
// @Accept json
// @Produce json
// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-best-of-the-year, steam-most-played)"
// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-month-top, steam-best-of-the-year, steam-most-played)"
// @Success 200 {object} GetPopularGamesResponse
// @Failure 400 {object} GetPopularGamesResponse
// @Failure 500 {object} GetPopularGamesResponse
@ -52,6 +53,8 @@ func GetPopularGameInfosHandler(c *gin.Context) {
steam250Func = crawler.GetSteam250BestOfTheYear
case "steam-most-played":
steam250Func = crawler.GetSteam250MostPlayed
case "steam-month-top":
steam250Func = crawler.GetSteam250MonthTop50
default:
c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
Status: "error",

10
server/handler/get_unorganized_games.go
View File

@ -17,16 +17,16 @@ type GetUnorganizedGameItemsResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
Size int `json:"size,omitempty"`
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
GameItems []*model.GameItem `json:"games,omitempty"`
}
// GetUnorganizedGameItems retrieves a list of unorganized game downloads.
// @Summary List unorganized game downloads
// @Description Retrieves game downloads that have not been organized
// GetUnorganizedGameItems retrieves a list of unorganized games.
// @Summary List unorganized games
// @Description Retrieves games that have not been organized
// @Tags game
// @Accept json
// @Produce json
// @Param num query int false "Number of game downloads to retrieve"
// @Param num query int false "Number of games to retrieve"
// @Success 200 {object} GetUnorganizedGameItemsResponse
// @Failure 400 {object} GetUnorganizedGameItemsResponse
// @Failure 500 {object} GetUnorganizedGameItemsResponse

9
server/handler/healthcheck.go
View File

@ -16,15 +16,14 @@ import (
type HealthCheckResponse struct {
Version string `json:"version"`
Status string `json:"status"`
Message string `json:"message,omitempty"`
Date string `json:"date"`
Uptime string `json:"uptime"`
Alloc string `json:"alloc"`
AutoCrawl bool `json:"auto_crawl"`
AutoCrawlCron string `json:"auto_crawl_cron,omitempty"`
GameItem int64 `json:"game_download,omitempty"`
GameInfo int64 `json:"game_info,omitempty"`
Unorganized int64 `json:"unorganized,omitempty"`
AutoCrawlCron string `json:"auto_crawl_cron"`
GameItem int64 `json:"game_num"`
GameInfo int64 `json:"game_info_num"`
Unorganized int64 `json:"unorganized_game_num"`
RedisAvaliable bool `json:"redis_avaliable"`
OnlineFixAvaliable bool `json:"online_fix_avaliable"`
MegaAvaliable bool `json:"mega_avaliable"`

100
server/handler/update_game_info.go
View File

@ -1,100 +0,0 @@
package handler
import (
"net/http"
"strings"
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/model"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UpdateGameInfoRequest struct {
GameID string `json:"game_id" binding:"required"`
Platform string `json:"platform" binding:"required"`
PlatformID int `json:"platform_id" binding:"required"`
}
type UpdateGameInfoResponse struct {
Status string `json:"status"`
Message string `json:"message"`
GameInfo *model.GameInfo `json:"game_info,omitempty"`
}
// UpdateGameInfoHandler updates game information.
// @Summary Update game info
// @Description Updates details of a game
// @Tags game
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization: Bearer <api_key>"
// @Param body body handler.UpdateGameInfoRequest true "Update Game Info Request"
// @Success 200 {object} handler.UpdateGameInfoResponse
// @Failure 400 {object} handler.UpdateGameInfoResponse
// @Failure 401 {object} handler.UpdateGameInfoResponse
// @Failure 500 {object} handler.UpdateGameInfoResponse
// @Router /game/update [post]
func UpdateGameInfoHandler(c *gin.Context) {
var req UpdateGameInfoRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
req.Platform = strings.ToLower(req.Platform)
platformMap := map[string]bool{
"steam": true,
"igdb": true,
}
if _, ok := platformMap[req.Platform]; !ok {
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
Status: "error",
Message: "Invalid platform",
})
return
}
objID, err := primitive.ObjectIDFromHex(req.GameID)
if err != nil {
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
info, err := db.GetGameInfoByID(objID)
if err != nil {
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
newInfo, err := crawler.GenerateGameInfo(req.Platform, req.PlatformID)
if err != nil {
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
newInfo.ID = objID
newInfo.GameIDs = info.GameIDs
err = db.SaveGameInfo(newInfo)
if err != nil {
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
c.JSON(http.StatusOK, UpdateGameInfoResponse{
Status: "ok",
Message: "Game info updated successfully",
GameInfo: newInfo,
})
}

1
server/route.go
View File

@ -31,7 +31,6 @@ func initRoute(app *gin.Engine) {
GameInfoGroup.GET("/name/:name", handler.GetGameInfosByNameHandler)
GameInfoGroup.GET("/platform/:platform_type/:platform_id", handler.GetGameInfoByPlatformIDHandler)
GameInfoGroup.GET("/id/:id", handler.GetGameInfoByIDHandler)
GameInfoGroup.PUT("/update", middleware.Auth(), handler.UpdateGameInfoHandler)
GameInfoGroup.DELETE("/id/:id", middleware.Auth(), handler.DeleteGameInfoHandler)
app.GET("/popular/:type", handler.GetPopularGameInfosHandler)

13
server/server.go
View File

@ -30,6 +30,8 @@ func Run() {
app.Use(middleware.Recovery())
initRoute(app)
log.Logger.Info("Server running", zap.String("port", config.Config.Server.Port))
// Start auto-crawl task
if config.Config.Server.AutoCrawl {
go func() {
c := cron.New()
@ -40,6 +42,17 @@ func Run() {
c.Start()
}()
}
// Start auto-update task
go func() {
c := cron.New()
_, err := c.AddFunc("0 */3 * * *", func() { task.UpdateOutdatedGameInfos(log.TaskLogger) })
if err != nil {
log.Logger.Error("Error adding cron job", zap.Error(err))
}
c.Start()
}()
err := app.Run(":" + config.Config.Server.Port)
if err != nil {
log.Logger.Panic("Failed to run server", zap.Error(err))

27
task/update_game_info.go Normal file
View File

@ -0,0 +1,27 @@
package task
import (
"pcgamedb/crawler"
"go.uber.org/zap"
)
func UpdateOutdatedGameInfos(logger *zap.Logger) {
channel, err := crawler.UpdateGameInfo(10)
count := 0
if err != nil {
logger.Error("Failed to update game info", zap.Error(err))
return
}
for info := range channel {
logger.Info("Updated game info",
zap.String("id", info.ID.String()),
zap.String("name", info.Name),
)
count++
if count == 10 {
break
}
}
logger.Info("Updated game info count", zap.Int("count", count))
}