fix: OrganizeGameItem

fix: save twitchtoken
remove: getSteamID
remove: update cmd&api
add: getIGDBIDBySteamSearch
add: update game info task
This commit is contained in:
Nite07 2024-11-28 18:37:01 +08:00
parent 71a2ac545b
commit 8702d3e93f
22 changed files with 352 additions and 363 deletions

View File

@ -10,32 +10,63 @@ import (
)
type taskCommandConfig struct {
Crawl bool
CrawlCron string
Cron string
Now bool
}
var taskCmdCfg taskCommandConfig
var taskCommandCfg taskCommandConfig
var taskCmd = &cobra.Command{
Use: "task",
Long: "Start task",
Short: "Start task",
var crawlTaskCmd = &cobra.Command{
Use: "crawl",
Long: "Start crawl task",
Short: "Start crawl task",
Run: func(cmd *cobra.Command, args []string) {
if taskCmdCfg.Crawl {
if taskCommandCfg.Now {
task.Crawl(log.Logger)
}
c := cron.New()
_, err := c.AddFunc(taskCmdCfg.CrawlCron, func() { task.Crawl(log.Logger) })
_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.Crawl(log.Logger) })
if err != nil {
log.Logger.Error("Failed to add task", zap.Error(err))
}
c.Start()
select {}
}
},
}
var updateTaskCmd = &cobra.Command{
Use: "update",
Long: "Start update outdated game infos task",
Short: "Start update outdated game infos task",
Run: func(cmd *cobra.Command, args []string) {
if taskCommandCfg.Now {
task.UpdateOutdatedGameInfos(log.Logger)
}
c := cron.New()
_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.UpdateOutdatedGameInfos(log.Logger) })
if err != nil {
log.Logger.Error("Failed to add task", zap.Error(err))
}
c.Start()
select {}
},
}
var taskCmd = &cobra.Command{
Use: "task",
Long: "Start task",
Short: "Start task",
}
func init() {
taskCmd.Flags().BoolVar(&taskCmdCfg.Crawl, "crawl", false, "enable auto crawl")
taskCmd.Flags().StringVar(&taskCmdCfg.CrawlCron, "crawl-cron", "0 */3 * * *", "crawl cron expression")
crawlTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
crawlTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
updateTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
updateTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
taskCmd.AddCommand(crawlTaskCmd)
taskCmd.AddCommand(updateTaskCmd)
RootCmd.AddCommand(taskCmd)
}

View File

@ -1,57 +0,0 @@
package cmd
import (
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/log"
"github.com/spf13/cobra"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.uber.org/zap"
)
var updateCmd = &cobra.Command{
Use: "update",
Long: "Update game info by game data platform",
Short: "Update game info by game data platform",
Run: updateRun,
}
type updateCommandConfig struct {
PlatformID int
Platform string
ID string
}
var updateCmdcfx updateCommandConfig
func init() {
updateCmd.Flags().IntVarP(&updateCmdcfx.PlatformID, "platform-id", "p", 0, "platform id")
updateCmd.Flags().StringVarP(&updateCmdcfx.Platform, "platform", "t", "", "platform")
updateCmd.Flags().StringVarP(&updateCmdcfx.ID, "game-id", "i", "", "game info id")
RootCmd.AddCommand(updateCmd)
}
func updateRun(cmd *cobra.Command, args []string) {
id, err := primitive.ObjectIDFromHex(updateCmdcfx.ID)
if err != nil {
log.Logger.Error("Failed to parse game info id", zap.Error(err))
return
}
oldInfo, err := db.GetGameInfoByID(id)
if err != nil {
log.Logger.Error("Failed to get game info", zap.Error(err))
return
}
newInfo, err := crawler.GenerateGameInfo(updateCmdcfx.Platform, updateCmdcfx.PlatformID)
if err != nil {
log.Logger.Error("Failed to generate game info", zap.Error(err))
return
}
newInfo.ID = id
newInfo.GameIDs = oldInfo.GameIDs
err = db.SaveGameInfo(newInfo)
if err != nil {
log.Logger.Error("Failed to save game info", zap.Error(err))
}
}

View File

@ -24,6 +24,7 @@ const (
Steam250Top250URL = "https://steam250.com/top250"
Steam250BestOfTheYearURL = "https://steam250.com/%v"
Steam250WeekTop50URL = "https://steam250.com/7day"
Steam250MonthTop50URL = "https://steam250.com/30day"
Steam250MostPlayedURL = "https://steam250.com/most_played"
FitGirlURL = "https://fitgirl-repacks.site/page/%v/"
SteamRIPBaseURL = "https://steamrip.com"

View File

@ -6,11 +6,12 @@ import (
"strings"
"time"
"go.uber.org/zap"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"go.uber.org/zap"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
)
@ -26,14 +27,14 @@ func GenerateGameInfo(platform string, id int) (*model.GameInfo, error) {
}
}
// OrganizeGameItem Organize and save GameInfo
// OrganizeGameItem Organize game item and save game info to database
func OrganizeGameItem(game *model.GameItem) error {
hasOriganized, _ := db.HasGameItemOrganized(game.ID)
if hasOriganized {
return nil
}
item, err := OrganizeGameItemWithIGDB(0, game)
item, err := OrganizeGameItemWithIGDB(game)
if err == nil {
if item.SteamID == 0 {
// get steam id from igdb
@ -41,20 +42,6 @@ func OrganizeGameItem(game *model.GameItem) error {
if err == nil {
item.SteamID = steamID
}
err = db.SaveGameInfo(item)
if err != nil {
return err
}
return nil
}
}
item, err = OrganizeGameItemWithSteam(0, game)
if err == nil {
if item.IGDBID == 0 {
igdbID, err := GetIGDBIDBySteamIDCache(item.SteamID)
if err == nil {
item.IGDBID = igdbID
}
}
err = db.SaveGameInfo(item)
if err != nil {
@ -100,7 +87,7 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo
}
}
if platform == "steam" {
igdbID, err := GetIGDBIDBySteamIDCache(platformID)
igdbID, err := GetIGDBIDBySteamAppIDCache(platformID)
if err == nil {
info.IGDBID = igdbID
}
@ -138,7 +125,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
changed = true
}
if info.SteamID != 0 && info.IGDBID == 0 {
igdbID, err := GetIGDBIDBySteamIDCache(info.SteamID)
igdbID, err := GetIGDBIDBySteamAppIDCache(info.SteamID)
time.Sleep(time.Millisecond * 100)
if err != nil {
continue
@ -147,9 +134,38 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
changed = true
}
if changed {
logger.Info("Supplemented platform id for game info", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
logger.Info("supp", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
_ = db.SaveGameInfo(info)
} else {
logger.Info("skip", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
}
}
return nil
}
func UpdateGameInfo(num int) (chan *model.GameInfo, error) {
infos, err := db.GetOutdatedGameInfos(num)
if err != nil {
return nil, err
}
updateChan := make(chan *model.GameInfo)
go func() {
for _, info := range infos {
if info.IGDBID != 0 {
newInfo, err := GenerateIGDBGameInfo(info.IGDBID)
if err != nil {
continue
}
db.MergeGameInfo(info, newInfo)
err = db.SaveGameInfo(newInfo)
if err != nil {
continue
}
updateChan <- newInfo
}
}
}()
return updateChan, nil
}

View File

@ -1,6 +1,7 @@
package crawler
import (
"bytes"
"encoding/json"
"errors"
"fmt"
@ -18,14 +19,18 @@ import (
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"github.com/PuerkitoBio/goquery"
)
type twitchToken struct {
token string
expires time.Time
Token string `json:"token"`
Expires time.Time `json:"expires"`
once sync.Once
}
var token = twitchToken{}
func (t *twitchToken) getToken() (string, error) {
t.once.Do(func() {
if config.Config.RedisAvaliable {
@ -34,19 +39,19 @@ func (t *twitchToken) getToken() (string, error) {
}
}
})
if t.token == "" || time.Now().After(t.expires) {
if t.Token == "" || time.Now().After(t.Expires) {
token, expires, err := loginTwitch()
if err != nil {
return "", fmt.Errorf("failed to login twitch: %w", err)
}
t.token = token
t.expires = expires
t.Token = token
t.Expires = expires
j, err := json.Marshal(t)
if err == nil {
_ = cache.Add("twitch_token", j)
}
}
return t.token, nil
return t.Token, nil
}
func loginTwitch() (string, time.Time, error) {
@ -100,8 +105,6 @@ func igdbFetch(URL string, dataBody any) (*utils.FetchResponse, error) {
return resp, nil
}
var token = twitchToken{}
func getIGDBID(name string) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
@ -110,6 +113,9 @@ func getIGDBID(name string) (int, error) {
}
if string(resp.Data) == "[]" {
resp, err = igdbFetch(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
if err != nil {
return 0, err
}
}
var data model.IGDBSearches
@ -150,6 +156,83 @@ func getIGDBID(name string) (int, error) {
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
func getIGDBIDBySteamSearch(name string) (int, error) {
baseURL, _ := url.Parse(constant.SteamSearchURL)
params := url.Values{}
params.Add("term", name)
baseURL.RawQuery = params.Encode()
resp, err := utils.Fetch(utils.FetchConfig{
Url: baseURL.String(),
})
if err != nil {
return 0, err
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
if err != nil {
return 0, err
}
type searchResult struct {
ID int
Type string
Name string
}
var items []searchResult
doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
if strings.HasPrefix(itemKey, "App_") {
id, err := strconv.Atoi(itemKey[4:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "App",
Name: name,
})
}
if strings.HasPrefix(itemKey, "Bundle_") {
id, err := strconv.Atoi(itemKey[7:])
if err != nil {
return
}
name := s.Find(".title").Text()
items = append(items, searchResult{
ID: id,
Type: "Bundle",
Name: name,
})
}
}
})
maxSim := 0.0
var maxSimItem searchResult
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
maxSimItem = item
break
} else {
sim := utils.Similarity(item.Name, name)
if sim >= 0.8 && sim > maxSim {
maxSim = sim
maxSimItem = item
}
}
}
if maxSim != 0 {
if maxSimItem.Type == "App" {
return GetIGDBIDBySteamAppIDCache(maxSimItem.ID)
}
if maxSimItem.Type == "Bundle" {
return GetIGDBIDBySteamBundleIDCache(maxSimItem.ID)
}
}
return 0, fmt.Errorf("steam ID not found: %s", name)
}
// GetIGDBAppParent returns the parent of the game, if no parent return itself
func GetIGDBAppParent(id int) (int, error) {
detail, err := GetIGDBAppDetailCache(id)
@ -226,7 +309,7 @@ func GetIGDBAppParentCache(id int) (int, error) {
return GetIGDBAppParent(id)
}
// GetIGDBID returns the IGDB ID of the game, try raw name first then formated names
// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
func GetIGDBID(name string) (int, error) {
name1 := name
name2 := FormatName(name)
@ -240,6 +323,12 @@ func GetIGDBID(name string) (int, error) {
return id, nil
}
}
for _, name := range names {
id, err := getIGDBIDBySteamSearch(name)
if err == nil {
return id, nil
}
}
return 0, errors.New("IGDB ID not found")
}
@ -305,7 +394,7 @@ func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
if err != nil {
return nil, err
}
_ = cache.Add(key, dataBytes)
_ = cache.AddWithExpire(key, dataBytes, 7*24*time.Hour)
return data, nil
}
} else {
@ -361,6 +450,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
item.Name = detail.Name
item.Description = detail.Summary
item.Cover = strings.Replace(detail.Cover.URL, "t_thumb", "t_original", 1)
item.InfoUpdatedAt = time.Now()
for _, lang := range detail.LanguageSupports {
if lang.LanguageSupportType == 3 {
@ -399,14 +489,11 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
}
// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
var err error
if id == 0 {
id, err = GetIGDBIDCache(game.Name)
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
id, err := GetIGDBIDCache(game.Name)
if err != nil {
return nil, err
}
}
d, err := db.GetGameInfoByPlatformID("igdb", id)
if err == nil {
d.GameIDs = append(d.GameIDs, game.ID)
@ -422,7 +509,7 @@ func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, er
return info, nil
}
func GetIGDBIDBySteamID(id int) (int, error) {
func GetIGDBIDBySteamAppID(id int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
@ -438,19 +525,19 @@ func GetIGDBIDBySteamID(id int) (int, error) {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamID(id)
return GetIGDBIDBySteamAppID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamIDCache(id int) (int, error) {
func GetIGDBIDBySteamAppIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_id:%v", id)
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamID(id)
data, err := GetIGDBIDBySteamAppID(id)
if err != nil {
return 0, err
}
@ -458,7 +545,47 @@ func GetIGDBIDBySteamIDCache(id int) (int, error) {
return data, nil
}
} else {
return GetIGDBIDBySteamID(id)
return GetIGDBIDBySteamAppID(id)
}
}
func GetIGDBIDBySteamBundleID(id int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
if err != nil {
return 0, err
}
var data []struct {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Data, &data); err != nil {
return 0, err
}
if len(data) == 0 {
return 0, errors.New("not found")
}
if data[0].Game == 0 {
return GetIGDBIDBySteamBundleID(id)
}
return GetIGDBAppParentCache(data[0].Game)
}
func GetIGDBIDBySteamBundleIDCache(id int) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
val, exist := cache.Get(key)
if exist {
return strconv.Atoi(val)
} else {
data, err := GetIGDBIDBySteamBundleID(id)
if err != nil {
return 0, err
}
_ = cache.Add(key, strconv.Itoa(data))
return data, nil
}
} else {
return GetIGDBIDBySteamBundleID(id)
}
}

View File

@ -8,99 +8,15 @@ import (
"regexp"
"strconv"
"strings"
"time"
"pcgamedb/cache"
"pcgamedb/config"
"pcgamedb/constant"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
)
func getSteamID(name string) (int, error) {
baseURL, _ := url.Parse(constant.SteamSearchURL)
params := url.Values{}
params.Add("term", name)
baseURL.RawQuery = params.Encode()
resp, err := utils.Fetch(utils.FetchConfig{
Url: baseURL.String(),
})
if err != nil {
return 0, err
}
idRegex := regexp.MustCompile(`data-ds-appid="(.*?)"`)
nameRegex := regexp.MustCompile(`<span class="title">(.*?)</span>`)
idRegexRes := idRegex.FindAllStringSubmatch(string(resp.Data), -1)
nameRegexRes := nameRegex.FindAllStringSubmatch(string(resp.Data), -1)
if len(idRegexRes) == 0 {
return 0, fmt.Errorf("steam ID not found: %s", name)
}
maxSim := 0.0
maxSimID := 0
for i, id := range idRegexRes {
idStr := id[1]
nameStr := nameRegexRes[i][1]
if index := strings.Index(idStr, ","); index != -1 {
idStr = idStr[:index]
}
if strings.EqualFold(strings.TrimSpace(nameStr), strings.TrimSpace(name)) {
return strconv.Atoi(idStr)
} else {
sim := utils.Similarity(nameStr, name)
if sim >= 0.8 && sim > maxSim {
maxSim = sim
maxSimID, _ = strconv.Atoi(idStr)
}
}
}
if maxSimID != 0 {
return maxSimID, nil
}
return 0, fmt.Errorf("steam ID not found: %s", name)
}
func GetSteamID(name string) (int, error) {
name1 := name
name2 := FormatName(name)
names := []string{name1}
if name1 != name2 {
names = append(names, name2)
}
for _, n := range names {
id, err := getSteamID(n)
if err == nil {
return id, nil
}
}
return 0, errors.New("steam ID not found")
}
func GetSteamIDCache(name string) (int, error) {
if config.Config.RedisAvaliable {
key := fmt.Sprintf("steam_id:%s", name)
val, exist := cache.Get(key)
if exist {
id, err := strconv.Atoi(val)
if err != nil {
return 0, err
}
return id, nil
} else {
id, err := GetSteamID(name)
if err != nil {
return 0, err
}
_ = cache.Add(key, id)
return id, nil
}
} else {
return GetSteamID(name)
}
}
func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
baseURL, _ := url.Parse(constant.SteamAppDetailURL)
params := url.Values{}
@ -168,6 +84,7 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
item.Cover = fmt.Sprintf("https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/%v/library_600x900_2x.jpg", id)
item.Developers = detail.Data.Developers
item.Publishers = detail.Data.Publishers
item.InfoUpdatedAt = time.Now()
var screenshots []string
for _, screenshot := range detail.Data.Screenshots {
screenshots = append(screenshots, screenshot.PathFull)
@ -176,30 +93,6 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
return item, nil
}
// OrganizeGameItemWithSteam Will add GameItem.ID to the newly added GameInfo.GameIDs
func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, error) {
var err error
if id == 0 {
id, err = GetSteamIDCache(game.Name)
if err != nil {
return nil, err
}
}
d, err := db.GetGameInfoByPlatformID("steam", id)
if err == nil {
d.GameIDs = append(d.GameIDs, game.ID)
d.GameIDs = utils.Unique(d.GameIDs)
return d, nil
}
detail, err := GenerateGameInfo("steam", id)
if err != nil {
return nil, err
}
detail.GameIDs = append(detail.GameIDs, game.ID)
detail.GameIDs = utils.Unique(detail.GameIDs)
return detail, nil
}
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
var err error
resp, err := igdbFetch(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))

View File

@ -62,6 +62,10 @@ func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250WeekTop50URL)
}
func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250MonthTop50URL)
}
func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
return GetSteam250(constant.Steam250MostPlayedURL)
}

View File

@ -141,6 +141,9 @@ func SaveGameInfo(item *model.GameInfo) error {
if item.CreatedAt.IsZero() {
item.CreatedAt = time.Now()
}
if item.InfoUpdatedAt.IsZero() {
item.InfoUpdatedAt = item.CreatedAt
}
item.UpdatedAt = time.Now()
filter := bson.M{"_id": item.ID}
update := bson.M{"$set": item}
@ -153,7 +156,7 @@ func SaveGameInfo(item *model.GameInfo) error {
}
func SaveGameInfos(items []*model.GameInfo) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
operations := make([]mongo.WriteModel, len(items))
@ -164,6 +167,9 @@ func SaveGameInfos(items []*model.GameInfo) error {
if item.CreatedAt.IsZero() {
item.CreatedAt = time.Now()
}
if item.InfoUpdatedAt.IsZero() {
item.InfoUpdatedAt = item.CreatedAt
}
item.UpdatedAt = time.Now()
operations[i] = mongo.NewUpdateOneModel().
SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
@ -889,3 +895,31 @@ func GetGameInfoByGameItemID(id primitive.ObjectID) (*model.GameInfo, error) {
}
return res[0], nil
}
func GetOutdatedGameInfos(maxNum int) ([]*model.GameInfo, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
opts := options.Find().SetLimit(int64(maxNum))
filter := bson.M{
"info_updated_at": bson.M{"$lt": time.Now().Add(-24 * time.Hour * 30)},
}
cursor, err := GameInfoCollection.Find(ctx, filter, opts)
if err != nil {
return nil, err
}
var res []*model.GameInfo
if err = cursor.All(ctx, &res); err != nil {
return nil, err
}
return res, nil
}
func MergeGameInfo(oldInfo *model.GameInfo, newInfo *model.GameInfo) {
newInfo.ID = oldInfo.ID
newInfo.UpdatedAt = time.Now()
newInfo.GameIDs = oldInfo.GameIDs
newInfo.IGDBID = oldInfo.IGDBID
newInfo.SteamID = oldInfo.SteamID
newInfo.CreatedAt = oldInfo.CreatedAt
newInfo.InfoUpdatedAt = time.Now()
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -19,9 +19,10 @@ type GameInfo struct {
Languages []string `json:"languages" bson:"languages"`
Screenshots []string `json:"screenshots" bson:"screenshots"`
GameIDs []primitive.ObjectID `json:"game_ids" bson:"games"`
Games []*GameItem `json:"game_downloads" bson:"-"`
Games []*GameItem `json:"games" bson:"-"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
InfoUpdatedAt time.Time `json:"info_updated_at" bson:"info_updated_at"`
}
type GameItem struct {

View File

@ -17,7 +17,7 @@ type GetGameItemByRawNameRequest struct {
type GetGameItemByRawNameResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
GameItem []*model.GameItem `json:"game_downloads,omitempty"`
GameItem []*model.GameItem `json:"games,omitempty"`
}
// GetGameItemByRawName retrieves game download details by raw name.
@ -26,7 +26,7 @@ type GetGameItemByRawNameResponse struct {
// @Tags game
// @Accept json
// @Produce json
// @Param name path string true "Game Download Raw Name"
// @Param name path string true "Game Raw Name"
// @Success 200 {object} GetGameItemByRawNameResponse
// @Failure 400 {object} GetGameItemByRawNameResponse
// @Failure 500 {object} GetGameItemByRawNameResponse

View File

@ -19,10 +19,10 @@ type GetGameItemsByAuthorResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
TotalPage int `json:"total_page"`
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
GameItems []*model.GameItem `json:"games,omitempty"`
}
// GetGameItemsByAuthorHandler returns all game downloads by author
// GetGameItemsByAuthorHandler returns games by author
// @Summary Get game downloads by author
// @Description Get game downloads by author
// @Tags game

View File

@ -1,11 +1,12 @@
package handler
import (
"github.com/gin-gonic/gin"
"net/http"
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/model"
"github.com/gin-gonic/gin"
)
type GetPopularGamesResponse struct {
@ -20,7 +21,7 @@ type GetPopularGamesResponse struct {
// @Tags popular
// @Accept json
// @Produce json
// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-best-of-the-year, steam-most-played)"
// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-month-top, steam-best-of-the-year, steam-most-played)"
// @Success 200 {object} GetPopularGamesResponse
// @Failure 400 {object} GetPopularGamesResponse
// @Failure 500 {object} GetPopularGamesResponse
@ -52,6 +53,8 @@ func GetPopularGameInfosHandler(c *gin.Context) {
steam250Func = crawler.GetSteam250BestOfTheYear
case "steam-most-played":
steam250Func = crawler.GetSteam250MostPlayed
case "steam-month-top":
steam250Func = crawler.GetSteam250MonthTop50
default:
c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
Status: "error",

View File

@ -17,16 +17,16 @@ type GetUnorganizedGameItemsResponse struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
Size int `json:"size,omitempty"`
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
GameItems []*model.GameItem `json:"games,omitempty"`
}
// GetUnorganizedGameItems retrieves a list of unorganized game downloads.
// @Summary List unorganized game downloads
// @Description Retrieves game downloads that have not been organized
// GetUnorganizedGameItems retrieves a list of unorganized games.
// @Summary List unorganized games
// @Description Retrieves games that have not been organized
// @Tags game
// @Accept json
// @Produce json
// @Param num query int false "Number of game downloads to retrieve"
// @Param num query int false "Number of games to retrieve"
// @Success 200 {object} GetUnorganizedGameItemsResponse
// @Failure 400 {object} GetUnorganizedGameItemsResponse
// @Failure 500 {object} GetUnorganizedGameItemsResponse

View File

@ -16,15 +16,14 @@ import (
type HealthCheckResponse struct {
Version string `json:"version"`
Status string `json:"status"`
Message string `json:"message,omitempty"`
Date string `json:"date"`
Uptime string `json:"uptime"`
Alloc string `json:"alloc"`
AutoCrawl bool `json:"auto_crawl"`
AutoCrawlCron string `json:"auto_crawl_cron,omitempty"`
GameItem int64 `json:"game_download,omitempty"`
GameInfo int64 `json:"game_info,omitempty"`
Unorganized int64 `json:"unorganized,omitempty"`
AutoCrawlCron string `json:"auto_crawl_cron"`
GameItem int64 `json:"game_num"`
GameInfo int64 `json:"game_info_num"`
Unorganized int64 `json:"unorganized_game_num"`
RedisAvaliable bool `json:"redis_avaliable"`
OnlineFixAvaliable bool `json:"online_fix_avaliable"`
MegaAvaliable bool `json:"mega_avaliable"`

View File

@ -1,100 +0,0 @@
package handler
import (
"net/http"
"strings"
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/model"
"github.com/gin-gonic/gin"
"go.mongodb.org/mongo-driver/bson/primitive"
)
type UpdateGameInfoRequest struct {
GameID string `json:"game_id" binding:"required"`
Platform string `json:"platform" binding:"required"`
PlatformID int `json:"platform_id" binding:"required"`
}
type UpdateGameInfoResponse struct {
Status string `json:"status"`
Message string `json:"message"`
GameInfo *model.GameInfo `json:"game_info,omitempty"`
}
// UpdateGameInfoHandler updates game information.
// @Summary Update game info
// @Description Updates details of a game
// @Tags game
// @Accept json
// @Produce json
// @Param Authorization header string true "Authorization: Bearer <api_key>"
// @Param body body handler.UpdateGameInfoRequest true "Update Game Info Request"
// @Success 200 {object} handler.UpdateGameInfoResponse
// @Failure 400 {object} handler.UpdateGameInfoResponse
// @Failure 401 {object} handler.UpdateGameInfoResponse
// @Failure 500 {object} handler.UpdateGameInfoResponse
// @Router /game/update [post]
func UpdateGameInfoHandler(c *gin.Context) {
var req UpdateGameInfoRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
req.Platform = strings.ToLower(req.Platform)
platformMap := map[string]bool{
"steam": true,
"igdb": true,
}
if _, ok := platformMap[req.Platform]; !ok {
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
Status: "error",
Message: "Invalid platform",
})
return
}
objID, err := primitive.ObjectIDFromHex(req.GameID)
if err != nil {
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
info, err := db.GetGameInfoByID(objID)
if err != nil {
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
newInfo, err := crawler.GenerateGameInfo(req.Platform, req.PlatformID)
if err != nil {
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
newInfo.ID = objID
newInfo.GameIDs = info.GameIDs
err = db.SaveGameInfo(newInfo)
if err != nil {
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
Status: "error",
Message: err.Error(),
})
return
}
c.JSON(http.StatusOK, UpdateGameInfoResponse{
Status: "ok",
Message: "Game info updated successfully",
GameInfo: newInfo,
})
}

View File

@ -31,7 +31,6 @@ func initRoute(app *gin.Engine) {
GameInfoGroup.GET("/name/:name", handler.GetGameInfosByNameHandler)
GameInfoGroup.GET("/platform/:platform_type/:platform_id", handler.GetGameInfoByPlatformIDHandler)
GameInfoGroup.GET("/id/:id", handler.GetGameInfoByIDHandler)
GameInfoGroup.PUT("/update", middleware.Auth(), handler.UpdateGameInfoHandler)
GameInfoGroup.DELETE("/id/:id", middleware.Auth(), handler.DeleteGameInfoHandler)
app.GET("/popular/:type", handler.GetPopularGameInfosHandler)

View File

@ -30,6 +30,8 @@ func Run() {
app.Use(middleware.Recovery())
initRoute(app)
log.Logger.Info("Server running", zap.String("port", config.Config.Server.Port))
// Start auto-crawl task
if config.Config.Server.AutoCrawl {
go func() {
c := cron.New()
@ -40,6 +42,17 @@ func Run() {
c.Start()
}()
}
// Start auto-update task
go func() {
c := cron.New()
_, err := c.AddFunc("0 */3 * * *", func() { task.UpdateOutdatedGameInfos(log.TaskLogger) })
if err != nil {
log.Logger.Error("Error adding cron job", zap.Error(err))
}
c.Start()
}()
err := app.Run(":" + config.Config.Server.Port)
if err != nil {
log.Logger.Panic("Failed to run server", zap.Error(err))

27
task/update_game_info.go Normal file
View File

@ -0,0 +1,27 @@
package task
import (
"pcgamedb/crawler"
"go.uber.org/zap"
)
func UpdateOutdatedGameInfos(logger *zap.Logger) {
channel, err := crawler.UpdateGameInfo(10)
count := 0
if err != nil {
logger.Error("Failed to update game info", zap.Error(err))
return
}
for info := range channel {
logger.Info("Updated game info",
zap.String("id", info.ID.String()),
zap.String("name", info.Name),
)
count++
if count == 10 {
break
}
}
logger.Info("Updated game info count", zap.Int("count", count))
}