reorganized game infos

change ranking route to popular route

.gitignore

@@ -8,3 +8,4 @@ docs
 deploy.sh
 config.json
 organize.json
+cmd/test.go

cmd/organize.go

@@ -36,6 +36,7 @@ func organizeRun(cmd *cobra.Command, args []string) {
 		err := crawler.OrganizeGameItem(game)
 		if err != nil {
 			log.Logger.Error("failed to organize game item")
+			continue
 		}
 		log.Logger.Info("game item organized", zap.String("name", game.Name))
 	}

cmd/task.go

@@ -19,6 +19,7 @@ var taskCmdCfg taskCommandConfig
 var taskCmd = &cobra.Command{
 	Use:   "task",
 	Long:  "Start task",
+	Short: "Start task",
 	Run: func(cmd *cobra.Command, args []string) {
 		if taskCmdCfg.Crawl {
 			task.Crawl(log.Logger)

constant/url.go

@@ -19,6 +19,7 @@ const (
 	IGDBSearchURL            = "https://api.igdb.com/v4/search"
 	IGDBCompaniesURL         = "https://api.igdb.com/v4/companies"
 	IGDBWebsitesURL          = "https://api.igdb.com/v4/websites"
+	IGDBPopularityURL        = "https://api.igdb.com/v4/popularity_primitives"
 	TwitchAuthURL            = "https://id.twitch.tv/oauth2/token"
 	Steam250Top250URL        = "https://steam250.com/top250"
 	Steam250BestOfTheYearURL = "https://steam250.com/%v"

crawler/igdb.go

@@ -20,7 +20,7 @@ import (
 
 var TwitchToken string
 
-func _GetIGDBID(name string) (int, error) {
+func getIGDBID(name string) (int, error) {
 	var err error
 	if TwitchToken == "" {
 		TwitchToken, err = LoginTwitch()
@@ -62,26 +62,80 @@ func _GetIGDBID(name string) (int, error) {
 	if len(data) == 1 {
 		return data[0].Game, nil
 	}
-	for _, item := range data {
+	maxSimilairty := 0.0
+	maxSimilairtyIndex := 0
+	for i, item := range data {
 		if strings.EqualFold(item.Name, name) {
 			return item.Game, nil
 		}
-		if utils.Similarity(name, item.Name) >= 0.8 {
+		if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
-			return item.Game, nil
+			if sim > maxSimilairty {
+				maxSimilairty = sim
+				maxSimilairtyIndex = i
+			}
 		}
 		detail, err := GetIGDBAppDetailCache(item.Game)
 		if err != nil {
 			return 0, err
 		}
 		for _, alternativeNames := range detail.AlternativeNames {
-			if utils.Similarity(alternativeNames.Name, name) >= 0.8 {
+			if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 {
-				return item.Game, nil
+				if sim > maxSimilairty {
+					maxSimilairty = sim
+					maxSimilairtyIndex = i
 				}
 			}
 		}
+	}
+	if maxSimilairty >= 0.8 {
+		return GetIGDBAppParentCache(data[maxSimilairtyIndex].Game)
+	}
 	return 0, fmt.Errorf("IGDB ID not found: %s", name)
 }
 
+// GetIGDBAppParent returns the parent of the game, if no parent return itself
+func GetIGDBAppParent(id int) (int, error) {
+	detail, err := GetIGDBAppDetailCache(id)
+	if err != nil {
+		return 0, err
+	}
+	versionParent := detail.VersionParent
+	for versionParent != 0 {
+		detail, err = GetIGDBAppDetailCache(versionParent)
+		if err != nil {
+			return 0, err
+		}
+		versionParent = detail.VersionParent
+	}
+	if versionParent != 0 {
+		return versionParent, nil
+	}
+	return id, nil
+}
+
+func GetIGDBAppParentCache(id int) (int, error) {
+	if config.Config.RedisAvaliable {
+		key := fmt.Sprintf("igdb_parent:%d", id)
+		val, exist := cache.Get(key)
+		if exist {
+			id, err := strconv.Atoi(val)
+			if err != nil {
+				return 0, err
+			}
+			return id, nil
+		} else {
+			id, err := GetIGDBAppParent(id)
+			if err != nil {
+				return 0, err
+			}
+			_ = cache.Add(key, id)
+			return id, nil
+		}
+	}
+	return GetIGDBAppParent(id)
+}
+
+// GetIGDBID returns the IGDB ID of the game, try raw name first then formated names
 func GetIGDBID(name string) (int, error) {
 	name1 := name
 	name2 := FormatName(name)
@@ -90,7 +144,7 @@ func GetIGDBID(name string) (int, error) {
 		names = append(names, name2)
 	}
 	for _, name := range names {
-		id, err := _GetIGDBID(name)
+		id, err := getIGDBID(name)
 		if err == nil {
 			return id, nil
 		}
@@ -369,7 +423,7 @@ func GetIGDBIDBySteamID(id int) (int, error) {
 	if data[0].Game == 0 {
 		return GetIGDBIDBySteamID(id)
 	}
-	return data[0].Game, nil
+	return GetIGDBAppParentCache(data[0].Game)
 }
 
 func GetIGDBIDBySteamIDCache(id int) (int, error) {
@@ -436,7 +490,12 @@ func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
 		}
 		id, err := strconv.Atoi(idStr[1])
 		if err == nil {
-			ret[id] = d.Game
+			pid, err := GetIGDBAppParentCache(d.Game)
+			if err == nil {
+				ret[id] = pid
+			} else {
+				ret[id] = 0
+			}
 		}
 	}
 	for _, id := range ids {
@@ -479,3 +538,50 @@ func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
 		return GetIGDBIDsBySteamIDs(ids)
 	}
 }
+
+// GetIGDBPopularGameIDs get IGDB popular game IDs
+// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
+// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists.
+// popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists.
+// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
+func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
+	var err error
+	if TwitchToken == "" {
+		TwitchToken, err = LoginTwitch()
+		if err != nil {
+			return nil, err
+		}
+	}
+	resp, err := utils.Fetch(utils.FetchConfig{
+		Url:    constant.IGDBPopularityURL,
+		Method: "POST",
+		Headers: map[string]string{
+			"Client-ID":     config.Config.Twitch.ClientID,
+			"Authorization": "Bearer " + TwitchToken,
+			"User-Agent":    "",
+			"Content-Type":  "text/plain",
+		},
+		Data: fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType),
+	})
+	if err != nil {
+		return nil, err
+	}
+	type IGDBPopularity struct {
+		GameID int     `json:"game_id"`
+		Value  float64 `json:"value"`
+	}
+	var data []IGDBPopularity
+	if err = json.Unmarshal(resp.Data, &data); err != nil {
+		return nil, err
+	}
+	ret := make([]int, 0)
+	for _, d := range data {
+		pid, err := GetIGDBAppParentCache(d.GameID)
+		if err != nil {
+			ret = append(ret, d.GameID)
+			continue
+		}
+		ret = append(ret, pid)
+	}
+	return ret, nil
+}

crawler/steam.go

@@ -17,7 +17,7 @@ import (
 	"pcgamedb/utils"
 )
 
-func _GetSteamID(name string) (int, error) {
+func getSteamID(name string) (int, error) {
 	baseURL, _ := url.Parse(constant.SteamSearchURL)
 	params := url.Values{}
 	params.Add("term", name)
@@ -70,7 +70,7 @@ func GetSteamID(name string) (int, error) {
 		names = append(names, name2)
 	}
 	for _, n := range names {
-		id, err := _GetSteamID(n)
+		id, err := getSteamID(n)
 		if err == nil {
 			return id, nil
 		}

crawler/steam250.go

@@ -2,16 +2,13 @@ package crawler
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
+	"pcgamedb/db"
 	"regexp"
 	"strconv"
 	"time"
 
-	"pcgamedb/cache"
-	"pcgamedb/config"
 	"pcgamedb/constant"
-	"pcgamedb/db"
 	"pcgamedb/model"
 	"pcgamedb/utils"
 
@@ -43,81 +40,25 @@ func GetSteam250(url string) ([]*model.GameInfo, error) {
 		rank = append(rank, item)
 		steamIDs = append(steamIDs, item.SteamID)
 	})
-	var res []*model.GameInfo
+	infos, err := db.GetGameInfosByPlatformIDs("steam", steamIDs)
-	count := 0
+	if err != nil {
-	for _, steamID := range steamIDs {
+		return nil, err
-		if count >= 10 {
-			break
 	}
-		info, err := db.GetGameInfoByPlatformID("steam", steamID)
+	return infos[:10], nil
-		if err == nil {
-			res = append(res, info)
-			count++
-			continue
-		}
-	}
-	return res, nil
 }
 
 func GetSteam250Top250() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250Top250URL)
 }
 
-func GetSteam250Top250Cache() ([]*model.GameInfo, error) {
-	return GetSteam250Cache("top250", GetSteam250Top250)
-}
-
 func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) {
 	return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, time.Now().UTC().Year()))
 }
 
-func GetSteam250BestOfTheYearCache() ([]*model.GameInfo, error) {
-	return GetSteam250Cache(fmt.Sprintf("bestoftheyear:%v", time.Now().UTC().Year()), GetSteam250BestOfTheYear)
-}
-
 func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250WeekTop50URL)
 }
 
-func GetSteam250WeekTop50Cache() ([]*model.GameInfo, error) {
-	return GetSteam250Cache("weektop50", GetSteam250WeekTop50)
-}
-
 func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
 	return GetSteam250(constant.Steam250MostPlayedURL)
 }
-
-func GetSteam250MostPlayedCache() ([]*model.GameInfo, error) {
-	return GetSteam250Cache("mostplayed", GetSteam250MostPlayed)
-}
-
-func GetSteam250Cache(k string, f func() ([]*model.GameInfo, error)) ([]*model.GameInfo, error) {
-	if config.Config.RedisAvaliable {
-		key := k
-		val, exist := cache.Get(key)
-		if exist {
-			var res []*model.GameInfo
-			err := json.Unmarshal([]byte(val), &res)
-			if err != nil {
-				return nil, err
-			}
-			return res, nil
-		} else {
-			data, err := f()
-			if err != nil {
-				return nil, err
-			}
-			dataBytes, err := json.Marshal(data)
-			if err != nil {
-				return data, nil
-			}
-			err = cache.AddWithExpire(key, dataBytes, 12*time.Hour)
-			if err != nil {
-				return data, nil
-			}
-			return data, nil
-		}
-	} else {
-		return f()
-	}
-}

db/game.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"pcgamedb/utils"
 	"regexp"
 	"strings"
 	"time"
@@ -349,6 +350,27 @@ func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {
 	return &game, nil
 }
 
+func GetGameInfosByPlatformIDs(platform string, ids []int) ([]*model.GameInfo, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	var filter interface{}
+	switch platform {
+	case "steam":
+		filter = bson.M{"steam_id": bson.M{"$in": ids}}
+	case "igdb":
+		filter = bson.M{"igdb_id": bson.M{"$in": ids}}
+	}
+	var games []*model.GameInfo
+	cursor, err := GameInfoCollection.Find(ctx, filter)
+	if err != nil {
+		return nil, err
+	}
+	if err = cursor.All(ctx, &games); err != nil {
+		return nil, err
+	}
+	return games, nil
+}
+
 func HasGameItemOrganized(id primitive.ObjectID) (bool, []*model.GameInfo) {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -608,7 +630,60 @@ func GetSameNameGameInfos() (map[string][]primitive.ObjectID, error) {
 	return res, nil
 }
 
-func MergeSameNameGameInfos() error {
+func DeleteGameInfosByIDs(ids []primitive.ObjectID) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	_, err := GameInfoCollection.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func MergeGameInfosWithSameIGDBID() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	pipeline := mongo.Pipeline{
+		bson.D{{Key: "$group", Value: bson.D{
+			{Key: "_id", Value: "$igdb_id"},
+			{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
+			{Key: "docs", Value: bson.D{{Key: "$push", Value: "$$ROOT"}}},
+		}}},
+		bson.D{{Key: "$match", Value: bson.D{{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}}}}},
+	}
+
+	cursor, err := GameInfoCollection.Aggregate(ctx, pipeline)
+	if err != nil {
+		return err
+	}
+	type queryRes struct {
+		IGDBID int              `bson:"_id"`
+		Infos  []model.GameInfo `bson:"docs"`
+	}
+	var res []queryRes
+	if err = cursor.All(ctx, &res); err != nil {
+		return err
+	}
+	for _, item := range res {
+		gameIDs := make([]primitive.ObjectID, 0)
+		deleteInfoIDs := make([]primitive.ObjectID, 0)
+		for _, info := range item.Infos[1:] {
+			gameIDs = append(gameIDs, info.GameIDs...)
+			deleteInfoIDs = append(deleteInfoIDs, info.ID)
+		}
+		item.Infos[0].GameIDs = utils.Unique(append(item.Infos[0].GameIDs, gameIDs...))
+		err = DeleteGameInfosByIDs(deleteInfoIDs)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func MergeGameInfosWithSameName() error {
 	games, err := GetSameNameGameInfos()
 	if err != nil {
 		return err

server/handler/get_popular_game_infos.go

@@ -0,0 +1,106 @@
+package handler
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+	"pcgamedb/crawler"
+	"pcgamedb/db"
+	"pcgamedb/model"
+)
+
+type GetPopularGamesResponse struct {
+	Status  string            `json:"status"`
+	Message string            `json:"message,omitempty"`
+	Games   []*model.GameInfo `json:"games"`
+}
+
+// GetPopularGameInfosHandler Get popular games
+// @Summary Get popular games
+// @Description Get popular games based on a specified type
+// @Tags popular
+// @Accept json
+// @Produce json
+// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, v, steam-best-of-the-year, steam-most-played)"
+// @Success 200 {object} GetPopularGamesResponse
+// @Failure 400 {object} GetPopularGamesResponse
+// @Failure 500 {object} GetPopularGamesResponse
+// @Router /popular/{type} [get]
+func GetPopularGameInfosHandler(c *gin.Context) {
+	rankingType, exist := c.Params.Get("type")
+	if !exist {
+		c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
+			Status:  "error",
+			Message: "Missing ranking type",
+		})
+	}
+	popularityType := 0
+	var steam250Func func() ([]*model.GameInfo, error) = nil
+	switch rankingType {
+	case "igdb-most-visited":
+		popularityType = 1
+	case "igdb-most-wanted-to-play":
+		popularityType = 2
+	case "igdb-most-playing":
+		popularityType = 3
+	case "igdb-most-played":
+		popularityType = 4
+	case "steam-top":
+		steam250Func = crawler.GetSteam250Top250
+	case "steam-week-top":
+		steam250Func = crawler.GetSteam250WeekTop50
+	case "steam-best-of-the-year":
+		steam250Func = crawler.GetSteam250BestOfTheYear
+	case "steam-most-played":
+		steam250Func = crawler.GetSteam250MostPlayed
+	default:
+		c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
+			Status:  "error",
+			Message: "Invalid ranking type",
+		})
+	}
+	var infos []*model.GameInfo
+	var err error
+	if steam250Func != nil {
+		infos, err = steam250Func()
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, GetPopularGamesResponse{
+				Status:  "error",
+				Message: err.Error(),
+			})
+		}
+		infos = infos[:10]
+	} else {
+		offset := 0
+		for len(infos) < 10 {
+			ids, err := crawler.GetIGDBPopularGameIDs(popularityType, offset, 20)
+			if err != nil {
+				c.JSON(http.StatusInternalServerError, GetPopularGamesResponse{
+					Status:  "error",
+					Message: err.Error(),
+				})
+			}
+			offset += 20
+			pids := make([]int, 20)
+			for _, id := range ids {
+				pid, err := crawler.GetIGDBAppParentCache(id)
+				if err != nil {
+					continue
+				}
+				pids = append(pids, pid)
+			}
+			newInfos, err := db.GetGameInfosByPlatformIDs("igdb", pids)
+			if err != nil {
+				c.JSON(http.StatusInternalServerError, GetPopularGamesResponse{
+					Status:  "error",
+					Message: err.Error(),
+				})
+			}
+			infos = append(infos, newInfos...)
+		}
+		infos = infos[:10]
+	}
+	c.JSON(http.StatusOK, GetPopularGamesResponse{
+		Status: "ok",
+		Games:  infos,
+	})
+}

server/handler/get_ranking.go

@@ -1,66 +0,0 @@
-package handler
-
-import (
-	"net/http"
-
-	"pcgamedb/crawler"
-	"pcgamedb/model"
-
-	"github.com/gin-gonic/gin"
-)
-
-type GetRankingResponse struct {
-	Status  string            `json:"status"`
-	Message string            `json:"message,omitempty"`
-	Games   []*model.GameInfo `json:"games"`
-}
-
-// GetRanking retrieves game rankings.
-// @Summary Retrieve rankings
-// @Description Retrieves rankings based on a specified type
-// @Tags ranking
-// @Accept json
-// @Produce json
-// @Param type path string true "Ranking Type(top, week-top, best-of-the-year, most-played)"
-// @Success 200 {object} GetRankingResponse
-// @Failure 400 {object} GetRankingResponse
-// @Failure 500 {object} GetRankingResponse
-// @Router /ranking/{type} [get]
-func GetRankingHandler(c *gin.Context) {
-	rankingType, exist := c.Params.Get("type")
-	if !exist {
-		c.JSON(http.StatusBadRequest, GetRankingResponse{
-			Status:  "error",
-			Message: "Missing ranking type",
-		})
-	}
-	var f func() ([]*model.GameInfo, error)
-	switch rankingType {
-	case "top":
-		f = crawler.GetSteam250Top250Cache
-	case "week-top":
-		f = crawler.GetSteam250WeekTop50Cache
-	case "best-of-the-year":
-		f = crawler.GetSteam250BestOfTheYearCache
-	case "most-played":
-		f = crawler.GetSteam250MostPlayedCache
-	default:
-		c.JSON(http.StatusBadRequest, GetRankingResponse{
-			Status:  "error",
-			Message: "Invalid ranking type",
-		})
-		return
-	}
-	rank, err := f()
-	if err != nil {
-		c.JSON(http.StatusInternalServerError, GetRankingResponse{
-			Status:  "error",
-			Message: err.Error(),
-		})
-		return
-	}
-	c.JSON(http.StatusOK, GetRankingResponse{
-		Status: "ok",
-		Games:  rank,
-	})
-}

server/route.go

@@ -34,7 +34,7 @@ func initRoute(app *gin.Engine) {
 	GameInfoGroup.PUT("/update", middleware.Auth(), handler.UpdateGameInfoHandler)
 	GameInfoGroup.DELETE("/id/:id", middleware.Auth(), handler.DeleteGameInfoHandler)
 
-	app.GET("/ranking/:type", handler.GetRankingHandler)
+	app.GET("/popular/:type", handler.GetPopularGameInfosHandler)
 	app.GET("/healthcheck", handler.HealthCheckHandler)
 	app.GET("/author", handler.GetAllAuthorsHandler)
 	app.POST("/clean", middleware.Auth(), handler.CleanGameHandler)

task/clean.go

@@ -28,7 +28,7 @@ func Clean(logger *zap.Logger) {
 	for _, id := range ids {
 		logger.Info("Cleaned game info with empty game ids", zap.Any("game_id", id))
 	}
-	err = db.MergeSameNameGameInfos()
+	err = db.MergeGameInfosWithSameName()
 	if err != nil {
 		logger.Error("Failed to merge same name game infos", zap.Error(err))
 	}

utils/size.go

@@ -54,7 +54,7 @@ func BytesToSize(size uint64) string {
 		_         = iota
 		KB uint64 = 1 << (10 * iota)
 		MB
-		GBc
+		GB
 		TB
 	)
 	switch {