nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

fix: DeduplicateGameItems

Browse Source
This commit is contained in:
Nite07 2024-11-28 19:34:54 +08:00
parent 8702d3e93f
commit 156b8fbb65
4 changed files with 52 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
db/game.go
View File

@ -132,6 +132,34 @@ func SaveGameItem(item *model.GameItem) error {
return nil
}
func SaveGameItems(items []*model.GameItem) error {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
operations := make([]mongo.WriteModel, len(items))
for i, item := range items {
if item.ID.IsZero() {
item.ID = primitive.NewObjectID()
}
if item.CreatedAt.IsZero() {
item.CreatedAt = time.Now()
}
item.UpdatedAt = time.Now()
item.Size = strings.Replace(item.Size, "gb", "GB", -1)
item.Size = strings.Replace(item.Size, "mb", "MB", -1)
operations[i] = mongo.NewUpdateOneModel().
SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
SetUpdate(bson.D{{Key: "$set", Value: item}}).
SetUpsert(true)
}
opts := options.BulkWrite().SetOrdered(false)
_, err := GameItemCollection.BulkWrite(ctx, operations, opts)
if err != nil {
return err
}
return nil
}
func SaveGameInfo(item *model.GameInfo) error {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
@ -187,7 +215,7 @@ func SaveGameInfos(items []*model.GameInfo) error {
func GetAllGameItems() ([]*model.GameItem, error) {
var items []*model.GameItem
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
cursor, err := GameItemCollection.Find(ctx, bson.D{})
if err != nil {
@ -459,7 +487,10 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
defer cancel()
type queryRes struct {
ID string `bson:"_id"`
ID struct {
RawName string `bson:"raw_name"`
Download string `bson:"download"`
} `bson:"_id"`
Count int `bson:"count"`
IDs []primitive.ObjectID `bson:"ids"`
}
@ -467,17 +498,21 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
var res []primitive.ObjectID
pipeline := mongo.Pipeline{
bson.D{{Key: "$group", Value: bson.D{
{Key: "_id", Value: bson.D{
{Key: "raw_name", Value: "$raw_name"},
{Key: "download", Value: "$download"},
}},
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
}}},
bson.D{{Key: "$match", Value: bson.D{
{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}},
}}},
bson.D{
{Key: "$group",
Value: bson.D{
{Key: "_id",
Value: bson.D{
{Key: "raw_name", Value: "$raw_name"},
{Key: "download", Value: "$download"},
},
},
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
},
},
},
bson.D{{Key: "$match", Value: bson.D{{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}}}}},
}
var qres []queryRes

2
game_infos.json
View File

File diff suppressed because one or more lines are too long

2
games.json
View File

File diff suppressed because one or more lines are too long

4
task/clean.go
View File

@ -28,9 +28,9 @@ func Clean(logger *zap.Logger) {
for _, id := range ids {
logger.Info("Cleaned game info with empty game ids", zap.Any("game_id", id))
}
err = db.MergeGameInfosWithSameName()
err = db.MergeGameInfosWithSameIGDBID()
if err != nil {
logger.Error("Failed to merge same name game infos", zap.Error(err))
logger.Error("Failed to merge game infos with same igdb id", zap.Error(err))
}
logger.Info("Cleaning task completed")
}