fix: DeduplicateGameItems
This commit is contained in:
parent
8702d3e93f
commit
156b8fbb65
53
db/game.go
53
db/game.go
@ -132,6 +132,34 @@ func SaveGameItem(item *model.GameItem) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func SaveGameItems(items []*model.GameItem) error {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
operations := make([]mongo.WriteModel, len(items))
|
||||||
|
for i, item := range items {
|
||||||
|
if item.ID.IsZero() {
|
||||||
|
item.ID = primitive.NewObjectID()
|
||||||
|
}
|
||||||
|
if item.CreatedAt.IsZero() {
|
||||||
|
item.CreatedAt = time.Now()
|
||||||
|
}
|
||||||
|
item.UpdatedAt = time.Now()
|
||||||
|
item.Size = strings.Replace(item.Size, "gb", "GB", -1)
|
||||||
|
item.Size = strings.Replace(item.Size, "mb", "MB", -1)
|
||||||
|
operations[i] = mongo.NewUpdateOneModel().
|
||||||
|
SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
|
||||||
|
SetUpdate(bson.D{{Key: "$set", Value: item}}).
|
||||||
|
SetUpsert(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := options.BulkWrite().SetOrdered(false)
|
||||||
|
_, err := GameItemCollection.BulkWrite(ctx, operations, opts)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func SaveGameInfo(item *model.GameInfo) error {
|
func SaveGameInfo(item *model.GameInfo) error {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@ -187,7 +215,7 @@ func SaveGameInfos(items []*model.GameInfo) error {
|
|||||||
|
|
||||||
func GetAllGameItems() ([]*model.GameItem, error) {
|
func GetAllGameItems() ([]*model.GameItem, error) {
|
||||||
var items []*model.GameItem
|
var items []*model.GameItem
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
cursor, err := GameItemCollection.Find(ctx, bson.D{})
|
cursor, err := GameItemCollection.Find(ctx, bson.D{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -459,7 +487,10 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
type queryRes struct {
|
type queryRes struct {
|
||||||
ID string `bson:"_id"`
|
ID struct {
|
||||||
|
RawName string `bson:"raw_name"`
|
||||||
|
Download string `bson:"download"`
|
||||||
|
} `bson:"_id"`
|
||||||
Count int `bson:"count"`
|
Count int `bson:"count"`
|
||||||
IDs []primitive.ObjectID `bson:"ids"`
|
IDs []primitive.ObjectID `bson:"ids"`
|
||||||
}
|
}
|
||||||
@ -467,17 +498,21 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
|
|||||||
var res []primitive.ObjectID
|
var res []primitive.ObjectID
|
||||||
|
|
||||||
pipeline := mongo.Pipeline{
|
pipeline := mongo.Pipeline{
|
||||||
bson.D{{Key: "$group", Value: bson.D{
|
bson.D{
|
||||||
{Key: "_id", Value: bson.D{
|
{Key: "$group",
|
||||||
|
Value: bson.D{
|
||||||
|
{Key: "_id",
|
||||||
|
Value: bson.D{
|
||||||
{Key: "raw_name", Value: "$raw_name"},
|
{Key: "raw_name", Value: "$raw_name"},
|
||||||
{Key: "download", Value: "$download"},
|
{Key: "download", Value: "$download"},
|
||||||
}},
|
},
|
||||||
|
},
|
||||||
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
|
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
|
||||||
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
|
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
|
||||||
}}},
|
},
|
||||||
bson.D{{Key: "$match", Value: bson.D{
|
},
|
||||||
{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}},
|
},
|
||||||
}}},
|
bson.D{{Key: "$match", Value: bson.D{{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}}}}},
|
||||||
}
|
}
|
||||||
|
|
||||||
var qres []queryRes
|
var qres []queryRes
|
||||||
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -28,9 +28,9 @@ func Clean(logger *zap.Logger) {
|
|||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
logger.Info("Cleaned game info with empty game ids", zap.Any("game_id", id))
|
logger.Info("Cleaned game info with empty game ids", zap.Any("game_id", id))
|
||||||
}
|
}
|
||||||
err = db.MergeGameInfosWithSameName()
|
err = db.MergeGameInfosWithSameIGDBID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("Failed to merge same name game infos", zap.Error(err))
|
logger.Error("Failed to merge game infos with same igdb id", zap.Error(err))
|
||||||
}
|
}
|
||||||
logger.Info("Cleaning task completed")
|
logger.Info("Cleaning task completed")
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user