From 95f46cc7e75887115bdb5aade5eaafbcb238445c Mon Sep 17 00:00:00 2001 From: nite Date: Tue, 8 Apr 2025 12:58:14 +1000 Subject: [PATCH] u --- collector/store.go | 40 +------ collector/webhook.go | 104 +++++++++++++----- db/count.go | 137 ++++++++++++++++++++++++ db/db.go | 249 +++++++++++++++++++------------------------ db/game.go | 194 ++++++++++++++------------------- main.go | 93 ++++++++++++---- model/game.go | 2 - model/model.go | 36 +++---- 8 files changed, 495 insertions(+), 360 deletions(-) create mode 100644 db/count.go diff --git a/collector/store.go b/collector/store.go index 27cb903..ee17caf 100644 --- a/collector/store.go +++ b/collector/store.go @@ -2,7 +2,6 @@ package collector import ( "igdb-database/db" - "igdb-database/model" "log" "math" "sync" @@ -33,50 +32,13 @@ func FetchAndStore[T any]( defer wg.Done() defer func() { <-concurrence }() - // items data from igdb items, err := e.Paginated(uint64(i), 500) if err != nil { log.Printf("failed to get items from igdb %s: %v", e.GetEndpointName(), err) return } - type IdGetter interface { - GetId() uint64 - } - - ids := make([]uint64, 0, len(items)) - for _, item := range items { - if v, ok := any(item).(IdGetter); ok { - ids = append(ids, v.GetId()) - } else { - log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err) - return - } - } - - // items data from database - data, err := db.GetItemsByIGDBIDs[T](e.GetEndpointName(), ids) - if err != nil { - log.Printf("failed to get items from database %s: %v", e.GetEndpointName(), err) - return - } - - newItems := make([]*model.Item[T], 0, len(items)) - for _, item := range items { - v, ok := any(item).(IdGetter) - if !ok { - log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err) - return - } else { - if data[v.GetId()] == nil { - newItems = append(newItems, model.NewItem(item)) - } else { - data[v.GetId()].Item = item - newItems = append(newItems, data[v.GetId()]) - } - } - } - err = db.SaveItems(e.GetEndpointName(), newItems) + err = db.SaveItems(e.GetEndpointName(), items) if err != nil { log.Printf("failed to save games %s: %v", e.GetEndpointName(), err) return diff --git a/collector/webhook.go b/collector/webhook.go index 5506efa..155b634 100644 --- a/collector/webhook.go +++ b/collector/webhook.go @@ -6,7 +6,6 @@ import ( "fmt" "igdb-database/config" "igdb-database/db" - "igdb-database/model" "io" "log" "net" @@ -167,57 +166,110 @@ func webhook[T any]( log.Printf("failed to get %s: %v", e.GetEndpointName(), err) return } - oldItem, err := db.GetItemByIGDBID[T](e.GetEndpointName(), data.ID) + oldItem, err := db.GetItemById[T](e.GetEndpointName(), data.ID) if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { log.Printf("failed to get %s: %v", e.GetEndpointName(), err) return } - newItem := model.NewItem(item) - if oldItem != nil { - newItem.MId = oldItem.MId - } - err = db.SaveItem(e.GetEndpointName(), newItem) - if err != nil { - log.Printf("failed to save %s: %v", e.GetEndpointName(), err) - return - } if _, ok := any(e).(*endpoint.Games); ok { - game, err := db.GetItemByIGDBID[pb.Game](endpoint.EPGames, data.ID) - if err == nil { - g, err := db.ConvertGame(game.Item) - if err == nil { - g.MId = game.MId + if oldItem != nil { + // oldGame update + oldGame := any(oldItem).(*pb.Game) + // compare themes and genres + game := any(item).(*pb.Game) + + oldGameThemeIds := make([]uint64, 0, len(oldGame.Themes)) + for _, t := range oldGame.Themes { + oldGameThemeIds = append(oldGameThemeIds, t.Id) + } + gameThemeIds := make([]uint64, 0, len(game.Themes)) + for _, t := range game.Themes { + gameThemeIds = append(gameThemeIds, t.Id) + } + if !slices.Equal(oldGameThemeIds, gameThemeIds) { + for _, t := range oldGameThemeIds { + if !slices.Contains(gameThemeIds, t) { + _ = db.MinusThemeCount(t) + } + } + for _, t := range gameThemeIds { + if !slices.Contains(oldGameThemeIds, t) { + _ = db.AddThemeCount(t) + } + } + } + + oldGameGenreIds := make([]uint64, 0, len(oldGame.Genres)) + for _, t := range oldGame.Genres { + oldGameGenreIds = append(oldGameGenreIds, t.Id) + } + gameGenreIds := make([]uint64, 0, len(game.Genres)) + for _, t := range game.Genres { + gameGenreIds = append(gameGenreIds, t.Id) + } + if !slices.Equal(oldGameGenreIds, gameGenreIds) { + for _, t := range oldGameGenreIds { + if !slices.Contains(gameGenreIds, t) { + _ = db.MinusGenreCount(t) + } + } + for _, t := range gameGenreIds { + if !slices.Contains(oldGameGenreIds, t) { + _ = db.AddGenreCount(t) + } + } + } + + g, err := db.ConvertGame(game) + if err != nil { + log.Printf("failed to convert game: %v", err) + } else { + _ = db.SaveGame(g) + log.Printf("game %d aggregated", data.ID) + } + + } else { + // new game + game := any(item).(*pb.Game) + for _, t := range game.Themes { + _ = db.AddThemeCount(t.Id) + } + for _, t := range game.Genres { + _ = db.AddGenreCount(t.Id) + } + g, err := db.ConvertGame(game) + if err != nil { + log.Printf("failed to convert game: %v", err) + } else { _ = db.SaveGame(g) log.Printf("game %d aggregated", data.ID) } } } + err = db.SaveItem(e.GetEndpointName(), item) + if err != nil { + log.Printf("failed to save %s: %v", e.GetEndpointName(), err) + return + } + // update associated game type gameGetter interface { GetGame() *pb.Game } if v, ok := any(item).(gameGetter); ok { - game, err := db.GetItemByIGDBID[pb.Game](endpoint.EPGames, v.GetGame().Id) + game, err := db.GetItemById[pb.Game](endpoint.EPGames, v.GetGame().Id) if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { log.Printf("failed to get game: %v", err) goto END } - g, err := db.ConvertGame(game.Item) + g, err := db.ConvertGame(game) if err != nil { log.Printf("failed to convert game: %v", err) goto END } - oldGame, err := db.GetGameByIGDBID(game.Item.Id) - if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { - log.Printf("failed to get game: %v", err) - goto END - } - if oldGame != nil { - g.MId = oldGame.MId - } err = db.SaveGame(g) if err != nil { log.Printf("failed to save game: %v", err) diff --git a/db/count.go b/db/count.go new file mode 100644 index 0000000..e71a262 --- /dev/null +++ b/db/count.go @@ -0,0 +1,137 @@ +package db + +import ( + "context" + "fmt" + "time" + + "github.com/bestnite/go-igdb/endpoint" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +type Count struct { + Theme uint64 `json:"theme,omitempty"` + Genre uint64 `json:"genre,omitempty"` + Count int64 `json:"count,omitempty"` +} + +func GetCountByThemeId(themeId uint64) (*Count, error) { + var count Count + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := GetInstance().CountCollection.FindOne(ctx, bson.M{"theme": themeId}).Decode(&count) + if err != nil { + return nil, fmt.Errorf("failed to get count: %w", err) + } + return &count, nil +} + +func GetCountByGenreId(genreId uint64) (*Count, error) { + var count Count + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + err := GetInstance().CountCollection.FindOne(ctx, bson.M{"genre": genreId}).Decode(&count) + if err != nil { + return nil, fmt.Errorf("failed to get count: %w", err) + } + return &count, nil +} + +func SaveCount(count *Count) error { + var filter bson.M + if count.Genre != 0 { + filter = bson.M{"genre": count.Genre} + } + if count.Theme != 0 { + filter = bson.M{"theme": count.Theme} + } + update := bson.M{"$set": count} + opts := options.UpdateOne().SetUpsert(true) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func AddThemeCount(themeId uint64) error { + filter := bson.M{"theme": themeId} + update := bson.M{"$inc": bson.M{"count": 1}} + opts := options.UpdateOne().SetUpsert(true) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func AddGenreCount(genreId uint64) error { + filter := bson.M{"genre": genreId} + update := bson.M{"$inc": bson.M{"count": 1}} + opts := options.UpdateOne().SetUpsert(true) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func MinusThemeCount(themeId uint64) error { + filter := bson.M{"theme": themeId} + update := bson.M{"$inc": bson.M{"count": -1}} + opts := options.UpdateOne().SetUpsert(true) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func MinusGenreCount(genreId uint64) error { + filter := bson.M{"genre": genreId} + update := bson.M{"$inc": bson.M{"count": -1}} + opts := options.UpdateOne().SetUpsert(true) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func CountTheme(themeId uint64) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + filter := bson.M{"themes.id": themeId} + count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter) + if err != nil { + return 0, fmt.Errorf("failed to count theme: %w", err) + } + return count, nil +} + +func CountGenre(genreId uint64) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + filter := bson.M{"genres.id": genreId} + count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter) + if err != nil { + return 0, fmt.Errorf("failed to count genre: %w", err) + } + return count, nil +} diff --git a/db/db.go b/db/db.go index 86f1768..a277315 100644 --- a/db/db.go +++ b/db/db.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "igdb-database/config" - "igdb-database/model" "log" "sync" "time" @@ -21,9 +20,10 @@ var ( ) type MongoDB struct { - client *mongo.Client - Collections map[endpoint.Name]*mongo.Collection - GameCollection *mongo.Collection + client *mongo.Client + Collections map[endpoint.Name]*mongo.Collection + GameCollection *mongo.Collection + CountCollection *mongo.Collection } func GetInstance() *MongoDB { @@ -54,6 +54,7 @@ func GetInstance() *MongoDB { } instance.GameCollection = client.Database(config.C().Database.Database).Collection("game_details") + instance.CountCollection = client.Database(config.C().Database.Database).Collection("count") instance.createIndex() }) @@ -65,8 +66,8 @@ func (m *MongoDB) createIndex() { defer cancel() textIndexMap := map[endpoint.Name]string{ - endpoint.EPGames: "item.name", - endpoint.EPAlternativeNames: "item.name", + endpoint.EPGames: "name", + endpoint.EPAlternativeNames: "name", } for e, idx := range textIndexMap { @@ -81,23 +82,23 @@ func (m *MongoDB) createIndex() { } indexMap := map[endpoint.Name][]string{ - endpoint.EPAlternativeNames: {"item.game.id"}, - endpoint.EPArtworks: {"item.game.id"}, - endpoint.EPCollectionMemberships: {"item.game.id"}, - endpoint.EPCovers: {"item.game.id"}, - endpoint.EPExternalGames: {"item.game.id"}, - endpoint.EPGameEngines: {"item.game.id"}, - endpoint.EPGameLocalizations: {"item.game.id"}, - endpoint.EPGameVersions: {"item.game.id"}, - endpoint.EPGameVersionFeatureValues: {"item.game.id"}, - endpoint.EPGameVideos: {"item.game.id"}, - endpoint.EPInvolvedCompanies: {"item.game.id"}, - endpoint.EPLanguageSupports: {"item.game.id"}, - endpoint.EPMultiplayerModes: {"item.game.id"}, - endpoint.EPReleaseDates: {"item.game.id"}, - endpoint.EPScreenshots: {"item.game.id"}, - endpoint.EPWebsites: {"item.game.id"}, - endpoint.EPGames: {"item.parent_game.id", "item.version_parent.id"}, + endpoint.EPAlternativeNames: {"game.id"}, + endpoint.EPArtworks: {"game.id"}, + endpoint.EPCollectionMemberships: {"game.id"}, + endpoint.EPCovers: {"game.id"}, + endpoint.EPExternalGames: {"game.id"}, + endpoint.EPGameEngines: {"game.id"}, + endpoint.EPGameLocalizations: {"game.id"}, + endpoint.EPGameVersions: {"game.id"}, + endpoint.EPGameVersionFeatureValues: {"game.id"}, + endpoint.EPGameVideos: {"game.id"}, + endpoint.EPInvolvedCompanies: {"game.id"}, + endpoint.EPLanguageSupports: {"game.id"}, + endpoint.EPMultiplayerModes: {"game.id"}, + endpoint.EPReleaseDates: {"game.id"}, + endpoint.EPScreenshots: {"game.id"}, + endpoint.EPWebsites: {"game.id"}, + endpoint.EPGames: {"parent_game.id", "version_parent.id"}, } for e, idxes := range indexMap { @@ -119,11 +120,12 @@ func (m *MongoDB) createIndex() { } _, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{ Keys: bson.D{ - {Key: "item.id", Value: 1}, + {Key: "id", Value: 1}, }, + Options: options.Index().SetUnique(true), }) if err != nil { - log.Printf("failed to create index item.id for %s: %v", string(e), err) + log.Printf("failed to create index id for %s: %v", string(e), err) } } @@ -131,62 +133,14 @@ func (m *MongoDB) createIndex() { Keys: bson.D{ {Key: "id", Value: 1}, }, + Options: options.Index().SetUnique(true), }) if err != nil { log.Printf("failed to create index id for game_details: %v", err) } } -func SaveItem[T any](e endpoint.Name, item *model.Item[T]) error { - if item.MId.IsZero() { - item.MId = bson.NewObjectID() - } - filter := bson.M{"_id": item.MId} - update := bson.M{"$set": item} - opts := options.UpdateOne().SetUpsert(true) - - coll := GetInstance().Collections[e] - if coll == nil { - return fmt.Errorf("collection not found") - } - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := coll.UpdateOne(ctx, filter, update, opts) - if err != nil { - return fmt.Errorf("failed to save item to %s: %w", string(e), err) - } - return nil -} - -func SaveItems[T any](e endpoint.Name, items []*model.Item[T]) error { - var models []mongo.WriteModel - - for _, item := range items { - if item.MId.IsZero() { - item.MId = bson.NewObjectID() - } - filter := bson.M{"_id": item.MId} - update := bson.M{"$set": item} - oneModel := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true) - models = append(models, oneModel) - } - - coll := GetInstance().Collections[e] - if coll == nil { - return fmt.Errorf("collection not found") - } - - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond) - defer cancel() - _, err := coll.BulkWrite(ctx, models) - if err != nil { - return fmt.Errorf("failed to save items in bulk %s: %w", string(e), err) - } - return nil -} - -func CountItems(e endpoint.Name) (int64, error) { +func EstimatedDocumentCount(e endpoint.Name) (int64, error) { coll := GetInstance().Collections[e] if coll == nil { return 0, fmt.Errorf("collection not found") @@ -195,66 +149,12 @@ func CountItems(e endpoint.Name) (int64, error) { defer cancel() count, err := coll.EstimatedDocumentCount(ctx) if err != nil { - return 0, fmt.Errorf("failed to count items %s: %w", string(e), err) + return 0, fmt.Errorf("failed to count %s: %w", string(e), err) } return count, nil } -func GetItemByIGDBID[T any](e endpoint.Name, id uint64) (*model.Item[T], error) { - var item model.Item[T] - coll := GetInstance().Collections[e] - if coll == nil { - return nil, fmt.Errorf("collection not found") - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item) - if err != nil { - return nil, fmt.Errorf("failed to get item %s: %w", string(e), err) - } - return &item, nil -} - -func GetItemsByIGDBIDs[T any](e endpoint.Name, ids []uint64) (map[uint64]*model.Item[T], error) { - if len(ids) == 0 { - return nil, nil - } - - coll := GetInstance().Collections[e] - if coll == nil { - return nil, fmt.Errorf("collection not found") - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}}) - if err != nil { - return nil, fmt.Errorf("failed to get items %s: %w", string(e), err) - } - - type IdGetter interface { - GetId() uint64 - } - - ctx, cancel = context.WithTimeout(context.Background(), time.Duration(len(ids))*200*time.Millisecond) - defer cancel() - res := make(map[uint64]*model.Item[T]) - for cursor.Next(ctx) { - item := model.Item[T]{} - err := cursor.Decode(&item) - if err != nil { - return nil, fmt.Errorf("failed to decode item %s: %w", string(e), err) - } - if v, ok := any(item.Item).(IdGetter); ok { - res[v.GetId()] = &item - } else { - return nil, fmt.Errorf("failed to get id from item %s: %w", string(e), err) - } - } - - return res, nil -} - -func RemoveItemsByID(e endpoint.Name, ids []bson.ObjectID) error { +func RemoveByID(e endpoint.Name, ids []bson.ObjectID) error { coll := GetInstance().Collections[e] if coll == nil { return fmt.Errorf("collection not found") @@ -269,25 +169,98 @@ func RemoveItemsByID(e endpoint.Name, ids []bson.ObjectID) error { return nil } -func GetItemsPaginated[T any](e endpoint.Name, offset int64, limit int64) ([]*model.Item[T], error) { - +func GetItemsByIds[T any](e endpoint.Name, ids []uint64) ([]*T, error) { coll := GetInstance().Collections[e] if coll == nil { return nil, fmt.Errorf("collection not found") } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second+time.Duration(len(ids)*200)*time.Millisecond) defer cancel() - cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}})) + cursor, err := coll.Find(ctx, bson.M{"id": bson.M{"$in": ids}}) + if err != nil { + return nil, fmt.Errorf("failed to get items: %w", err) + } + + var items []*T + err = cursor.All(ctx, &items) + if err != nil { + return nil, fmt.Errorf("failed to get items: %w", err) + } + return items, nil +} + +func GetItemById[T any](e endpoint.Name, id uint64) (*T, error) { + coll := GetInstance().Collections[e] + if coll == nil { + return nil, fmt.Errorf("collection not found") + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var item *T + err := coll.FindOne(ctx, bson.M{"id": id}).Decode(&item) + if err != nil { + return nil, fmt.Errorf("failed to get item: %w", err) + } + return item, nil +} + +func SaveItem[T any](e endpoint.Name, item *T) error { + type IdGetter interface { + GetId() uint64 + } + id := any(item).(IdGetter).GetId() + filter := bson.M{"id": id} + update := bson.M{"$set": item} + opts := options.UpdateOne().SetUpsert(true) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := GetInstance().Collections[e].UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func SaveItems[T any](e endpoint.Name, items []*T) error { + type IdGetter interface { + GetId() uint64 + } + + updateModel := make([]mongo.WriteModel, 0, len(items)) + for _, item := range items { + updateModel = append(updateModel, mongo.NewUpdateOneModel().SetFilter(bson.M{"id": any(item).(IdGetter).GetId()}).SetUpdate(bson.M{"$set": item}).SetUpsert(true)) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond) + defer cancel() + _, err := GetInstance().Collections[e].BulkWrite(ctx, updateModel) + if err != nil { + return err + } + return nil +} + +func GetItemsPaginated[T any](e endpoint.Name, skip int64, limit int64) ([]*T, error) { + coll := GetInstance().Collections[e] + if coll == nil { + return nil, fmt.Errorf("collection not found") + } + + opts := options.Find().SetSort(bson.M{"id": 1}).SetSkip(skip).SetLimit(limit) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*time.Millisecond*200) + defer cancel() + cursor, err := coll.Find(ctx, bson.M{}, opts) if err != nil { return nil, fmt.Errorf("failed to get items %s: %w", string(e), err) } - var items []*model.Item[T] + var items []*T err = cursor.All(ctx, &items) if err != nil { - return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err) + return nil, fmt.Errorf("failed to get items %s: %w", string(e), err) } - return items, nil } diff --git a/db/game.go b/db/game.go index 202c0b6..f9b8325 100644 --- a/db/game.go +++ b/db/game.go @@ -41,10 +41,7 @@ func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) { } func SaveGame(game *model.Game) error { - if game.MId.IsZero() { - game.MId = bson.NewObjectID() - } - filter := bson.M{"_id": game.MId} + filter := bson.M{"id": game.Id} update := bson.M{"$set": game} opts := options.UpdateOne().SetUpsert(true) @@ -66,14 +63,11 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.AgeRatings { ageRatingsIds = append(ageRatingsIds, g.Id) } - ageRatings, err := GetItemsByIGDBIDs[pb.AgeRating](endpoint.EPAgeRatings, ageRatingsIds) + ageRatings, err := GetItemsByIds[pb.AgeRating](endpoint.EPAgeRatings, ageRatingsIds) if err != nil { return nil, err } - res.AgeRatings = make([]*pb.AgeRating, 0, len(ageRatings)) - for _, item := range ageRatings { - res.AgeRatings = append(res.AgeRatings, item.Item) - } + res.AgeRatings = ageRatings res.AggregatedRating = game.AggregatedRating res.AggregatedRatingCount = game.AggregatedRatingCount @@ -82,27 +76,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.AlternativeNames { alternativeNameIds = append(alternativeNameIds, g.Id) } - alternativeNames, err := GetItemsByIGDBIDs[pb.AlternativeName](endpoint.EPAlternativeNames, alternativeNameIds) + alternativeNames, err := GetItemsByIds[pb.AlternativeName](endpoint.EPAlternativeNames, alternativeNameIds) if err != nil { return nil, err } - res.AlternativeNames = make([]*pb.AlternativeName, 0, len(alternativeNames)) - for _, item := range alternativeNames { - res.AlternativeNames = append(res.AlternativeNames, item.Item) - } + res.AlternativeNames = alternativeNames ArtworkIds := make([]uint64, 0, len(game.Artworks)) for _, g := range game.Artworks { ArtworkIds = append(ArtworkIds, g.Id) } - artworks, err := GetItemsByIGDBIDs[pb.Artwork](endpoint.EPArtworks, ArtworkIds) + artworks, err := GetItemsByIds[pb.Artwork](endpoint.EPArtworks, ArtworkIds) if err != nil { return nil, err } - res.Artworks = make([]*pb.Artwork, 0, len(artworks)) - for _, item := range artworks { - res.Artworks = append(res.Artworks, item.Item) - } + res.Artworks = artworks bundlesIds := make([]uint64, 0, len(game.Bundles)) for _, g := range game.Bundles { @@ -112,12 +100,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { if game.Cover != nil { coverId := game.Cover.Id - cover, err := GetItemByIGDBID[pb.Cover](endpoint.EPCovers, coverId) + cover, err := GetItemById[pb.Cover](endpoint.EPCovers, coverId) if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } if cover != nil { - res.Cover = cover.Item + res.Cover = cover } } @@ -139,14 +127,11 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.ExternalGames { externalGameIds = append(externalGameIds, g.Id) } - externalGames, err := GetItemsByIGDBIDs[pb.ExternalGame](endpoint.EPExternalGames, externalGameIds) + externalGames, err := GetItemsByIds[pb.ExternalGame](endpoint.EPExternalGames, externalGameIds) if err != nil { return nil, err } - res.ExternalGames = make([]*pb.ExternalGame, 0, len(externalGames)) - for _, item := range externalGames { - res.ExternalGames = append(res.ExternalGames, item.Item) - } + res.ExternalGames = externalGames res.FirstReleaseDate = game.FirstReleaseDate @@ -154,12 +139,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { if game.Franchise != nil { franchiseId := game.Franchise.Id - franchise, err := GetItemByIGDBID[pb.Franchise](endpoint.EPFranchises, franchiseId) + franchise, err := GetItemById[pb.Franchise](endpoint.EPFranchises, franchiseId) if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { return nil, err } if franchise != nil { - res.Franchise = franchise.Item + res.Franchise = franchise } } @@ -167,53 +152,41 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.Franchises { franchiseIds = append(franchiseIds, g.Id) } - franchises, err := GetItemsByIGDBIDs[pb.Franchise](endpoint.EPFranchises, franchiseIds) + franchises, err := GetItemsByIds[pb.Franchise](endpoint.EPFranchises, franchiseIds) if err != nil { return nil, err } - res.Franchises = make([]*pb.Franchise, 0, len(franchises)) - for _, item := range franchises { - res.Franchises = append(res.Franchises, item.Item) - } + res.Franchises = franchises gameEngineIds := make([]uint64, 0, len(game.GameEngines)) for _, g := range game.GameEngines { gameEngineIds = append(gameEngineIds, g.Id) } - gameEngines, err := GetItemsByIGDBIDs[pb.GameEngine](endpoint.EPGameEngines, gameEngineIds) + gameEngines, err := GetItemsByIds[pb.GameEngine](endpoint.EPGameEngines, gameEngineIds) if err != nil { return nil, err } - res.GameEngines = make([]*pb.GameEngine, 0, len(gameEngines)) - for _, item := range gameEngines { - res.GameEngines = append(res.GameEngines, item.Item) - } + res.GameEngines = gameEngines gameModeIds := make([]uint64, 0, len(game.GameModes)) for _, g := range game.GameModes { gameModeIds = append(gameModeIds, g.Id) } - gameModes, err := GetItemsByIGDBIDs[pb.GameMode](endpoint.EPGameModes, gameModeIds) + gameModes, err := GetItemsByIds[pb.GameMode](endpoint.EPGameModes, gameModeIds) if err != nil { return nil, err } - res.GameModes = make([]*pb.GameMode, 0, len(gameModes)) - for _, item := range gameModes { - res.GameModes = append(res.GameModes, item.Item) - } + res.GameModes = gameModes genreIds := make([]uint64, 0, len(game.Genres)) for _, g := range game.Genres { genreIds = append(genreIds, g.Id) } - genres, err := GetItemsByIGDBIDs[pb.Genre](endpoint.EPGenres, genreIds) + genres, err := GetItemsByIds[pb.Genre](endpoint.EPGenres, genreIds) if err != nil { return nil, err } - res.Genres = make([]*pb.Genre, 0, len(genres)) - for _, item := range genres { - res.Genres = append(res.Genres, item.Item) - } + res.Genres = genres res.Hypes = game.Hypes @@ -221,40 +194,31 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.InvolvedCompanies { involvedCompanyIds = append(involvedCompanyIds, g.Id) } - involvedCompanies, err := GetItemsByIGDBIDs[pb.InvolvedCompany](endpoint.EPInvolvedCompanies, involvedCompanyIds) + involvedCompanies, err := GetItemsByIds[pb.InvolvedCompany](endpoint.EPInvolvedCompanies, involvedCompanyIds) if err != nil { return nil, err } - res.InvolvedCompanies = make([]*pb.InvolvedCompany, 0, len(involvedCompanies)) - for _, item := range involvedCompanies { - res.InvolvedCompanies = append(res.InvolvedCompanies, item.Item) - } + res.InvolvedCompanies = involvedCompanies keywordIds := make([]uint64, 0, len(game.Keywords)) for _, g := range game.Keywords { keywordIds = append(keywordIds, g.Id) } - keyword, err := GetItemsByIGDBIDs[pb.Keyword](endpoint.EPKeywords, keywordIds) + keyword, err := GetItemsByIds[pb.Keyword](endpoint.EPKeywords, keywordIds) if err != nil { return nil, err } - res.Keywords = make([]*pb.Keyword, 0, len(keyword)) - for _, item := range keyword { - res.Keywords = append(res.Keywords, item.Item) - } + res.Keywords = keyword multiplayerModeIds := make([]uint64, 0, len(game.MultiplayerModes)) for _, g := range game.MultiplayerModes { multiplayerModeIds = append(multiplayerModeIds, g.Id) } - multiplayerModes, err := GetItemsByIGDBIDs[pb.MultiplayerMode](endpoint.EPMultiplayerModes, multiplayerModeIds) + multiplayerModes, err := GetItemsByIds[pb.MultiplayerMode](endpoint.EPMultiplayerModes, multiplayerModeIds) if err != nil { return nil, err } - res.MultiplayerModes = make([]*pb.MultiplayerMode, 0, len(multiplayerModes)) - for _, item := range multiplayerModes { - res.MultiplayerModes = append(res.MultiplayerModes, item.Item) - } + res.MultiplayerModes = multiplayerModes res.Name = game.Name @@ -266,27 +230,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.Platforms { platformIds = append(platformIds, g.Id) } - platforms, err := GetItemsByIGDBIDs[pb.Platform](endpoint.EPPlatforms, platformIds) + platforms, err := GetItemsByIds[pb.Platform](endpoint.EPPlatforms, platformIds) if err != nil { return nil, err } - res.Platforms = make([]*pb.Platform, 0, len(platforms)) - for _, item := range platforms { - res.Platforms = append(res.Platforms, item.Item) - } + res.Platforms = platforms playerPerspectiveIds := make([]uint64, 0, len(game.PlayerPerspectives)) for _, g := range game.PlayerPerspectives { playerPerspectiveIds = append(playerPerspectiveIds, g.Id) } - playerPerspectives, err := GetItemsByIGDBIDs[pb.PlayerPerspective](endpoint.EPPlayerPerspectives, playerPerspectiveIds) + playerPerspectives, err := GetItemsByIds[pb.PlayerPerspective](endpoint.EPPlayerPerspectives, playerPerspectiveIds) if err != nil { return nil, err } - res.PlayerPerspectives = make([]*pb.PlayerPerspective, 0, len(playerPerspectives)) - for _, item := range playerPerspectives { - res.PlayerPerspectives = append(res.PlayerPerspectives, item.Item) - } + res.PlayerPerspectives = playerPerspectives res.Rating = game.Rating res.RatingCount = game.RatingCount @@ -295,27 +253,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.ReleaseDates { releaseDateIds = append(releaseDateIds, g.Id) } - releaseDates, err := GetItemsByIGDBIDs[pb.ReleaseDate](endpoint.EPReleaseDates, releaseDateIds) + releaseDates, err := GetItemsByIds[pb.ReleaseDate](endpoint.EPReleaseDates, releaseDateIds) if err != nil { return nil, err } - res.ReleaseDates = make([]*pb.ReleaseDate, 0, len(releaseDates)) - for _, item := range releaseDates { - res.ReleaseDates = append(res.ReleaseDates, item.Item) - } + res.ReleaseDates = releaseDates screenshotIds := make([]uint64, 0, len(game.Screenshots)) for _, g := range game.Screenshots { screenshotIds = append(screenshotIds, g.Id) } - screenshots, err := GetItemsByIGDBIDs[pb.Screenshot](endpoint.EPScreenshots, screenshotIds) + screenshots, err := GetItemsByIds[pb.Screenshot](endpoint.EPScreenshots, screenshotIds) if err != nil { return nil, err } - res.Screenshots = make([]*pb.Screenshot, 0, len(screenshots)) - for _, item := range screenshots { - res.Screenshots = append(res.Screenshots, item.Item) - } + res.Screenshots = screenshots similarGamesIds := make([]uint64, 0, len(game.SimilarGames)) for _, g := range game.SimilarGames { @@ -340,14 +292,11 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.Themes { themeIds = append(themeIds, g.Id) } - themes, err := GetItemsByIGDBIDs[pb.Theme](endpoint.EPThemes, themeIds) + themes, err := GetItemsByIds[pb.Theme](endpoint.EPThemes, themeIds) if err != nil { return nil, err } - res.Themes = make([]*pb.Theme, 0, len(themes)) - for _, item := range themes { - res.Themes = append(res.Themes, item.Item) - } + res.Themes = themes res.TotalRating = game.TotalRating res.TotalRatingCount = game.TotalRatingCount @@ -366,27 +315,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.Videos { videoIds = append(videoIds, g.Id) } - videos, err := GetItemsByIGDBIDs[pb.GameVideo](endpoint.EPGameVideos, videoIds) + videos, err := GetItemsByIds[pb.GameVideo](endpoint.EPGameVideos, videoIds) if err != nil { return nil, err } - res.Videos = make([]*pb.GameVideo, 0, len(videos)) - for _, item := range videos { - res.Videos = append(res.Videos, item.Item) - } + res.Videos = videos websiteIds := make([]uint64, 0, len(game.Websites)) for _, g := range game.Websites { websiteIds = append(websiteIds, g.Id) } - websites, err := GetItemsByIGDBIDs[pb.Website](endpoint.EPWebsites, websiteIds) + websites, err := GetItemsByIds[pb.Website](endpoint.EPWebsites, websiteIds) if err != nil { return nil, err } - res.Websites = make([]*pb.Website, 0, len(websites)) - for _, item := range websites { - res.Websites = append(res.Websites, item.Item) - } + res.Websites = websites remakesIds := make([]uint64, 0, len(game.Remakes)) for _, g := range game.Remakes { @@ -422,40 +365,31 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { for _, g := range game.LanguageSupports { languageSupportIds = append(languageSupportIds, g.Id) } - languageSupports, err := GetItemsByIGDBIDs[pb.LanguageSupport](endpoint.EPLanguageSupports, languageSupportIds) + languageSupports, err := GetItemsByIds[pb.LanguageSupport](endpoint.EPLanguageSupports, languageSupportIds) if err != nil { return nil, err } - res.LanguageSupports = make([]*pb.LanguageSupport, 0, len(languageSupports)) - for _, item := range languageSupports { - res.LanguageSupports = append(res.LanguageSupports, item.Item) - } + res.LanguageSupports = languageSupports gameLocalizationIds := make([]uint64, 0, len(game.GameLocalizations)) for _, g := range game.GameLocalizations { gameLocalizationIds = append(gameLocalizationIds, g.Id) } - gameLocalizations, err := GetItemsByIGDBIDs[pb.GameLocalization](endpoint.EPGameLocalizations, gameLocalizationIds) + gameLocalizations, err := GetItemsByIds[pb.GameLocalization](endpoint.EPGameLocalizations, gameLocalizationIds) if err != nil { return nil, err } - res.GameLocalizations = make([]*pb.GameLocalization, 0, len(gameLocalizations)) - for _, item := range gameLocalizations { - res.GameLocalizations = append(res.GameLocalizations, item.Item) - } + res.GameLocalizations = gameLocalizations collectionIds := make([]uint64, 0, len(game.Collections)) for _, g := range game.Collections { collectionIds = append(collectionIds, g.Id) } - collections, err := GetItemsByIGDBIDs[pb.Collection](endpoint.EPCollections, collectionIds) + collections, err := GetItemsByIds[pb.Collection](endpoint.EPCollections, collectionIds) if err != nil { return nil, err } - res.Collections = make([]*pb.Collection, 0, len(collections)) - for _, item := range collections { - res.Collections = append(res.Collections, item.Item) - } + res.Collections = collections res.GameStatus = nil res.GameType = nil @@ -463,13 +397,13 @@ func ConvertGame(game *pb.Game) (*model.Game, error) { res.AllNames = make([]string, 0, len(alternativeNames)+1) res.AllNames = append(res.AllNames, game.Name) for _, item := range alternativeNames { - res.AllNames = append(res.AllNames, item.Item.Name) + res.AllNames = append(res.AllNames, item.Name) } return res, nil } -func GetGameByIGDBID(id uint64) (*model.Game, error) { +func GetGameById(id uint64) (*model.Game, error) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -480,3 +414,33 @@ func GetGameByIGDBID(id uint64) (*model.Game, error) { } return &game, nil } + +func GetAllItemsIDs[T any](e endpoint.Name) ([]uint64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var ids []uint64 + coll := GetInstance().Collections[e] + if coll == nil { + return nil, fmt.Errorf("collection not found") + } + cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1})) + if err != nil { + return nil, fmt.Errorf("failed to get items: %w", err) + } + + type IdGetter interface { + GetId() uint64 + } + + for cursor.Next(ctx) { + var item *T + err := cursor.Decode(&item) + if err != nil { + return nil, fmt.Errorf("failed to decode item: %w", err) + } + ids = append(ids, any(item).(IdGetter).GetId()) + } + + return ids, nil +} diff --git a/main.go b/main.go index cf25072..37176ce 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "errors" "flag" "igdb-database/collector" "igdb-database/config" @@ -13,7 +12,6 @@ import ( "github.com/bestnite/go-igdb" "github.com/bestnite/go-igdb/endpoint" pb "github.com/bestnite/go-igdb/proto" - "go.mongodb.org/mongo-driver/v2/mongo" ) var ( @@ -22,6 +20,7 @@ var ( enableReFetch = flag.Bool("re-fetch", false, "re fetch data even if collection is not empty") enableReAggregate = flag.Bool("re-aggregate", false, "re aggregate games even if game_details is not empty") enableWebhook = flag.Bool("webhook", true, "start webhook server") + onlyRefetchGames = flag.Bool("only-refetch-games", false, "only refetch games") ) func main() { @@ -35,6 +34,8 @@ func main() { log.Printf("data fetched") } + Count() + if *enableAggregate || *enableReAggregate { log.Printf("aggregating games") aggregateGames() @@ -48,10 +49,11 @@ func main() { } func aggregateGames() { - total, err := db.CountItems(endpoint.EPGames) + total, err := db.EstimatedDocumentCount(endpoint.EPGames) if err != nil { log.Fatalf("failed to count games: %v", err) } + log.Printf("games length: %d", total) finished := int64(0) wg := sync.WaitGroup{} @@ -71,39 +73,28 @@ func aggregateGames() { if err != nil { log.Fatalf("failed to get games: %v", err) } - games := make([]*pb.Game, 0, len(items)) - for _, item := range items { - games = append(games, item.Item) - } - isAggregated := make(map[uint64]bool, len(games)) + isAggregated := make(map[uint64]bool, len(items)) if !*enableReAggregate { - isAggregated, err = db.IsGamesAggregated(games) + isAggregated, err = db.IsGamesAggregated(items) if err != nil { log.Fatalf("failed to check if games are aggregated: %v", err) } } else { - for _, game := range games { + for _, game := range items { isAggregated[game.Id] = false } } for _, item := range items { - if isAggregated[item.Item.Id] { + if isAggregated[item.Id] { p := atomic.AddInt64(&finished, 1) log.Printf("game aggregated %d/%d", p, total) continue } - game, err := db.ConvertGame(item.Item) + game, err := db.ConvertGame(item) if err != nil { log.Fatalf("failed to convert game: %v", err) } - oldGame, err := db.GetGameByIGDBID(item.Item.Id) - if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { - log.Fatalf("failed to get game: %v", err) - } - if oldGame != nil { - game.MId = oldGame.MId - } err = db.SaveGame(game) if err != nil { @@ -120,14 +111,76 @@ func aggregateGames() { func fetchAndStore[T any]( e endpoint.EntityEndpoint[T], ) { - if count, err := db.CountItems(e.GetEndpointName()); (err == nil && count == 0) || *enableReFetch { + if count, err := db.EstimatedDocumentCount(e.GetEndpointName()); (err == nil && count == 0) || *enableReFetch { collector.FetchAndStore(e) } else if err != nil { log.Printf("failed to count items: %v", err) } } +func Count() { + ids, err := db.GetAllItemsIDs[pb.Theme](endpoint.EPThemes) + if err != nil { + log.Fatalf("failed to get all items ids %s: %v", endpoint.EPThemes, err) + } + + concurrence := make(chan struct{}, 10) + defer close(concurrence) + wg := sync.WaitGroup{} + + for _, id := range ids { + concurrence <- struct{}{} + wg.Add(1) + go func(id uint64) { + defer func() { + <-concurrence + wg.Done() + }() + count, err := db.CountTheme(id) + if err != nil { + log.Fatalf("failed to count theme: %v", err) + } + log.Printf("theme %d count: %d", id, count) + err = db.SaveCount(&db.Count{Theme: id, Count: count}) + if err != nil { + log.Fatalf("failed to save count: %v", err) + } + }(id) + } + + ids, err = db.GetAllItemsIDs[pb.Genre](endpoint.EPGenres) + if err != nil { + log.Fatalf("failed to get all items ids %s: %v", endpoint.EPGenres, err) + } + for _, id := range ids { + concurrence <- struct{}{} + wg.Add(1) + go func(id uint64) { + defer func() { + <-concurrence + wg.Done() + }() + count, err := db.CountGenre(id) + if err != nil { + log.Fatalf("failed to count genre: %v", err) + } + log.Printf("genre %d count: %d", id, count) + err = db.SaveCount(&db.Count{Genre: id, Count: count}) + if err != nil { + log.Fatalf("failed to save count: %v", err) + } + }(id) + } + + wg.Wait() +} + func allFetchAndStore(client *igdb.Client) { + if *onlyRefetchGames { + fetchAndStore(client.Games) + return + } + fetchAndStore(client.AgeRatingCategories) fetchAndStore(client.AgeRatingContentDescriptions) fetchAndStore(client.AgeRatingContentDescriptionsV2) diff --git a/model/game.go b/model/game.go index d9780b2..a66c330 100644 --- a/model/game.go +++ b/model/game.go @@ -2,7 +2,6 @@ package model import ( pb "github.com/bestnite/go-igdb/proto" - "go.mongodb.org/mongo-driver/v2/bson" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -10,7 +9,6 @@ type GameIds []uint64 type GameId uint64 type Game struct { - MId bson.ObjectID `json:"_id,omitempty"` Id uint64 `json:"id,omitempty"` AgeRatings []*pb.AgeRating `json:"age_ratings,omitempty"` AggregatedRating float64 `json:"aggregated_rating,omitempty"` diff --git a/model/model.go b/model/model.go index 3e5dfc7..2214973 100644 --- a/model/model.go +++ b/model/model.go @@ -1,24 +1,20 @@ package model -import ( - "go.mongodb.org/mongo-driver/v2/bson" -) +// type Item[T any] struct { +// Item *T `bson:"item"` +// MId bson.ObjectID `bson:"_id"` +// } -type Item[T any] struct { - Item *T `bson:"item"` - MId bson.ObjectID `bson:"_id"` -} +// func NewItem[T any](item *T) *Item[T] { +// return &Item[T]{ +// Item: item, +// } +// } -func NewItem[T any](item *T) *Item[T] { - return &Item[T]{ - Item: item, - } -} - -func NewItems[T any](items []*T) []*Item[T] { - var result []*Item[T] - for _, item := range items { - result = append(result, NewItem(item)) - } - return result -} +// func NewItems[T any](items []*T) []*Item[T] { +// var result []*Item[T] +// for _, item := range items { +// result = append(result, NewItem(item)) +// } +// return result +// }