From 4ce17cefb8320a555702de51ed3ef0f44f48afe2 Mon Sep 17 00:00:00 2001 From: nite Date: Sun, 6 Apr 2025 22:56:12 +1000 Subject: [PATCH] u --- collector/store.go | 9 +- collector/webhook.go | 45 +++-- db/db.go | 58 ++++++- db/game.go | 383 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 8 - main.go | 66 ++++++++ model/game.go | 71 ++++++++ 8 files changed, 618 insertions(+), 24 deletions(-) create mode 100644 db/game.go create mode 100644 model/game.go diff --git a/collector/store.go b/collector/store.go index d5ee0be..d7131ef 100644 --- a/collector/store.go +++ b/collector/store.go @@ -66,9 +66,12 @@ func FetchAndStore[T any]( log.Printf("failed to get id from item: %v", err) return } else { - n := model.NewItem(item) - n.MId = data[v.GetId()].MId - newItems = append(newItems, n) + if data[v.GetId()] == nil { + newItems = append(newItems, model.NewItem(item)) + } else { + data[v.GetId()].Item = item + newItems = append(newItems, data[v.GetId()]) + } } } err = db.SaveItems(e.GetEndpointName(), newItems) diff --git a/collector/webhook.go b/collector/webhook.go index cd8d338..8c21c99 100644 --- a/collector/webhook.go +++ b/collector/webhook.go @@ -13,6 +13,8 @@ import ( "net/url" "slices" + pb "github.com/bestnite/go-igdb/proto" + "github.com/bestnite/go-igdb" "github.com/bestnite/go-igdb/endpoint" "go.mongodb.org/mongo-driver/v2/mongo" @@ -111,9 +113,7 @@ func StartWebhookServer(client *igdb.Client) { } log.Printf("webhook \"%s\" registered", endp) } - if err != nil { - log.Fatalf("failed to active webhook \"%s\": %v", endpoint.EPGames, err) - } + log.Printf("all webhook registered") } http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -123,6 +123,7 @@ func StartWebhookServer(client *igdb.Client) { } }) + log.Printf("starting webhook server on %s", config.C().Address) err = http.ListenAndServe(config.C().Address, nil) if err != nil { log.Fatalf("failed to start webhook server: %v", err) @@ -138,35 +139,31 @@ func webhook[T any]( w.WriteHeader(401) return } + w.WriteHeader(200) data := struct { ID uint64 `json:"id"` }{} jsonBytes, err := io.ReadAll(r.Body) if err != nil { log.Printf("failed to read request body: %v", err) - w.WriteHeader(500) return } err = json.Unmarshal(jsonBytes, &data) if err != nil { log.Printf("failed to unmarshal request body: %v", err) - w.WriteHeader(500) return } if data.ID == 0 { - w.WriteHeader(400) return } item, err := e.GetByID(data.ID) if err != nil { log.Printf("failed to get %s: %v", e.GetEndpointName(), err) - w.WriteHeader(500) return } oldItem, err := db.GetItemByIGDBID[T](e.GetEndpointName(), data.ID) if err != nil && err != mongo.ErrNoDocuments { log.Printf("failed to get %s: %v", e.GetEndpointName(), err) - w.WriteHeader(500) return } newItem := model.NewItem(item) @@ -176,10 +173,38 @@ func webhook[T any]( err = db.SaveItem(e.GetEndpointName(), newItem) if err != nil { log.Printf("failed to save %s: %v", e.GetEndpointName(), err) - w.WriteHeader(500) return } + + // update associated game + type gameGetter interface { + GetGame() *pb.Game + } + + if v, ok := any(item).(gameGetter); ok { + game, err := db.GetItemByIGDBID[pb.Game](endpoint.EPGames, v.GetGame().Id) + if err != nil && err != mongo.ErrNoDocuments { + log.Printf("failed to get game: %v", err) + return + } + g, err := db.ConvertGame(game.Item) + if err != nil { + log.Printf("failed to convert game: %v", err) + return + } + oldGame, err := db.GetGameByIGDBID(game.Item.Id) + if err != nil && err != mongo.ErrNoDocuments { + log.Printf("failed to get game: %v", err) + return + } + g.MId = oldGame.MId + err = db.SaveGame(g) + if err != nil { + log.Printf("failed to save game: %v", err) + return + } + } + log.Printf("%s %d saved", e.GetEndpointName(), data.ID) - w.WriteHeader(200) } } diff --git a/db/db.go b/db/db.go index aadb5ed..e5e4d8e 100644 --- a/db/db.go +++ b/db/db.go @@ -21,8 +21,9 @@ var ( ) type MongoDB struct { - client *mongo.Client - Collections map[endpoint.EndpointName]*mongo.Collection + client *mongo.Client + Collections map[endpoint.EndpointName]*mongo.Collection + GameCollection *mongo.Collection } func GetInstance() *MongoDB { @@ -48,6 +49,7 @@ func GetInstance() *MongoDB { instance.Collections[e] = client.Database(config.C().Database.Database).Collection(string(e)) } + instance.GameCollection = client.Database(config.C().Database.Database).Collection("game_details") instance.createIndex() }) @@ -450,3 +452,55 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error { return nil } + +func GetItemsByIGDBGameID[T any](e endpoint.EndpointName, id uint64) ([]*model.Item[T], error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + coll := GetInstance().Collections[e] + if coll == nil { + return nil, fmt.Errorf("collection not found") + } + cursor, err := coll.Find(ctx, bson.M{"item.game.id": id}) + if err != nil { + return nil, fmt.Errorf("failed to get items %s: %v", string(e), err) + } + + var items []*model.Item[T] + for cursor.Next(ctx) { + item := model.Item[T]{} + err := cursor.Decode(&item) + if err != nil { + return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err) + } + items = append(items, &item) + } + + return items, nil +} + +func GetItemsPagnated[T any](e endpoint.EndpointName, offset int64, limit int64) ([]*model.Item[T], error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond) + defer cancel() + + coll := GetInstance().Collections[e] + if coll == nil { + return nil, fmt.Errorf("collection not found") + } + cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}})) + if err != nil { + return nil, fmt.Errorf("failed to get items %s: %v", string(e), err) + } + + var items []*model.Item[T] + for cursor.Next(ctx) { + item := model.Item[T]{} + err := cursor.Decode(&item) + if err != nil { + return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err) + } + items = append(items, &item) + } + + return items, nil +} diff --git a/db/game.go b/db/game.go new file mode 100644 index 0000000..d675aaa --- /dev/null +++ b/db/game.go @@ -0,0 +1,383 @@ +package db + +import ( + "context" + "fmt" + "igdb-database/model" + "time" + + "github.com/bestnite/go-igdb/endpoint" + pb "github.com/bestnite/go-igdb/proto" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond) + defer cancel() + + ids := make([]uint64, 0, len(games)) + for _, game := range games { + ids = append(ids, game.Id) + } + + cursor, err := GetInstance().GameCollection.Find(ctx, bson.M{"id": bson.M{"$in": ids}}) + if err != nil { + return nil, fmt.Errorf("failed to get games: %v", err) + } + + res := make(map[uint64]bool, len(games)) + g := []*model.Game{} + err = cursor.All(ctx, &g) + if err != nil { + return nil, fmt.Errorf("failed to get games: %v", err) + } + for _, game := range g { + res[game.Id] = true + } + + return res, nil +} + +func SaveGame(game *model.Game) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if game.MId.IsZero() { + game.MId = bson.NewObjectID() + } + filter := bson.M{"_id": game.MId} + update := bson.M{"$set": game} + opts := options.UpdateOne().SetUpsert(true) + + _, err := GetInstance().GameCollection.UpdateOne(ctx, filter, update, opts) + if err != nil { + return err + } + return nil +} + +func ConvertGame(game *pb.Game) (*model.Game, error) { + res := &model.Game{} + + res.Id = game.Id + + ageRatings, err := GetItemsByIGDBGameID[pb.AgeRating](endpoint.EPAgeRatings, game.Id) + if err != nil { + return nil, err + } + res.AgeRatings = make([]*pb.AgeRating, 0, len(ageRatings)) + for _, item := range ageRatings { + res.AgeRatings = append(res.AgeRatings, item.Item) + } + + res.AggregatedRating = game.AggregatedRating + res.AggregatedRatingCount = game.AggregatedRatingCount + + alternativeNames, err := GetItemsByIGDBGameID[pb.AlternativeName](endpoint.EPAlternativeNames, game.Id) + if err != nil { + return nil, err + } + res.AlternativeNames = make([]*pb.AlternativeName, 0, len(alternativeNames)) + for _, item := range alternativeNames { + res.AlternativeNames = append(res.AlternativeNames, item.Item) + } + + Artworks, err := GetItemsByIGDBGameID[pb.Artwork](endpoint.EPArtworks, game.Id) + if err != nil { + return nil, err + } + res.Artworks = make([]*pb.Artwork, 0, len(Artworks)) + for _, item := range Artworks { + res.Artworks = append(res.Artworks, item.Item) + } + + bundlesIds := make([]uint64, 0, len(game.Bundles)) + for _, g := range game.Bundles { + bundlesIds = append(bundlesIds, g.Id) + } + res.Bundles = bundlesIds + + covers, err := GetItemsByIGDBGameID[pb.Cover](endpoint.EPCovers, game.Id) + if err != nil { + return nil, err + } + if len(covers) != 0 { + res.Cover = covers[0].Item + } + + res.CreatedAt = game.CreatedAt + + dlcsIds := make([]uint64, 0, len(game.Dlcs)) + for _, g := range game.Dlcs { + dlcsIds = append(dlcsIds, g.Id) + } + res.Dlcs = dlcsIds + + expansionsIds := make([]uint64, 0, len(game.Expansions)) + for _, g := range game.Expansions { + expansionsIds = append(expansionsIds, g.Id) + } + res.Expansions = expansionsIds + + externalGames, err := GetItemsByIGDBGameID[pb.ExternalGame](endpoint.EPExternalGames, game.Id) + if err != nil { + return nil, err + } + res.ExternalGames = make([]*pb.ExternalGame, 0, len(externalGames)) + for _, item := range externalGames { + res.ExternalGames = append(res.ExternalGames, item.Item) + } + + res.FirstReleaseDate = game.FirstReleaseDate + + res.Franchise = nil + + franchises, err := GetItemsByIGDBGameID[pb.Franchise](endpoint.EPFranchises, game.Id) + if err != nil { + return nil, err + } + res.Franchises = make([]*pb.Franchise, 0, len(franchises)) + for _, item := range franchises { + res.Franchises = append(res.Franchises, item.Item) + } + + gameEngines, err := GetItemsByIGDBGameID[pb.GameEngine](endpoint.EPGameEngines, game.Id) + if err != nil { + return nil, err + } + res.GameEngines = make([]*pb.GameEngine, 0, len(gameEngines)) + for _, item := range gameEngines { + res.GameEngines = append(res.GameEngines, item.Item) + } + + gameModes, err := GetItemsByIGDBGameID[pb.GameMode](endpoint.EPGameModes, game.Id) + if err != nil { + return nil, err + } + res.GameModes = make([]*pb.GameMode, 0, len(gameModes)) + for _, item := range gameModes { + res.GameModes = append(res.GameModes, item.Item) + } + + genres, err := GetItemsByIGDBGameID[pb.Genre](endpoint.EPGenres, game.Id) + if err != nil { + return nil, err + } + res.Genres = make([]*pb.Genre, 0, len(genres)) + for _, item := range genres { + res.Genres = append(res.Genres, item.Item) + } + + res.Hypes = game.Hypes + + involvedCompanies, err := GetItemsByIGDBGameID[pb.InvolvedCompany](endpoint.EPInvolvedCompanies, game.Id) + if err != nil { + return nil, err + } + res.InvolvedCompanies = make([]*pb.InvolvedCompany, 0, len(involvedCompanies)) + for _, item := range involvedCompanies { + res.InvolvedCompanies = append(res.InvolvedCompanies, item.Item) + } + + keywords, err := GetItemsByIGDBGameID[pb.Keyword](endpoint.EPKeywords, game.Id) + if err != nil { + return nil, err + } + res.Keywords = make([]*pb.Keyword, 0, len(keywords)) + for _, item := range keywords { + res.Keywords = append(res.Keywords, item.Item) + } + + multiplayerModes, err := GetItemsByIGDBGameID[pb.MultiplayerMode](endpoint.EPMultiplayerModes, game.Id) + if err != nil { + return nil, err + } + res.MultiplayerModes = make([]*pb.MultiplayerMode, 0, len(multiplayerModes)) + for _, item := range multiplayerModes { + res.MultiplayerModes = append(res.MultiplayerModes, item.Item) + } + + res.Name = game.Name + + if game.ParentGame != nil { + res.ParentGame = model.GameId(game.ParentGame.Id) + } + + platforms, err := GetItemsByIGDBGameID[pb.Platform](endpoint.EPPlatforms, game.Id) + if err != nil { + return nil, err + } + res.Platforms = make([]*pb.Platform, 0, len(platforms)) + for _, item := range platforms { + res.Platforms = append(res.Platforms, item.Item) + } + + playerPerspectives, err := GetItemsByIGDBGameID[pb.PlayerPerspective](endpoint.EPPlayerPerspectives, game.Id) + if err != nil { + return nil, err + } + res.PlayerPerspectives = make([]*pb.PlayerPerspective, 0, len(playerPerspectives)) + for _, item := range playerPerspectives { + res.PlayerPerspectives = append(res.PlayerPerspectives, item.Item) + } + + res.Rating = game.Rating + res.RatingCount = game.RatingCount + + releaseDates, err := GetItemsByIGDBGameID[pb.ReleaseDate](endpoint.EPReleaseDates, game.Id) + if err != nil { + return nil, err + } + res.ReleaseDates = make([]*pb.ReleaseDate, 0, len(releaseDates)) + for _, item := range releaseDates { + res.ReleaseDates = append(res.ReleaseDates, item.Item) + } + + screenshots, err := GetItemsByIGDBGameID[pb.Screenshot](endpoint.EPScreenshots, game.Id) + if err != nil { + return nil, err + } + res.Screenshots = make([]*pb.Screenshot, 0, len(screenshots)) + for _, item := range screenshots { + res.Screenshots = append(res.Screenshots, item.Item) + } + + similarGamesIds := make([]uint64, 0, len(game.SimilarGames)) + for _, g := range game.SimilarGames { + similarGamesIds = append(similarGamesIds, g.Id) + } + res.SimilarGames = similarGamesIds + + res.Slug = game.Slug + + standaloneExpansionsIds := make([]uint64, 0, len(game.StandaloneExpansions)) + for _, g := range game.StandaloneExpansions { + standaloneExpansionsIds = append(standaloneExpansionsIds, g.Id) + } + res.StandaloneExpansions = standaloneExpansionsIds + + res.Storyline = game.Storyline + res.Summary = game.Summary + + res.Tags = game.Tags + + themes, err := GetItemsByIGDBGameID[pb.Theme](endpoint.EPThemes, game.Id) + if err != nil { + return nil, err + } + res.Themes = make([]*pb.Theme, 0, len(themes)) + for _, item := range themes { + res.Themes = append(res.Themes, item.Item) + } + + res.TotalRating = game.TotalRating + res.TotalRatingCount = game.TotalRatingCount + + res.UpdatedAt = game.UpdatedAt + + res.Url = game.Url + + if game.VersionParent != nil { + res.VersionParent = model.GameId(game.VersionParent.Id) + } + + res.VersionTitle = game.VersionTitle + + videos, err := GetItemsByIGDBGameID[pb.GameVideo](endpoint.EPGameVideos, game.Id) + if err != nil { + return nil, err + } + res.Videos = make([]*pb.GameVideo, 0, len(videos)) + for _, item := range videos { + res.Videos = append(res.Videos, item.Item) + } + + websites, err := GetItemsByIGDBGameID[pb.Website](endpoint.EPWebsites, game.Id) + if err != nil { + return nil, err + } + res.Websites = make([]*pb.Website, 0, len(websites)) + for _, item := range websites { + res.Websites = append(res.Websites, item.Item) + } + + remakesIds := make([]uint64, 0, len(game.Remakes)) + for _, g := range game.Remakes { + remakesIds = append(remakesIds, g.Id) + } + res.Remakes = remakesIds + + remastersIds := make([]uint64, 0, len(game.Remasters)) + for _, g := range game.Remasters { + remastersIds = append(remastersIds, g.Id) + } + res.Remasters = remastersIds + + expandedGamesIds := make([]uint64, 0, len(game.ExpandedGames)) + for _, g := range game.ExpandedGames { + expandedGamesIds = append(expandedGamesIds, g.Id) + } + res.ExpandedGames = expandedGamesIds + + portsIds := make([]uint64, 0, len(game.Ports)) + for _, g := range game.Ports { + portsIds = append(portsIds, g.Id) + } + res.Ports = portsIds + + forksIds := make([]uint64, 0, len(game.Forks)) + for _, g := range game.Forks { + forksIds = append(forksIds, g.Id) + } + res.Forks = forksIds + + languageSupports, err := GetItemsByIGDBGameID[pb.LanguageSupport](endpoint.EPLanguageSupports, game.Id) + if err != nil { + return nil, err + } + res.LanguageSupports = make([]*pb.LanguageSupport, 0, len(languageSupports)) + for _, item := range languageSupports { + res.LanguageSupports = append(res.LanguageSupports, item.Item) + } + + gameLocalizations, err := GetItemsByIGDBGameID[pb.GameLocalization](endpoint.EPGameLocalizations, game.Id) + if err != nil { + return nil, err + } + res.GameLocalizations = make([]*pb.GameLocalization, 0, len(gameLocalizations)) + for _, item := range gameLocalizations { + res.GameLocalizations = append(res.GameLocalizations, item.Item) + } + + collections, err := GetItemsByIGDBGameID[pb.Collection](endpoint.EPCollections, game.Id) + if err != nil { + return nil, err + } + res.Collections = make([]*pb.Collection, 0, len(collections)) + for _, item := range collections { + res.Collections = append(res.Collections, item.Item) + } + + res.GameStatus = nil + res.GameType = nil + + res.AllNames = make([]string, 0, len(alternativeNames)+1) + res.AllNames = append(res.AllNames, game.Name) + for _, item := range alternativeNames { + res.AllNames = append(res.AllNames, item.Item.Name) + } + + return res, nil +} + +func GetGameByIGDBID(id uint64) (*model.Game, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var game model.Game + err := GetInstance().GameCollection.FindOne(ctx, bson.M{"id": id}).Decode(&game) + if err != nil { + return nil, fmt.Errorf("failed to get game: %v", err) + } + return &game, nil +} diff --git a/go.mod b/go.mod index 95f9a53..713195f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.24.1 require ( github.com/bestnite/go-igdb v0.0.9 go.mongodb.org/mongo-driver/v2 v2.1.0 + google.golang.org/protobuf v1.36.6 ) require ( @@ -29,6 +30,5 @@ require ( golang.org/x/sync v0.12.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect - google.golang.org/protobuf v1.36.6 // indirect h12.io/socks v1.0.3 // indirect ) diff --git a/go.sum b/go.sum index ae83a2d..42aed03 100644 --- a/go.sum +++ b/go.sum @@ -23,14 +23,6 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bestnite/go-flaresolverr v0.0.0-20250404141941-4644c2e66727 h1:F1fNb9j7wgPXa54SWAIYn1l8NJTg74Qx3EJ8qmys6FY= github.com/bestnite/go-flaresolverr v0.0.0-20250404141941-4644c2e66727/go.mod h1:LX2oPIfG4LnUtQ7FAWV727IXuODZVbzRwG/7t2KdMNo= -github.com/bestnite/go-igdb v0.0.5 h1:/1cFbbYjQCGMFcE1bzYlvkYG+n//A08t/3lWb0OgU5s= -github.com/bestnite/go-igdb v0.0.5/go.mod h1:HBPwYCgSVd7oaiLWJVV72UpqsmIYjUmaIvGmuFk3CwY= -github.com/bestnite/go-igdb v0.0.6 h1:TDVK5JxOoy+YQX4cX7zupJWDWZCZaRmuyAny3NnBbSs= -github.com/bestnite/go-igdb v0.0.6/go.mod h1:HBPwYCgSVd7oaiLWJVV72UpqsmIYjUmaIvGmuFk3CwY= -github.com/bestnite/go-igdb v0.0.7 h1:a4JyQH/k/aWoWeD9RN929LqayRXibif+CxtsDaKKCQc= -github.com/bestnite/go-igdb v0.0.7/go.mod h1:HBPwYCgSVd7oaiLWJVV72UpqsmIYjUmaIvGmuFk3CwY= -github.com/bestnite/go-igdb v0.0.8 h1:BRf2EFESIqHfsYEVJKywTWxZ23tg4O4XYU+ucdchvOE= -github.com/bestnite/go-igdb v0.0.8/go.mod h1:HBPwYCgSVd7oaiLWJVV72UpqsmIYjUmaIvGmuFk3CwY= github.com/bestnite/go-igdb v0.0.9 h1:3n1OHSf6mA2ygoktbcVvAlQXZjkmtKuP1+Ql1pNAN5I= github.com/bestnite/go-igdb v0.0.9/go.mod h1:HBPwYCgSVd7oaiLWJVV72UpqsmIYjUmaIvGmuFk3CwY= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= diff --git a/main.go b/main.go index 6af5b62..932be0e 100644 --- a/main.go +++ b/main.go @@ -5,9 +5,12 @@ import ( "igdb-database/config" "igdb-database/db" "log" + "sync" + "sync/atomic" "github.com/bestnite/go-igdb" "github.com/bestnite/go-igdb/endpoint" + pb "github.com/bestnite/go-igdb/proto" ) func main() { @@ -83,9 +86,72 @@ func main() { fetchAndStore(client.Websites) fetchAndStore(client.WebsiteTypes) + log.Printf("aggregating games") + aggregateGames() + log.Printf("games aggregated") + + log.Printf("starting webhook server") collector.StartWebhookServer(client) } +func aggregateGames() { + total, err := db.CountItems(endpoint.EPGames) + if err != nil { + log.Fatalf("failed to count games: %v", err) + } + + finished := int64(0) + wg := sync.WaitGroup{} + + concurrenceNum := 10 + taskOneLoop := int64(500) + + concurrence := make(chan struct{}, concurrenceNum) + defer close(concurrence) + for i := int64(0); i < total; i += taskOneLoop { + concurrence <- struct{}{} + wg.Add(1) + go func(i int64) { + defer func() { <-concurrence }() + defer wg.Done() + items, err := db.GetItemsPagnated[pb.Game](endpoint.EPGames, i, taskOneLoop) + if err != nil { + log.Fatalf("failed to get games: %v", err) + } + games := make([]*pb.Game, 0, len(items)) + for _, item := range items { + games = append(games, item.Item) + } + isAggregated, err := db.IsGamesAggregated(games) + if err != nil { + log.Fatalf("failed to check if games are aggregated: %v", err) + } + for _, item := range items { + if isAggregated[item.Item.Id] { + p := atomic.AddInt64(&finished, 1) + log.Printf("game aggregated %d/%d", p, total) + continue + } + if err != nil { + log.Fatalf("failed to check if game is aggregated: %v", err) + } + + game, err := db.ConvertGame(item.Item) + if err != nil { + log.Fatalf("failed to convert game: %v", err) + } + err = db.SaveGame(game) + if err != nil { + log.Fatalf("failed to save game: %v", err) + } + p := atomic.AddInt64(&finished, 1) + log.Printf("game aggregated %d/%d", p, total) + } + }(i) + } + wg.Wait() +} + func fetchAndStore[T any]( e endpoint.EntityEndpoint[T], ) { diff --git a/model/game.go b/model/game.go new file mode 100644 index 0000000..292f181 --- /dev/null +++ b/model/game.go @@ -0,0 +1,71 @@ +package model + +import ( + pb "github.com/bestnite/go-igdb/proto" + "go.mongodb.org/mongo-driver/v2/bson" + "google.golang.org/protobuf/types/known/timestamppb" +) + +type GameIds []uint64 +type GameId uint64 + +type Game struct { + MId bson.ObjectID `bson:"_id,omitempty" json:"_id,omitempty"` + Id uint64 `json:"id,omitempty"` + AgeRatings []*pb.AgeRating `json:"age_ratings,omitempty"` + AggregatedRating float64 `json:"aggregated_rating,omitempty"` + AggregatedRatingCount int32 `json:"aggregated_rating_count,omitempty"` + AlternativeNames []*pb.AlternativeName `json:"alternative_names,omitempty"` + Artworks []*pb.Artwork `json:"artworks,omitempty"` + Bundles GameIds `json:"bundles,omitempty"` + Cover *pb.Cover `json:"cover,omitempty"` + CreatedAt *timestamppb.Timestamp `json:"created_at,omitempty"` + Dlcs GameIds `json:"dlcs,omitempty"` + Expansions GameIds `json:"expansions,omitempty"` + ExternalGames []*pb.ExternalGame `json:"external_games,omitempty"` + FirstReleaseDate *timestamppb.Timestamp `json:"first_release_date,omitempty"` + Franchise *pb.Franchise `json:"franchise,omitempty"` + Franchises []*pb.Franchise `json:"franchises,omitempty"` + GameEngines []*pb.GameEngine `json:"game_engines,omitempty"` + GameModes []*pb.GameMode `json:"game_modes,omitempty"` + Genres []*pb.Genre `json:"genres,omitempty"` + Hypes int32 `json:"hypes,omitempty"` + InvolvedCompanies []*pb.InvolvedCompany `json:"involved_companies,omitempty"` + Keywords []*pb.Keyword `json:"keywords,omitempty"` + MultiplayerModes []*pb.MultiplayerMode `json:"multiplayer_modes,omitempty"` + Name string `json:"name,omitempty"` + ParentGame GameId `json:"parent_game,omitempty"` + Platforms []*pb.Platform `json:"platforms,omitempty"` + PlayerPerspectives []*pb.PlayerPerspective `json:"player_perspectives,omitempty"` + Rating float64 `json:"rating,omitempty"` + RatingCount int32 `json:"rating_count,omitempty"` + ReleaseDates []*pb.ReleaseDate `json:"release_dates,omitempty"` + Screenshots []*pb.Screenshot `json:"screenshots,omitempty"` + SimilarGames GameIds `json:"similar_games,omitempty"` + Slug string `json:"slug,omitempty"` + StandaloneExpansions GameIds `json:"standalone_expansions,omitempty"` + Storyline string `json:"storyline,omitempty"` + Summary string `json:"summary,omitempty"` + Tags []int32 `json:"tags,omitempty"` + Themes []*pb.Theme `json:"themes,omitempty"` + TotalRating float64 `json:"total_rating,omitempty"` + TotalRatingCount int32 `json:"total_rating_count,omitempty"` + UpdatedAt *timestamppb.Timestamp `json:"updated_at,omitempty"` + Url string `json:"url,omitempty"` + VersionParent GameId `json:"version_parent,omitempty"` + VersionTitle string `json:"version_title,omitempty"` + Videos []*pb.GameVideo `json:"videos,omitempty"` + Websites []*pb.Website `json:"websites,omitempty"` + Remakes GameIds `json:"remakes,omitempty"` + Remasters GameIds `json:"remasters,omitempty"` + ExpandedGames GameIds `json:"expanded_games,omitempty"` + Ports GameIds `json:"ports,omitempty"` + Forks GameIds `json:"forks,omitempty"` + LanguageSupports []*pb.LanguageSupport `json:"language_supports,omitempty"` + GameLocalizations []*pb.GameLocalization `json:"game_localizations,omitempty"` + Collections []*pb.Collection `json:"collections,omitempty"` + GameStatus *pb.GameStatus `json:"game_status,omitempty"` + GameType *pb.GameType `json:"game_type,omitempty"` + + AllNames []string `bson:"all_names,omitempty"` +}