From 5bd2627d5ec057792ee53511804a126d2c9cc1b7 Mon Sep 17 00:00:00 2001 From: nite Date: Tue, 8 Apr 2025 16:06:44 +1000 Subject: [PATCH] u --- collector/webhook.go | 82 ++------------------------ db/count.go | 137 ------------------------------------------- main.go | 59 ------------------- 3 files changed, 6 insertions(+), 272 deletions(-) delete mode 100644 db/count.go diff --git a/collector/webhook.go b/collector/webhook.go index 155b634..ede1ce0 100644 --- a/collector/webhook.go +++ b/collector/webhook.go @@ -166,85 +166,15 @@ func webhook[T any]( log.Printf("failed to get %s: %v", e.GetEndpointName(), err) return } - oldItem, err := db.GetItemById[T](e.GetEndpointName(), data.ID) - if err != nil && !errors.Is(err, mongo.ErrNoDocuments) { - log.Printf("failed to get %s: %v", e.GetEndpointName(), err) - return - } if _, ok := any(e).(*endpoint.Games); ok { - if oldItem != nil { - // oldGame update - oldGame := any(oldItem).(*pb.Game) - // compare themes and genres - game := any(item).(*pb.Game) - - oldGameThemeIds := make([]uint64, 0, len(oldGame.Themes)) - for _, t := range oldGame.Themes { - oldGameThemeIds = append(oldGameThemeIds, t.Id) - } - gameThemeIds := make([]uint64, 0, len(game.Themes)) - for _, t := range game.Themes { - gameThemeIds = append(gameThemeIds, t.Id) - } - if !slices.Equal(oldGameThemeIds, gameThemeIds) { - for _, t := range oldGameThemeIds { - if !slices.Contains(gameThemeIds, t) { - _ = db.MinusThemeCount(t) - } - } - for _, t := range gameThemeIds { - if !slices.Contains(oldGameThemeIds, t) { - _ = db.AddThemeCount(t) - } - } - } - - oldGameGenreIds := make([]uint64, 0, len(oldGame.Genres)) - for _, t := range oldGame.Genres { - oldGameGenreIds = append(oldGameGenreIds, t.Id) - } - gameGenreIds := make([]uint64, 0, len(game.Genres)) - for _, t := range game.Genres { - gameGenreIds = append(gameGenreIds, t.Id) - } - if !slices.Equal(oldGameGenreIds, gameGenreIds) { - for _, t := range oldGameGenreIds { - if !slices.Contains(gameGenreIds, t) { - _ = db.MinusGenreCount(t) - } - } - for _, t := range gameGenreIds { - if !slices.Contains(oldGameGenreIds, t) { - _ = db.AddGenreCount(t) - } - } - } - - g, err := db.ConvertGame(game) - if err != nil { - log.Printf("failed to convert game: %v", err) - } else { - _ = db.SaveGame(g) - log.Printf("game %d aggregated", data.ID) - } - + game := any(item).(*pb.Game) + g, err := db.ConvertGame(game) + if err != nil { + log.Printf("failed to convert game: %v", err) } else { - // new game - game := any(item).(*pb.Game) - for _, t := range game.Themes { - _ = db.AddThemeCount(t.Id) - } - for _, t := range game.Genres { - _ = db.AddGenreCount(t.Id) - } - g, err := db.ConvertGame(game) - if err != nil { - log.Printf("failed to convert game: %v", err) - } else { - _ = db.SaveGame(g) - log.Printf("game %d aggregated", data.ID) - } + _ = db.SaveGame(g) + log.Printf("game %d aggregated", data.ID) } } diff --git a/db/count.go b/db/count.go deleted file mode 100644 index e71a262..0000000 --- a/db/count.go +++ /dev/null @@ -1,137 +0,0 @@ -package db - -import ( - "context" - "fmt" - "time" - - "github.com/bestnite/go-igdb/endpoint" - "go.mongodb.org/mongo-driver/v2/bson" - "go.mongodb.org/mongo-driver/v2/mongo/options" -) - -type Count struct { - Theme uint64 `json:"theme,omitempty"` - Genre uint64 `json:"genre,omitempty"` - Count int64 `json:"count,omitempty"` -} - -func GetCountByThemeId(themeId uint64) (*Count, error) { - var count Count - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - err := GetInstance().CountCollection.FindOne(ctx, bson.M{"theme": themeId}).Decode(&count) - if err != nil { - return nil, fmt.Errorf("failed to get count: %w", err) - } - return &count, nil -} - -func GetCountByGenreId(genreId uint64) (*Count, error) { - var count Count - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - err := GetInstance().CountCollection.FindOne(ctx, bson.M{"genre": genreId}).Decode(&count) - if err != nil { - return nil, fmt.Errorf("failed to get count: %w", err) - } - return &count, nil -} - -func SaveCount(count *Count) error { - var filter bson.M - if count.Genre != 0 { - filter = bson.M{"genre": count.Genre} - } - if count.Theme != 0 { - filter = bson.M{"theme": count.Theme} - } - update := bson.M{"$set": count} - opts := options.UpdateOne().SetUpsert(true) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) - if err != nil { - return err - } - return nil -} - -func AddThemeCount(themeId uint64) error { - filter := bson.M{"theme": themeId} - update := bson.M{"$inc": bson.M{"count": 1}} - opts := options.UpdateOne().SetUpsert(true) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) - if err != nil { - return err - } - return nil -} - -func AddGenreCount(genreId uint64) error { - filter := bson.M{"genre": genreId} - update := bson.M{"$inc": bson.M{"count": 1}} - opts := options.UpdateOne().SetUpsert(true) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) - if err != nil { - return err - } - return nil -} - -func MinusThemeCount(themeId uint64) error { - filter := bson.M{"theme": themeId} - update := bson.M{"$inc": bson.M{"count": -1}} - opts := options.UpdateOne().SetUpsert(true) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) - if err != nil { - return err - } - return nil -} - -func MinusGenreCount(genreId uint64) error { - filter := bson.M{"genre": genreId} - update := bson.M{"$inc": bson.M{"count": -1}} - opts := options.UpdateOne().SetUpsert(true) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts) - if err != nil { - return err - } - return nil -} - -func CountTheme(themeId uint64) (int64, error) { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - filter := bson.M{"themes.id": themeId} - count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter) - if err != nil { - return 0, fmt.Errorf("failed to count theme: %w", err) - } - return count, nil -} - -func CountGenre(genreId uint64) (int64, error) { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - defer cancel() - filter := bson.M{"genres.id": genreId} - count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter) - if err != nil { - return 0, fmt.Errorf("failed to count genre: %w", err) - } - return count, nil -} diff --git a/main.go b/main.go index 37176ce..2d04604 100644 --- a/main.go +++ b/main.go @@ -34,8 +34,6 @@ func main() { log.Printf("data fetched") } - Count() - if *enableAggregate || *enableReAggregate { log.Printf("aggregating games") aggregateGames() @@ -118,63 +116,6 @@ func fetchAndStore[T any]( } } -func Count() { - ids, err := db.GetAllItemsIDs[pb.Theme](endpoint.EPThemes) - if err != nil { - log.Fatalf("failed to get all items ids %s: %v", endpoint.EPThemes, err) - } - - concurrence := make(chan struct{}, 10) - defer close(concurrence) - wg := sync.WaitGroup{} - - for _, id := range ids { - concurrence <- struct{}{} - wg.Add(1) - go func(id uint64) { - defer func() { - <-concurrence - wg.Done() - }() - count, err := db.CountTheme(id) - if err != nil { - log.Fatalf("failed to count theme: %v", err) - } - log.Printf("theme %d count: %d", id, count) - err = db.SaveCount(&db.Count{Theme: id, Count: count}) - if err != nil { - log.Fatalf("failed to save count: %v", err) - } - }(id) - } - - ids, err = db.GetAllItemsIDs[pb.Genre](endpoint.EPGenres) - if err != nil { - log.Fatalf("failed to get all items ids %s: %v", endpoint.EPGenres, err) - } - for _, id := range ids { - concurrence <- struct{}{} - wg.Add(1) - go func(id uint64) { - defer func() { - <-concurrence - wg.Done() - }() - count, err := db.CountGenre(id) - if err != nil { - log.Fatalf("failed to count genre: %v", err) - } - log.Printf("genre %d count: %d", id, count) - err = db.SaveCount(&db.Count{Genre: id, Count: count}) - if err != nil { - log.Fatalf("failed to save count: %v", err) - } - }(id) - } - - wg.Wait() -} - func allFetchAndStore(client *igdb.Client) { if *onlyRefetchGames { fetchAndStore(client.Games)