u

collector/store.go

@@ -2,7 +2,6 @@ package collector
 
 import (
 	"igdb-database/db"
-	"igdb-database/model"
 	"log"
 	"math"
 	"sync"
@@ -33,50 +32,13 @@ func FetchAndStore[T any](
 			defer wg.Done()
 			defer func() { <-concurrence }()
 
-			// items data from igdb
 			items, err := e.Paginated(uint64(i), 500)
 			if err != nil {
 				log.Printf("failed to get items from igdb %s: %v", e.GetEndpointName(), err)
 				return
 			}
 
-			type IdGetter interface {
-				GetId() uint64
-			}
-
-			ids := make([]uint64, 0, len(items))
-			for _, item := range items {
-				if v, ok := any(item).(IdGetter); ok {
-					ids = append(ids, v.GetId())
-				} else {
-					log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err)
-					return
-				}
-			}
-
-			// items data from database
-			data, err := db.GetItemsByIGDBIDs[T](e.GetEndpointName(), ids)
-			if err != nil {
-				log.Printf("failed to get items from database %s: %v", e.GetEndpointName(), err)
-				return
-			}
-
-			newItems := make([]*model.Item[T], 0, len(items))
-			for _, item := range items {
-				v, ok := any(item).(IdGetter)
-				if !ok {
-					log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err)
-					return
-				} else {
-					if data[v.GetId()] == nil {
-						newItems = append(newItems, model.NewItem(item))
-					} else {
-						data[v.GetId()].Item = item
-						newItems = append(newItems, data[v.GetId()])
-					}
-				}
-			}
-			err = db.SaveItems(e.GetEndpointName(), newItems)
+			err = db.SaveItems(e.GetEndpointName(), items)
 			if err != nil {
 				log.Printf("failed to save games %s: %v", e.GetEndpointName(), err)
 				return

collector/webhook.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"igdb-database/config"
 	"igdb-database/db"
-	"igdb-database/model"
 	"io"
 	"log"
 	"net"
@@ -167,57 +166,110 @@ func webhook[T any](
 			log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
 			return
 		}
-		oldItem, err := db.GetItemByIGDBID[T](e.GetEndpointName(), data.ID)
+		oldItem, err := db.GetItemById[T](e.GetEndpointName(), data.ID)
 		if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
 			log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
 			return
 		}
-		newItem := model.NewItem(item)
-		if oldItem != nil {
-			newItem.MId = oldItem.MId
-		}
-		err = db.SaveItem(e.GetEndpointName(), newItem)
-		if err != nil {
-			log.Printf("failed to save %s: %v", e.GetEndpointName(), err)
-			return
-		}
 
 		if _, ok := any(e).(*endpoint.Games); ok {
-			game, err := db.GetItemByIGDBID[pb.Game](endpoint.EPGames, data.ID)
-			if err == nil {
-				g, err := db.ConvertGame(game.Item)
-				if err == nil {
-					g.MId = game.MId
+			if oldItem != nil {
+				// oldGame update
+				oldGame := any(oldItem).(*pb.Game)
+				// compare themes and genres
+				game := any(item).(*pb.Game)
+
+				oldGameThemeIds := make([]uint64, 0, len(oldGame.Themes))
+				for _, t := range oldGame.Themes {
+					oldGameThemeIds = append(oldGameThemeIds, t.Id)
+				}
+				gameThemeIds := make([]uint64, 0, len(game.Themes))
+				for _, t := range game.Themes {
+					gameThemeIds = append(gameThemeIds, t.Id)
+				}
+				if !slices.Equal(oldGameThemeIds, gameThemeIds) {
+					for _, t := range oldGameThemeIds {
+						if !slices.Contains(gameThemeIds, t) {
+							_ = db.MinusThemeCount(t)
+						}
+					}
+					for _, t := range gameThemeIds {
+						if !slices.Contains(oldGameThemeIds, t) {
+							_ = db.AddThemeCount(t)
+						}
+					}
+				}
+
+				oldGameGenreIds := make([]uint64, 0, len(oldGame.Genres))
+				for _, t := range oldGame.Genres {
+					oldGameGenreIds = append(oldGameGenreIds, t.Id)
+				}
+				gameGenreIds := make([]uint64, 0, len(game.Genres))
+				for _, t := range game.Genres {
+					gameGenreIds = append(gameGenreIds, t.Id)
+				}
+				if !slices.Equal(oldGameGenreIds, gameGenreIds) {
+					for _, t := range oldGameGenreIds {
+						if !slices.Contains(gameGenreIds, t) {
+							_ = db.MinusGenreCount(t)
+						}
+					}
+					for _, t := range gameGenreIds {
+						if !slices.Contains(oldGameGenreIds, t) {
+							_ = db.AddGenreCount(t)
+						}
+					}
+				}
+
+				g, err := db.ConvertGame(game)
+				if err != nil {
+					log.Printf("failed to convert game: %v", err)
+				} else {
+					_ = db.SaveGame(g)
+					log.Printf("game %d aggregated", data.ID)
+				}
+
+			} else {
+				// new game
+				game := any(item).(*pb.Game)
+				for _, t := range game.Themes {
+					_ = db.AddThemeCount(t.Id)
+				}
+				for _, t := range game.Genres {
+					_ = db.AddGenreCount(t.Id)
+				}
+				g, err := db.ConvertGame(game)
+				if err != nil {
+					log.Printf("failed to convert game: %v", err)
+				} else {
 					_ = db.SaveGame(g)
 					log.Printf("game %d aggregated", data.ID)
 				}
 			}
 		}
 
+		err = db.SaveItem(e.GetEndpointName(), item)
+		if err != nil {
+			log.Printf("failed to save %s: %v", e.GetEndpointName(), err)
+			return
+		}
+
 		// update associated game
 		type gameGetter interface {
 			GetGame() *pb.Game
 		}
 
 		if v, ok := any(item).(gameGetter); ok {
-			game, err := db.GetItemByIGDBID[pb.Game](endpoint.EPGames, v.GetGame().Id)
+			game, err := db.GetItemById[pb.Game](endpoint.EPGames, v.GetGame().Id)
 			if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
 				log.Printf("failed to get game: %v", err)
 				goto END
 			}
-			g, err := db.ConvertGame(game.Item)
+			g, err := db.ConvertGame(game)
 			if err != nil {
 				log.Printf("failed to convert game: %v", err)
 				goto END
 			}
-			oldGame, err := db.GetGameByIGDBID(game.Item.Id)
-			if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
-				log.Printf("failed to get game: %v", err)
-				goto END
-			}
-			if oldGame != nil {
-				g.MId = oldGame.MId
-			}
 			err = db.SaveGame(g)
 			if err != nil {
 				log.Printf("failed to save game: %v", err)

db/count.go

@@ -0,0 +1,137 @@
+package db
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/bestnite/go-igdb/endpoint"
+	"go.mongodb.org/mongo-driver/v2/bson"
+	"go.mongodb.org/mongo-driver/v2/mongo/options"
+)
+
+type Count struct {
+	Theme uint64 `json:"theme,omitempty"`
+	Genre uint64 `json:"genre,omitempty"`
+	Count int64  `json:"count,omitempty"`
+}
+
+func GetCountByThemeId(themeId uint64) (*Count, error) {
+	var count Count
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err := GetInstance().CountCollection.FindOne(ctx, bson.M{"theme": themeId}).Decode(&count)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get count: %w", err)
+	}
+	return &count, nil
+}
+
+func GetCountByGenreId(genreId uint64) (*Count, error) {
+	var count Count
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err := GetInstance().CountCollection.FindOne(ctx, bson.M{"genre": genreId}).Decode(&count)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get count: %w", err)
+	}
+	return &count, nil
+}
+
+func SaveCount(count *Count) error {
+	var filter bson.M
+	if count.Genre != 0 {
+		filter = bson.M{"genre": count.Genre}
+	}
+	if count.Theme != 0 {
+		filter = bson.M{"theme": count.Theme}
+	}
+	update := bson.M{"$set": count}
+	opts := options.UpdateOne().SetUpsert(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func AddThemeCount(themeId uint64) error {
+	filter := bson.M{"theme": themeId}
+	update := bson.M{"$inc": bson.M{"count": 1}}
+	opts := options.UpdateOne().SetUpsert(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func AddGenreCount(genreId uint64) error {
+	filter := bson.M{"genre": genreId}
+	update := bson.M{"$inc": bson.M{"count": 1}}
+	opts := options.UpdateOne().SetUpsert(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func MinusThemeCount(themeId uint64) error {
+	filter := bson.M{"theme": themeId}
+	update := bson.M{"$inc": bson.M{"count": -1}}
+	opts := options.UpdateOne().SetUpsert(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func MinusGenreCount(genreId uint64) error {
+	filter := bson.M{"genre": genreId}
+	update := bson.M{"$inc": bson.M{"count": -1}}
+	opts := options.UpdateOne().SetUpsert(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func CountTheme(themeId uint64) (int64, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	filter := bson.M{"themes.id": themeId}
+	count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter)
+	if err != nil {
+		return 0, fmt.Errorf("failed to count theme: %w", err)
+	}
+	return count, nil
+}
+
+func CountGenre(genreId uint64) (int64, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+	filter := bson.M{"genres.id": genreId}
+	count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter)
+	if err != nil {
+		return 0, fmt.Errorf("failed to count genre: %w", err)
+	}
+	return count, nil
+}

db/db.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"igdb-database/config"
-	"igdb-database/model"
 	"log"
 	"sync"
 	"time"
@@ -21,9 +20,10 @@ var (
 )
 
 type MongoDB struct {
-	client         *mongo.Client
-	Collections    map[endpoint.Name]*mongo.Collection
-	GameCollection *mongo.Collection
+	client          *mongo.Client
+	Collections     map[endpoint.Name]*mongo.Collection
+	GameCollection  *mongo.Collection
+	CountCollection *mongo.Collection
 }
 
 func GetInstance() *MongoDB {
@@ -54,6 +54,7 @@ func GetInstance() *MongoDB {
 		}
 
 		instance.GameCollection = client.Database(config.C().Database.Database).Collection("game_details")
+		instance.CountCollection = client.Database(config.C().Database.Database).Collection("count")
 		instance.createIndex()
 	})
 
@@ -65,8 +66,8 @@ func (m *MongoDB) createIndex() {
 	defer cancel()
 
 	textIndexMap := map[endpoint.Name]string{
-		endpoint.EPGames:            "item.name",
-		endpoint.EPAlternativeNames: "item.name",
+		endpoint.EPGames:            "name",
+		endpoint.EPAlternativeNames: "name",
 	}
 
 	for e, idx := range textIndexMap {
@@ -81,23 +82,23 @@ func (m *MongoDB) createIndex() {
 	}
 
 	indexMap := map[endpoint.Name][]string{
-		endpoint.EPAlternativeNames:         {"item.game.id"},
-		endpoint.EPArtworks:                 {"item.game.id"},
-		endpoint.EPCollectionMemberships:    {"item.game.id"},
-		endpoint.EPCovers:                   {"item.game.id"},
-		endpoint.EPExternalGames:            {"item.game.id"},
-		endpoint.EPGameEngines:              {"item.game.id"},
-		endpoint.EPGameLocalizations:        {"item.game.id"},
-		endpoint.EPGameVersions:             {"item.game.id"},
-		endpoint.EPGameVersionFeatureValues: {"item.game.id"},
-		endpoint.EPGameVideos:               {"item.game.id"},
-		endpoint.EPInvolvedCompanies:        {"item.game.id"},
-		endpoint.EPLanguageSupports:         {"item.game.id"},
-		endpoint.EPMultiplayerModes:         {"item.game.id"},
-		endpoint.EPReleaseDates:             {"item.game.id"},
-		endpoint.EPScreenshots:              {"item.game.id"},
-		endpoint.EPWebsites:                 {"item.game.id"},
-		endpoint.EPGames:                    {"item.parent_game.id", "item.version_parent.id"},
+		endpoint.EPAlternativeNames:         {"game.id"},
+		endpoint.EPArtworks:                 {"game.id"},
+		endpoint.EPCollectionMemberships:    {"game.id"},
+		endpoint.EPCovers:                   {"game.id"},
+		endpoint.EPExternalGames:            {"game.id"},
+		endpoint.EPGameEngines:              {"game.id"},
+		endpoint.EPGameLocalizations:        {"game.id"},
+		endpoint.EPGameVersions:             {"game.id"},
+		endpoint.EPGameVersionFeatureValues: {"game.id"},
+		endpoint.EPGameVideos:               {"game.id"},
+		endpoint.EPInvolvedCompanies:        {"game.id"},
+		endpoint.EPLanguageSupports:         {"game.id"},
+		endpoint.EPMultiplayerModes:         {"game.id"},
+		endpoint.EPReleaseDates:             {"game.id"},
+		endpoint.EPScreenshots:              {"game.id"},
+		endpoint.EPWebsites:                 {"game.id"},
+		endpoint.EPGames:                    {"parent_game.id", "version_parent.id"},
 	}
 
 	for e, idxes := range indexMap {
@@ -119,11 +120,12 @@ func (m *MongoDB) createIndex() {
 		}
 		_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
 			Keys: bson.D{
-				{Key: "item.id", Value: 1},
+				{Key: "id", Value: 1},
 			},
+			Options: options.Index().SetUnique(true),
 		})
 		if err != nil {
-			log.Printf("failed to create index item.id for %s: %v", string(e), err)
+			log.Printf("failed to create index id for %s: %v", string(e), err)
 		}
 	}
 
@@ -131,62 +133,14 @@ func (m *MongoDB) createIndex() {
 		Keys: bson.D{
 			{Key: "id", Value: 1},
 		},
+		Options: options.Index().SetUnique(true),
 	})
 	if err != nil {
 		log.Printf("failed to create index id for game_details: %v", err)
 	}
 }
 
-func SaveItem[T any](e endpoint.Name, item *model.Item[T]) error {
-	if item.MId.IsZero() {
-		item.MId = bson.NewObjectID()
-	}
-	filter := bson.M{"_id": item.MId}
-	update := bson.M{"$set": item}
-	opts := options.UpdateOne().SetUpsert(true)
-
-	coll := GetInstance().Collections[e]
-	if coll == nil {
-		return fmt.Errorf("collection not found")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	_, err := coll.UpdateOne(ctx, filter, update, opts)
-	if err != nil {
-		return fmt.Errorf("failed to save item to %s: %w", string(e), err)
-	}
-	return nil
-}
-
-func SaveItems[T any](e endpoint.Name, items []*model.Item[T]) error {
-	var models []mongo.WriteModel
-
-	for _, item := range items {
-		if item.MId.IsZero() {
-			item.MId = bson.NewObjectID()
-		}
-		filter := bson.M{"_id": item.MId}
-		update := bson.M{"$set": item}
-		oneModel := mongo.NewUpdateOneModel().SetFilter(filter).SetUpdate(update).SetUpsert(true)
-		models = append(models, oneModel)
-	}
-
-	coll := GetInstance().Collections[e]
-	if coll == nil {
-		return fmt.Errorf("collection not found")
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
-	defer cancel()
-	_, err := coll.BulkWrite(ctx, models)
-	if err != nil {
-		return fmt.Errorf("failed to save items in bulk %s: %w", string(e), err)
-	}
-	return nil
-}
-
-func CountItems(e endpoint.Name) (int64, error) {
+func EstimatedDocumentCount(e endpoint.Name) (int64, error) {
 	coll := GetInstance().Collections[e]
 	if coll == nil {
 		return 0, fmt.Errorf("collection not found")
@@ -195,66 +149,12 @@ func CountItems(e endpoint.Name) (int64, error) {
 	defer cancel()
 	count, err := coll.EstimatedDocumentCount(ctx)
 	if err != nil {
-		return 0, fmt.Errorf("failed to count items %s: %w", string(e), err)
+		return 0, fmt.Errorf("failed to count  %s: %w", string(e), err)
 	}
 	return count, nil
 }
 
-func GetItemByIGDBID[T any](e endpoint.Name, id uint64) (*model.Item[T], error) {
-	var item model.Item[T]
-	coll := GetInstance().Collections[e]
-	if coll == nil {
-		return nil, fmt.Errorf("collection not found")
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get item %s: %w", string(e), err)
-	}
-	return &item, nil
-}
-
-func GetItemsByIGDBIDs[T any](e endpoint.Name, ids []uint64) (map[uint64]*model.Item[T], error) {
-	if len(ids) == 0 {
-		return nil, nil
-	}
-
-	coll := GetInstance().Collections[e]
-	if coll == nil {
-		return nil, fmt.Errorf("collection not found")
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}})
-	if err != nil {
-		return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
-	}
-
-	type IdGetter interface {
-		GetId() uint64
-	}
-
-	ctx, cancel = context.WithTimeout(context.Background(), time.Duration(len(ids))*200*time.Millisecond)
-	defer cancel()
-	res := make(map[uint64]*model.Item[T])
-	for cursor.Next(ctx) {
-		item := model.Item[T]{}
-		err := cursor.Decode(&item)
-		if err != nil {
-			return nil, fmt.Errorf("failed to decode item %s: %w", string(e), err)
-		}
-		if v, ok := any(item.Item).(IdGetter); ok {
-			res[v.GetId()] = &item
-		} else {
-			return nil, fmt.Errorf("failed to get id from item %s: %w", string(e), err)
-		}
-	}
-
-	return res, nil
-}
-
-func RemoveItemsByID(e endpoint.Name, ids []bson.ObjectID) error {
+func RemoveByID(e endpoint.Name, ids []bson.ObjectID) error {
 	coll := GetInstance().Collections[e]
 	if coll == nil {
 		return fmt.Errorf("collection not found")
@@ -269,25 +169,98 @@ func RemoveItemsByID(e endpoint.Name, ids []bson.ObjectID) error {
 	return nil
 }
 
-func GetItemsPaginated[T any](e endpoint.Name, offset int64, limit int64) ([]*model.Item[T], error) {
-
+func GetItemsByIds[T any](e endpoint.Name, ids []uint64) ([]*T, error) {
 	coll := GetInstance().Collections[e]
 	if coll == nil {
 		return nil, fmt.Errorf("collection not found")
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second+time.Duration(len(ids)*200)*time.Millisecond)
 	defer cancel()
-	cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}}))
+	cursor, err := coll.Find(ctx, bson.M{"id": bson.M{"$in": ids}})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get items: %w", err)
+	}
+
+	var items []*T
+	err = cursor.All(ctx, &items)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get items: %w", err)
+	}
+	return items, nil
+}
+
+func GetItemById[T any](e endpoint.Name, id uint64) (*T, error) {
+	coll := GetInstance().Collections[e]
+	if coll == nil {
+		return nil, fmt.Errorf("collection not found")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	var item *T
+	err := coll.FindOne(ctx, bson.M{"id": id}).Decode(&item)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get item: %w", err)
+	}
+	return item, nil
+}
+
+func SaveItem[T any](e endpoint.Name, item *T) error {
+	type IdGetter interface {
+		GetId() uint64
+	}
+	id := any(item).(IdGetter).GetId()
+	filter := bson.M{"id": id}
+	update := bson.M{"$set": item}
+	opts := options.UpdateOne().SetUpsert(true)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_, err := GetInstance().Collections[e].UpdateOne(ctx, filter, update, opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func SaveItems[T any](e endpoint.Name, items []*T) error {
+	type IdGetter interface {
+		GetId() uint64
+	}
+
+	updateModel := make([]mongo.WriteModel, 0, len(items))
+	for _, item := range items {
+		updateModel = append(updateModel, mongo.NewUpdateOneModel().SetFilter(bson.M{"id": any(item).(IdGetter).GetId()}).SetUpdate(bson.M{"$set": item}).SetUpsert(true))
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
+	defer cancel()
+	_, err := GetInstance().Collections[e].BulkWrite(ctx, updateModel)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func GetItemsPaginated[T any](e endpoint.Name, skip int64, limit int64) ([]*T, error) {
+	coll := GetInstance().Collections[e]
+	if coll == nil {
+		return nil, fmt.Errorf("collection not found")
+	}
+
+	opts := options.Find().SetSort(bson.M{"id": 1}).SetSkip(skip).SetLimit(limit)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*time.Millisecond*200)
+	defer cancel()
+	cursor, err := coll.Find(ctx, bson.M{}, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
 	}
 
-	var items []*model.Item[T]
+	var items []*T
 	err = cursor.All(ctx, &items)
 	if err != nil {
-		return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
+		return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
 	}
-
 	return items, nil
 }

db/game.go

@@ -41,10 +41,7 @@ func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
 }
 
 func SaveGame(game *model.Game) error {
-	if game.MId.IsZero() {
-		game.MId = bson.NewObjectID()
-	}
-	filter := bson.M{"_id": game.MId}
+	filter := bson.M{"id": game.Id}
 	update := bson.M{"$set": game}
 	opts := options.UpdateOne().SetUpsert(true)
 
@@ -66,14 +63,11 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.AgeRatings {
 		ageRatingsIds = append(ageRatingsIds, g.Id)
 	}
-	ageRatings, err := GetItemsByIGDBIDs[pb.AgeRating](endpoint.EPAgeRatings, ageRatingsIds)
+	ageRatings, err := GetItemsByIds[pb.AgeRating](endpoint.EPAgeRatings, ageRatingsIds)
 	if err != nil {
 		return nil, err
 	}
-	res.AgeRatings = make([]*pb.AgeRating, 0, len(ageRatings))
-	for _, item := range ageRatings {
-		res.AgeRatings = append(res.AgeRatings, item.Item)
-	}
+	res.AgeRatings = ageRatings
 
 	res.AggregatedRating = game.AggregatedRating
 	res.AggregatedRatingCount = game.AggregatedRatingCount
@@ -82,27 +76,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.AlternativeNames {
 		alternativeNameIds = append(alternativeNameIds, g.Id)
 	}
-	alternativeNames, err := GetItemsByIGDBIDs[pb.AlternativeName](endpoint.EPAlternativeNames, alternativeNameIds)
+	alternativeNames, err := GetItemsByIds[pb.AlternativeName](endpoint.EPAlternativeNames, alternativeNameIds)
 	if err != nil {
 		return nil, err
 	}
-	res.AlternativeNames = make([]*pb.AlternativeName, 0, len(alternativeNames))
-	for _, item := range alternativeNames {
-		res.AlternativeNames = append(res.AlternativeNames, item.Item)
-	}
+	res.AlternativeNames = alternativeNames
 
 	ArtworkIds := make([]uint64, 0, len(game.Artworks))
 	for _, g := range game.Artworks {
 		ArtworkIds = append(ArtworkIds, g.Id)
 	}
-	artworks, err := GetItemsByIGDBIDs[pb.Artwork](endpoint.EPArtworks, ArtworkIds)
+	artworks, err := GetItemsByIds[pb.Artwork](endpoint.EPArtworks, ArtworkIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Artworks = make([]*pb.Artwork, 0, len(artworks))
-	for _, item := range artworks {
-		res.Artworks = append(res.Artworks, item.Item)
-	}
+	res.Artworks = artworks
 
 	bundlesIds := make([]uint64, 0, len(game.Bundles))
 	for _, g := range game.Bundles {
@@ -112,12 +100,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 
 	if game.Cover != nil {
 		coverId := game.Cover.Id
-		cover, err := GetItemByIGDBID[pb.Cover](endpoint.EPCovers, coverId)
+		cover, err := GetItemById[pb.Cover](endpoint.EPCovers, coverId)
 		if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
 			return nil, err
 		}
 		if cover != nil {
-			res.Cover = cover.Item
+			res.Cover = cover
 		}
 	}
 
@@ -139,14 +127,11 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.ExternalGames {
 		externalGameIds = append(externalGameIds, g.Id)
 	}
-	externalGames, err := GetItemsByIGDBIDs[pb.ExternalGame](endpoint.EPExternalGames, externalGameIds)
+	externalGames, err := GetItemsByIds[pb.ExternalGame](endpoint.EPExternalGames, externalGameIds)
 	if err != nil {
 		return nil, err
 	}
-	res.ExternalGames = make([]*pb.ExternalGame, 0, len(externalGames))
-	for _, item := range externalGames {
-		res.ExternalGames = append(res.ExternalGames, item.Item)
-	}
+	res.ExternalGames = externalGames
 
 	res.FirstReleaseDate = game.FirstReleaseDate
 
@@ -154,12 +139,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 
 	if game.Franchise != nil {
 		franchiseId := game.Franchise.Id
-		franchise, err := GetItemByIGDBID[pb.Franchise](endpoint.EPFranchises, franchiseId)
+		franchise, err := GetItemById[pb.Franchise](endpoint.EPFranchises, franchiseId)
 		if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
 			return nil, err
 		}
 		if franchise != nil {
-			res.Franchise = franchise.Item
+			res.Franchise = franchise
 		}
 	}
 
@@ -167,53 +152,41 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.Franchises {
 		franchiseIds = append(franchiseIds, g.Id)
 	}
-	franchises, err := GetItemsByIGDBIDs[pb.Franchise](endpoint.EPFranchises, franchiseIds)
+	franchises, err := GetItemsByIds[pb.Franchise](endpoint.EPFranchises, franchiseIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Franchises = make([]*pb.Franchise, 0, len(franchises))
-	for _, item := range franchises {
-		res.Franchises = append(res.Franchises, item.Item)
-	}
+	res.Franchises = franchises
 
 	gameEngineIds := make([]uint64, 0, len(game.GameEngines))
 	for _, g := range game.GameEngines {
 		gameEngineIds = append(gameEngineIds, g.Id)
 	}
-	gameEngines, err := GetItemsByIGDBIDs[pb.GameEngine](endpoint.EPGameEngines, gameEngineIds)
+	gameEngines, err := GetItemsByIds[pb.GameEngine](endpoint.EPGameEngines, gameEngineIds)
 	if err != nil {
 		return nil, err
 	}
-	res.GameEngines = make([]*pb.GameEngine, 0, len(gameEngines))
-	for _, item := range gameEngines {
-		res.GameEngines = append(res.GameEngines, item.Item)
-	}
+	res.GameEngines = gameEngines
 
 	gameModeIds := make([]uint64, 0, len(game.GameModes))
 	for _, g := range game.GameModes {
 		gameModeIds = append(gameModeIds, g.Id)
 	}
-	gameModes, err := GetItemsByIGDBIDs[pb.GameMode](endpoint.EPGameModes, gameModeIds)
+	gameModes, err := GetItemsByIds[pb.GameMode](endpoint.EPGameModes, gameModeIds)
 	if err != nil {
 		return nil, err
 	}
-	res.GameModes = make([]*pb.GameMode, 0, len(gameModes))
-	for _, item := range gameModes {
-		res.GameModes = append(res.GameModes, item.Item)
-	}
+	res.GameModes = gameModes
 
 	genreIds := make([]uint64, 0, len(game.Genres))
 	for _, g := range game.Genres {
 		genreIds = append(genreIds, g.Id)
 	}
-	genres, err := GetItemsByIGDBIDs[pb.Genre](endpoint.EPGenres, genreIds)
+	genres, err := GetItemsByIds[pb.Genre](endpoint.EPGenres, genreIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Genres = make([]*pb.Genre, 0, len(genres))
-	for _, item := range genres {
-		res.Genres = append(res.Genres, item.Item)
-	}
+	res.Genres = genres
 
 	res.Hypes = game.Hypes
 
@@ -221,40 +194,31 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.InvolvedCompanies {
 		involvedCompanyIds = append(involvedCompanyIds, g.Id)
 	}
-	involvedCompanies, err := GetItemsByIGDBIDs[pb.InvolvedCompany](endpoint.EPInvolvedCompanies, involvedCompanyIds)
+	involvedCompanies, err := GetItemsByIds[pb.InvolvedCompany](endpoint.EPInvolvedCompanies, involvedCompanyIds)
 	if err != nil {
 		return nil, err
 	}
-	res.InvolvedCompanies = make([]*pb.InvolvedCompany, 0, len(involvedCompanies))
-	for _, item := range involvedCompanies {
-		res.InvolvedCompanies = append(res.InvolvedCompanies, item.Item)
-	}
+	res.InvolvedCompanies = involvedCompanies
 
 	keywordIds := make([]uint64, 0, len(game.Keywords))
 	for _, g := range game.Keywords {
 		keywordIds = append(keywordIds, g.Id)
 	}
-	keyword, err := GetItemsByIGDBIDs[pb.Keyword](endpoint.EPKeywords, keywordIds)
+	keyword, err := GetItemsByIds[pb.Keyword](endpoint.EPKeywords, keywordIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Keywords = make([]*pb.Keyword, 0, len(keyword))
-	for _, item := range keyword {
-		res.Keywords = append(res.Keywords, item.Item)
-	}
+	res.Keywords = keyword
 
 	multiplayerModeIds := make([]uint64, 0, len(game.MultiplayerModes))
 	for _, g := range game.MultiplayerModes {
 		multiplayerModeIds = append(multiplayerModeIds, g.Id)
 	}
-	multiplayerModes, err := GetItemsByIGDBIDs[pb.MultiplayerMode](endpoint.EPMultiplayerModes, multiplayerModeIds)
+	multiplayerModes, err := GetItemsByIds[pb.MultiplayerMode](endpoint.EPMultiplayerModes, multiplayerModeIds)
 	if err != nil {
 		return nil, err
 	}
-	res.MultiplayerModes = make([]*pb.MultiplayerMode, 0, len(multiplayerModes))
-	for _, item := range multiplayerModes {
-		res.MultiplayerModes = append(res.MultiplayerModes, item.Item)
-	}
+	res.MultiplayerModes = multiplayerModes
 
 	res.Name = game.Name
 
@@ -266,27 +230,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.Platforms {
 		platformIds = append(platformIds, g.Id)
 	}
-	platforms, err := GetItemsByIGDBIDs[pb.Platform](endpoint.EPPlatforms, platformIds)
+	platforms, err := GetItemsByIds[pb.Platform](endpoint.EPPlatforms, platformIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Platforms = make([]*pb.Platform, 0, len(platforms))
-	for _, item := range platforms {
-		res.Platforms = append(res.Platforms, item.Item)
-	}
+	res.Platforms = platforms
 
 	playerPerspectiveIds := make([]uint64, 0, len(game.PlayerPerspectives))
 	for _, g := range game.PlayerPerspectives {
 		playerPerspectiveIds = append(playerPerspectiveIds, g.Id)
 	}
-	playerPerspectives, err := GetItemsByIGDBIDs[pb.PlayerPerspective](endpoint.EPPlayerPerspectives, playerPerspectiveIds)
+	playerPerspectives, err := GetItemsByIds[pb.PlayerPerspective](endpoint.EPPlayerPerspectives, playerPerspectiveIds)
 	if err != nil {
 		return nil, err
 	}
-	res.PlayerPerspectives = make([]*pb.PlayerPerspective, 0, len(playerPerspectives))
-	for _, item := range playerPerspectives {
-		res.PlayerPerspectives = append(res.PlayerPerspectives, item.Item)
-	}
+	res.PlayerPerspectives = playerPerspectives
 
 	res.Rating = game.Rating
 	res.RatingCount = game.RatingCount
@@ -295,27 +253,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.ReleaseDates {
 		releaseDateIds = append(releaseDateIds, g.Id)
 	}
-	releaseDates, err := GetItemsByIGDBIDs[pb.ReleaseDate](endpoint.EPReleaseDates, releaseDateIds)
+	releaseDates, err := GetItemsByIds[pb.ReleaseDate](endpoint.EPReleaseDates, releaseDateIds)
 	if err != nil {
 		return nil, err
 	}
-	res.ReleaseDates = make([]*pb.ReleaseDate, 0, len(releaseDates))
-	for _, item := range releaseDates {
-		res.ReleaseDates = append(res.ReleaseDates, item.Item)
-	}
+	res.ReleaseDates = releaseDates
 
 	screenshotIds := make([]uint64, 0, len(game.Screenshots))
 	for _, g := range game.Screenshots {
 		screenshotIds = append(screenshotIds, g.Id)
 	}
-	screenshots, err := GetItemsByIGDBIDs[pb.Screenshot](endpoint.EPScreenshots, screenshotIds)
+	screenshots, err := GetItemsByIds[pb.Screenshot](endpoint.EPScreenshots, screenshotIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Screenshots = make([]*pb.Screenshot, 0, len(screenshots))
-	for _, item := range screenshots {
-		res.Screenshots = append(res.Screenshots, item.Item)
-	}
+	res.Screenshots = screenshots
 
 	similarGamesIds := make([]uint64, 0, len(game.SimilarGames))
 	for _, g := range game.SimilarGames {
@@ -340,14 +292,11 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.Themes {
 		themeIds = append(themeIds, g.Id)
 	}
-	themes, err := GetItemsByIGDBIDs[pb.Theme](endpoint.EPThemes, themeIds)
+	themes, err := GetItemsByIds[pb.Theme](endpoint.EPThemes, themeIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Themes = make([]*pb.Theme, 0, len(themes))
-	for _, item := range themes {
-		res.Themes = append(res.Themes, item.Item)
-	}
+	res.Themes = themes
 
 	res.TotalRating = game.TotalRating
 	res.TotalRatingCount = game.TotalRatingCount
@@ -366,27 +315,21 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.Videos {
 		videoIds = append(videoIds, g.Id)
 	}
-	videos, err := GetItemsByIGDBIDs[pb.GameVideo](endpoint.EPGameVideos, videoIds)
+	videos, err := GetItemsByIds[pb.GameVideo](endpoint.EPGameVideos, videoIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Videos = make([]*pb.GameVideo, 0, len(videos))
-	for _, item := range videos {
-		res.Videos = append(res.Videos, item.Item)
-	}
+	res.Videos = videos
 
 	websiteIds := make([]uint64, 0, len(game.Websites))
 	for _, g := range game.Websites {
 		websiteIds = append(websiteIds, g.Id)
 	}
-	websites, err := GetItemsByIGDBIDs[pb.Website](endpoint.EPWebsites, websiteIds)
+	websites, err := GetItemsByIds[pb.Website](endpoint.EPWebsites, websiteIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Websites = make([]*pb.Website, 0, len(websites))
-	for _, item := range websites {
-		res.Websites = append(res.Websites, item.Item)
-	}
+	res.Websites = websites
 
 	remakesIds := make([]uint64, 0, len(game.Remakes))
 	for _, g := range game.Remakes {
@@ -422,40 +365,31 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	for _, g := range game.LanguageSupports {
 		languageSupportIds = append(languageSupportIds, g.Id)
 	}
-	languageSupports, err := GetItemsByIGDBIDs[pb.LanguageSupport](endpoint.EPLanguageSupports, languageSupportIds)
+	languageSupports, err := GetItemsByIds[pb.LanguageSupport](endpoint.EPLanguageSupports, languageSupportIds)
 	if err != nil {
 		return nil, err
 	}
-	res.LanguageSupports = make([]*pb.LanguageSupport, 0, len(languageSupports))
-	for _, item := range languageSupports {
-		res.LanguageSupports = append(res.LanguageSupports, item.Item)
-	}
+	res.LanguageSupports = languageSupports
 
 	gameLocalizationIds := make([]uint64, 0, len(game.GameLocalizations))
 	for _, g := range game.GameLocalizations {
 		gameLocalizationIds = append(gameLocalizationIds, g.Id)
 	}
-	gameLocalizations, err := GetItemsByIGDBIDs[pb.GameLocalization](endpoint.EPGameLocalizations, gameLocalizationIds)
+	gameLocalizations, err := GetItemsByIds[pb.GameLocalization](endpoint.EPGameLocalizations, gameLocalizationIds)
 	if err != nil {
 		return nil, err
 	}
-	res.GameLocalizations = make([]*pb.GameLocalization, 0, len(gameLocalizations))
-	for _, item := range gameLocalizations {
-		res.GameLocalizations = append(res.GameLocalizations, item.Item)
-	}
+	res.GameLocalizations = gameLocalizations
 
 	collectionIds := make([]uint64, 0, len(game.Collections))
 	for _, g := range game.Collections {
 		collectionIds = append(collectionIds, g.Id)
 	}
-	collections, err := GetItemsByIGDBIDs[pb.Collection](endpoint.EPCollections, collectionIds)
+	collections, err := GetItemsByIds[pb.Collection](endpoint.EPCollections, collectionIds)
 	if err != nil {
 		return nil, err
 	}
-	res.Collections = make([]*pb.Collection, 0, len(collections))
-	for _, item := range collections {
-		res.Collections = append(res.Collections, item.Item)
-	}
+	res.Collections = collections
 
 	res.GameStatus = nil
 	res.GameType = nil
@@ -463,13 +397,13 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
 	res.AllNames = make([]string, 0, len(alternativeNames)+1)
 	res.AllNames = append(res.AllNames, game.Name)
 	for _, item := range alternativeNames {
-		res.AllNames = append(res.AllNames, item.Item.Name)
+		res.AllNames = append(res.AllNames, item.Name)
 	}
 
 	return res, nil
 }
 
-func GetGameByIGDBID(id uint64) (*model.Game, error) {
+func GetGameById(id uint64) (*model.Game, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancel()
 
@@ -480,3 +414,33 @@ func GetGameByIGDBID(id uint64) (*model.Game, error) {
 	}
 	return &game, nil
 }
+
+func GetAllItemsIDs[T any](e endpoint.Name) ([]uint64, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	var ids []uint64
+	coll := GetInstance().Collections[e]
+	if coll == nil {
+		return nil, fmt.Errorf("collection not found")
+	}
+	cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetProjection(bson.M{"id": 1}))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get items: %w", err)
+	}
+
+	type IdGetter interface {
+		GetId() uint64
+	}
+
+	for cursor.Next(ctx) {
+		var item *T
+		err := cursor.Decode(&item)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode item: %w", err)
+		}
+		ids = append(ids, any(item).(IdGetter).GetId())
+	}
+
+	return ids, nil
+}

main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"igdb-database/collector"
 	"igdb-database/config"
@@ -13,7 +12,6 @@ import (
 	"github.com/bestnite/go-igdb"
 	"github.com/bestnite/go-igdb/endpoint"
 	pb "github.com/bestnite/go-igdb/proto"
-	"go.mongodb.org/mongo-driver/v2/mongo"
 )
 
 var (
@@ -22,6 +20,7 @@ var (
 	enableReFetch     = flag.Bool("re-fetch", false, "re fetch data even if collection is not empty")
 	enableReAggregate = flag.Bool("re-aggregate", false, "re aggregate games even if game_details is not empty")
 	enableWebhook     = flag.Bool("webhook", true, "start webhook server")
+	onlyRefetchGames  = flag.Bool("only-refetch-games", false, "only refetch games")
 )
 
 func main() {
@@ -35,6 +34,8 @@ func main() {
 		log.Printf("data fetched")
 	}
 
+	Count()
+
 	if *enableAggregate || *enableReAggregate {
 		log.Printf("aggregating games")
 		aggregateGames()
@@ -48,10 +49,11 @@ func main() {
 }
 
 func aggregateGames() {
-	total, err := db.CountItems(endpoint.EPGames)
+	total, err := db.EstimatedDocumentCount(endpoint.EPGames)
 	if err != nil {
 		log.Fatalf("failed to count games: %v", err)
 	}
+	log.Printf("games length: %d", total)
 
 	finished := int64(0)
 	wg := sync.WaitGroup{}
@@ -71,39 +73,28 @@ func aggregateGames() {
 			if err != nil {
 				log.Fatalf("failed to get games: %v", err)
 			}
-			games := make([]*pb.Game, 0, len(items))
-			for _, item := range items {
-				games = append(games, item.Item)
-			}
-			isAggregated := make(map[uint64]bool, len(games))
+			isAggregated := make(map[uint64]bool, len(items))
 			if !*enableReAggregate {
-				isAggregated, err = db.IsGamesAggregated(games)
+				isAggregated, err = db.IsGamesAggregated(items)
 				if err != nil {
 					log.Fatalf("failed to check if games are aggregated: %v", err)
 				}
 			} else {
-				for _, game := range games {
+				for _, game := range items {
 					isAggregated[game.Id] = false
 				}
 			}
 			for _, item := range items {
-				if isAggregated[item.Item.Id] {
+				if isAggregated[item.Id] {
 					p := atomic.AddInt64(&finished, 1)
 					log.Printf("game aggregated %d/%d", p, total)
 					continue
 				}
 
-				game, err := db.ConvertGame(item.Item)
+				game, err := db.ConvertGame(item)
 				if err != nil {
 					log.Fatalf("failed to convert game: %v", err)
 				}
-				oldGame, err := db.GetGameByIGDBID(item.Item.Id)
-				if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
-					log.Fatalf("failed to get game: %v", err)
-				}
-				if oldGame != nil {
-					game.MId = oldGame.MId
-				}
 
 				err = db.SaveGame(game)
 				if err != nil {
@@ -120,14 +111,76 @@ func aggregateGames() {
 func fetchAndStore[T any](
 	e endpoint.EntityEndpoint[T],
 ) {
-	if count, err := db.CountItems(e.GetEndpointName()); (err == nil && count == 0) || *enableReFetch {
+	if count, err := db.EstimatedDocumentCount(e.GetEndpointName()); (err == nil && count == 0) || *enableReFetch {
 		collector.FetchAndStore(e)
 	} else if err != nil {
 		log.Printf("failed to count items: %v", err)
 	}
 }
 
+func Count() {
+	ids, err := db.GetAllItemsIDs[pb.Theme](endpoint.EPThemes)
+	if err != nil {
+		log.Fatalf("failed to get all items ids %s: %v", endpoint.EPThemes, err)
+	}
+
+	concurrence := make(chan struct{}, 10)
+	defer close(concurrence)
+	wg := sync.WaitGroup{}
+
+	for _, id := range ids {
+		concurrence <- struct{}{}
+		wg.Add(1)
+		go func(id uint64) {
+			defer func() {
+				<-concurrence
+				wg.Done()
+			}()
+			count, err := db.CountTheme(id)
+			if err != nil {
+				log.Fatalf("failed to count theme: %v", err)
+			}
+			log.Printf("theme %d count: %d", id, count)
+			err = db.SaveCount(&db.Count{Theme: id, Count: count})
+			if err != nil {
+				log.Fatalf("failed to save count: %v", err)
+			}
+		}(id)
+	}
+
+	ids, err = db.GetAllItemsIDs[pb.Genre](endpoint.EPGenres)
+	if err != nil {
+		log.Fatalf("failed to get all items ids %s: %v", endpoint.EPGenres, err)
+	}
+	for _, id := range ids {
+		concurrence <- struct{}{}
+		wg.Add(1)
+		go func(id uint64) {
+			defer func() {
+				<-concurrence
+				wg.Done()
+			}()
+			count, err := db.CountGenre(id)
+			if err != nil {
+				log.Fatalf("failed to count genre: %v", err)
+			}
+			log.Printf("genre %d count: %d", id, count)
+			err = db.SaveCount(&db.Count{Genre: id, Count: count})
+			if err != nil {
+				log.Fatalf("failed to save count: %v", err)
+			}
+		}(id)
+	}
+
+	wg.Wait()
+}
+
 func allFetchAndStore(client *igdb.Client) {
+	if *onlyRefetchGames {
+		fetchAndStore(client.Games)
+		return
+	}
+
 	fetchAndStore(client.AgeRatingCategories)
 	fetchAndStore(client.AgeRatingContentDescriptions)
 	fetchAndStore(client.AgeRatingContentDescriptionsV2)

model/game.go

@@ -2,7 +2,6 @@ package model
 
 import (
 	pb "github.com/bestnite/go-igdb/proto"
-	"go.mongodb.org/mongo-driver/v2/bson"
 	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
@@ -10,7 +9,6 @@ type GameIds []uint64
 type GameId uint64
 
 type Game struct {
-	MId                   bson.ObjectID           `json:"_id,omitempty"`
 	Id                    uint64                  `json:"id,omitempty"`
 	AgeRatings            []*pb.AgeRating         `json:"age_ratings,omitempty"`
 	AggregatedRating      float64                 `json:"aggregated_rating,omitempty"`

model/model.go

@@ -1,24 +1,20 @@
 package model
 
-import (
-	"go.mongodb.org/mongo-driver/v2/bson"
-)
+// type Item[T any] struct {
+// 	Item *T            `bson:"item"`
+// 	MId  bson.ObjectID `bson:"_id"`
+// }
 
-type Item[T any] struct {
-	Item *T            `bson:"item"`
-	MId  bson.ObjectID `bson:"_id"`
-}
+// func NewItem[T any](item *T) *Item[T] {
+// 	return &Item[T]{
+// 		Item: item,
+// 	}
+// }
 
-func NewItem[T any](item *T) *Item[T] {
-	return &Item[T]{
-		Item: item,
-	}
-}
-
-func NewItems[T any](items []*T) []*Item[T] {
-	var result []*Item[T]
-	for _, item := range items {
-		result = append(result, NewItem(item))
-	}
-	return result
-}
+// func NewItems[T any](items []*T) []*Item[T] {
+// 	var result []*Item[T]
+// 	for _, item := range items {
+// 		result = append(result, NewItem(item))
+// 	}
+// 	return result
+// }