nite/igdb-database
1
Fork 0
mirror of https://github.com/bestnite/igdb-database.git synced 2025-06-16 09:53:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

u

Browse Source
This commit is contained in:
nite
2025-04-07 15:36:08 +10:00
parent 0214d95d3e
commit 4508b19f4b
5 changed files with 257 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
db/db.go
View File

@ -60,199 +60,71 @@ func (m *MongoDB) createIndex() {
ctx, cancel := context.WithTimeout(context.Background(), 3*60*time.Second)
defer cancel()
_, err := m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
textIndexMap := map[endpoint.EndpointName]string{
endpoint.EPGames: "item.name",
endpoint.EPAlternativeNames: "item.name",
}
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.name", Value: "text"},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
for e, idx := range textIndexMap {
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: idx, Value: "text"},
},
})
if err != nil {
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
}
}
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.name", Value: "text"},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
indexMap := map[endpoint.EndpointName][]string{
endpoint.EPAlternativeNames: {"item.game.id"},
endpoint.EPArtworks: {"item.game.id"},
endpoint.EPCollectionMemberships: {"item.game.id"},
endpoint.EPCovers: {"item.game.id"},
endpoint.EPExternalGames: {"item.game.id"},
endpoint.EPGameEngines: {"item.game.id"},
endpoint.EPGameLocalizations: {"item.game.id"},
endpoint.EPGameVersions: {"item.game.id"},
endpoint.EPGameVersionFeatureValues: {"item.game.id"},
endpoint.EPGameVideos: {"item.game.id"},
endpoint.EPInvolvedCompanies: {"item.game.id"},
endpoint.EPLanguageSupports: {"item.game.id"},
endpoint.EPMultiplayerModes: {"item.game.id"},
endpoint.EPReleaseDates: {"item.game.id"},
endpoint.EPScreenshots: {"item.game.id"},
endpoint.EPWebsites: {"item.game.id"},
endpoint.EPGames: {"item.parent_game.id", "item.version_parent.id"},
}
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
for e, idxes := range indexMap {
for _, idx := range idxes {
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: idx, Value: 1},
},
})
if err != nil {
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
}
}
}
_, err = m.Collections[endpoint.EPArtworks].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPCollectionMemberships].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPCovers].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPExternalGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.parent_game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.version_parent.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameEngines].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameLocalizations].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameVersions].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameVersionFeatureValues].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameVideos].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPInvolvedCompanies].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPLanguageSupports].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPMultiplayerModes].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPReleaseDates].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPScreenshots].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPWebsites].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
for _, e := range endpoint.AllEndpoints {
if e == endpoint.EPWebhooks || e == endpoint.EPSearch || e == endpoint.EPPopularityPrimitives {
continue
}
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index item.id for %s: %v", string(e), err)
}
}
}
func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if item.MId.IsZero() {
item.MId = bson.NewObjectID()
}
@ -265,17 +137,16 @@ func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
return fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := coll.UpdateOne(ctx, filter, update, opts)
if err != nil {
return fmt.Errorf("failed to save item to %s: %v", string(e), err)
return fmt.Errorf("failed to save item to %s: %w", string(e), err)
}
return nil
}
func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
defer cancel()
var models []mongo.WriteModel
for _, item := range items {
@ -293,9 +164,11 @@ func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
return fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
defer cancel()
_, err := coll.BulkWrite(ctx, models)
if err != nil {
return fmt.Errorf("failed to save items in bulk %s: %v", string(e), err)
return fmt.Errorf("failed to save items in bulk %s: %w", string(e), err)
}
return nil
}
@ -309,23 +182,22 @@ func CountItems(e endpoint.EndpointName) (int64, error) {
defer cancel()
count, err := coll.EstimatedDocumentCount(ctx)
if err != nil {
return 0, fmt.Errorf("failed to count items %s: %v", string(e), err)
return 0, fmt.Errorf("failed to count items %s: %w", string(e), err)
}
return count, nil
}
func GetItemByIGDBID[T any](e endpoint.EndpointName, id uint64) (*model.Item[T], error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var item model.Item[T]
coll := GetInstance().Collections[e]
if coll == nil {
return nil, fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item)
if err != nil {
return nil, fmt.Errorf("failed to get item %s: %v", string(e), err)
return nil, fmt.Errorf("failed to get item %s: %w", string(e), err)
}
return &item, nil
}
@ -334,33 +206,35 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
if len(ids) == 0 {
return nil, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second+time.Duration(len(ids))*200*time.Millisecond)
defer cancel()
coll := GetInstance().Collections[e]
if coll == nil {
return nil, fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}})
if err != nil {
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
}
type IdGetter interface {
GetId() uint64
}
ctx, cancel = context.WithTimeout(context.Background(), time.Duration(len(ids))*200*time.Millisecond)
defer cancel()
res := make(map[uint64]*model.Item[T])
for cursor.Next(ctx) {
item := model.Item[T]{}
err := cursor.Decode(&item)
if err != nil {
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
return nil, fmt.Errorf("failed to decode item %s: %w", string(e), err)
}
if v, ok := any(item.Item).(IdGetter); ok {
res[v.GetId()] = &item
} else {
return nil, fmt.Errorf("failed to get id from item %s: %v", string(e), err)
return nil, fmt.Errorf("failed to get id from item %s: %w", string(e), err)
}
}
@ -368,31 +242,31 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
}
func RemoveItemByID(e endpoint.EndpointName, id bson.ObjectID) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
coll := GetInstance().Collections[e]
if coll == nil {
return fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := coll.DeleteOne(ctx, bson.M{"_id": id})
if err != nil {
return fmt.Errorf("failed to remove game: %v", err)
return fmt.Errorf("failed to remove game: %w", err)
}
return nil
}
func RemoveItemsByID(e endpoint.EndpointName, ids []bson.ObjectID) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
coll := GetInstance().Collections[e]
if coll == nil {
return fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := coll.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
if err != nil {
return fmt.Errorf("failed to remove games: %v", err)
return fmt.Errorf("failed to remove games: %w", err)
}
return nil
}
@ -422,11 +296,11 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
}},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
cursor, err := coll.Aggregate(ctx, pipeline)
if err != nil {
return fmt.Errorf("failed to aggregate: %v", err)
return fmt.Errorf("failed to aggregate: %w", err)
}
var results []struct {
ID uint64 `bson:"_id"`
@ -434,7 +308,7 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
}
err = cursor.All(ctx, &results)
if err != nil {
return fmt.Errorf("failed to get results: %v", err)
return fmt.Errorf("failed to get results: %w", err)
}
removedIds := make([]bson.ObjectID, 0, len(results))
@ -445,59 +319,55 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
err = RemoveItemsByID(e, removedIds)
if err != nil {
return fmt.Errorf("failed to remove duplicate games: %v", err)
return fmt.Errorf("failed to remove duplicate games: %w", err)
}
return nil
}
func GetItemsByIGDBGameID[T any](e endpoint.EndpointName, id uint64) ([]*model.Item[T], error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
coll := GetInstance().Collections[e]
if coll == nil {
return nil, fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cursor, err := coll.Find(ctx, bson.M{"item.game.id": id})
if err != nil {
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
}
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var items []*model.Item[T]
for cursor.Next(ctx) {
item := model.Item[T]{}
err := cursor.Decode(&item)
if err != nil {
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
}
items = append(items, &item)
err = cursor.All(ctx, &items)
if err != nil {
return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
}
return items, nil
}
func GetItemsPagnated[T any](e endpoint.EndpointName, offset int64, limit int64) ([]*model.Item[T], error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
defer cancel()
coll := GetInstance().Collections[e]
if coll == nil {
return nil, fmt.Errorf("collection not found")
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
defer cancel()
cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}}))
if err != nil {
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
}
var items []*model.Item[T]
for cursor.Next(ctx) {
item := model.Item[T]{}
err := cursor.Decode(&item)
if err != nil {
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
}
items = append(items, &item)
err = cursor.All(ctx, &items)
if err != nil {
return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
}
return items, nil

29
db/game.go
View File

@ -2,6 +2,7 @@ package db
import (
"context"
"errors"
"fmt"
"igdb-database/model"
"time"
@ -9,28 +10,28 @@ import (
"github.com/bestnite/go-igdb/endpoint"
pb "github.com/bestnite/go-igdb/proto"
"go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
)
func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond)
defer cancel()
ids := make([]uint64, 0, len(games))
for _, game := range games {
ids = append(ids, game.Id)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond)
defer cancel()
cursor, err := GetInstance().GameCollection.Find(ctx, bson.M{"id": bson.M{"$in": ids}})
if err != nil {
return nil, fmt.Errorf("failed to get games: %v", err)
return nil, fmt.Errorf("failed to get games: %w", err)
}
res := make(map[uint64]bool, len(games))
g := []*model.Game{}
err = cursor.All(ctx, &g)
if err != nil {
return nil, fmt.Errorf("failed to get games: %v", err)
return nil, fmt.Errorf("failed to get games: %w", err)
}
for _, game := range g {
res[game.Id] = true
@ -40,8 +41,6 @@ func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
}
func SaveGame(game *model.Game) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if game.MId.IsZero() {
game.MId = bson.NewObjectID()
}
@ -49,6 +48,8 @@ func SaveGame(game *model.Game) error {
update := bson.M{"$set": game}
opts := options.UpdateOne().SetUpsert(true)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := GetInstance().GameCollection.UpdateOne(ctx, filter, update, opts)
if err != nil {
return err
@ -112,10 +113,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
if game.Cover != nil {
coverId := game.Cover.Id
cover, err := GetItemByIGDBID[pb.Cover](endpoint.EPCovers, coverId)
if err != nil {
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return nil, err
}
res.Cover = cover.Item
if cover != nil {
res.Cover = cover.Item
}
}
res.CreatedAt = game.CreatedAt
@ -152,10 +155,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
if game.Franchise != nil {
franchiseId := game.Franchise.Id
franchise, err := GetItemByIGDBID[pb.Franchise](endpoint.EPFranchises, franchiseId)
if err != nil {
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return nil, err
}
res.Franchise = franchise.Item
if franchise != nil {
res.Franchise = franchise.Item
}
}
franchiseIds := make([]uint64, 0, len(game.Franchises))
@ -471,7 +476,7 @@ func GetGameByIGDBID(id uint64) (*model.Game, error) {
var game model.Game
err := GetInstance().GameCollection.FindOne(ctx, bson.M{"id": id}).Decode(&game)
if err != nil {
return nil, fmt.Errorf("failed to get game: %v", err)
return nil, fmt.Errorf("failed to get game: %w", err)
}
return &game, nil
}