mirror of
https://github.com/bestnite/igdb-database.git
synced 2025-06-17 10:13:18 +08:00
u
This commit is contained in:
328
db/db.go
328
db/db.go
@ -60,199 +60,71 @@ func (m *MongoDB) createIndex() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
textIndexMap := map[endpoint.EndpointName]string{
|
||||
endpoint.EPGames: "item.name",
|
||||
endpoint.EPAlternativeNames: "item.name",
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.name", Value: "text"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
for e, idx := range textIndexMap {
|
||||
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: idx, Value: "text"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.name", Value: "text"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
indexMap := map[endpoint.EndpointName][]string{
|
||||
endpoint.EPAlternativeNames: {"item.game.id"},
|
||||
endpoint.EPArtworks: {"item.game.id"},
|
||||
endpoint.EPCollectionMemberships: {"item.game.id"},
|
||||
endpoint.EPCovers: {"item.game.id"},
|
||||
endpoint.EPExternalGames: {"item.game.id"},
|
||||
endpoint.EPGameEngines: {"item.game.id"},
|
||||
endpoint.EPGameLocalizations: {"item.game.id"},
|
||||
endpoint.EPGameVersions: {"item.game.id"},
|
||||
endpoint.EPGameVersionFeatureValues: {"item.game.id"},
|
||||
endpoint.EPGameVideos: {"item.game.id"},
|
||||
endpoint.EPInvolvedCompanies: {"item.game.id"},
|
||||
endpoint.EPLanguageSupports: {"item.game.id"},
|
||||
endpoint.EPMultiplayerModes: {"item.game.id"},
|
||||
endpoint.EPReleaseDates: {"item.game.id"},
|
||||
endpoint.EPScreenshots: {"item.game.id"},
|
||||
endpoint.EPWebsites: {"item.game.id"},
|
||||
endpoint.EPGames: {"item.parent_game.id", "item.version_parent.id"},
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
for e, idxes := range indexMap {
|
||||
for _, idx := range idxes {
|
||||
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: idx, Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPArtworks].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPCollectionMemberships].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPCovers].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPExternalGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.parent_game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.version_parent.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameEngines].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameLocalizations].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameVersions].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameVersionFeatureValues].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameVideos].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPInvolvedCompanies].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPLanguageSupports].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPMultiplayerModes].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPReleaseDates].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPScreenshots].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPWebsites].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
for _, e := range endpoint.AllEndpoints {
|
||||
if e == endpoint.EPWebhooks || e == endpoint.EPSearch || e == endpoint.EPPopularityPrimitives {
|
||||
continue
|
||||
}
|
||||
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index item.id for %s: %v", string(e), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if item.MId.IsZero() {
|
||||
item.MId = bson.NewObjectID()
|
||||
}
|
||||
@ -265,17 +137,16 @@ func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := coll.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save item to %s: %v", string(e), err)
|
||||
return fmt.Errorf("failed to save item to %s: %w", string(e), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
var models []mongo.WriteModel
|
||||
|
||||
for _, item := range items {
|
||||
@ -293,9 +164,11 @@ func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := coll.BulkWrite(ctx, models)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save items in bulk %s: %v", string(e), err)
|
||||
return fmt.Errorf("failed to save items in bulk %s: %w", string(e), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -309,23 +182,22 @@ func CountItems(e endpoint.EndpointName) (int64, error) {
|
||||
defer cancel()
|
||||
count, err := coll.EstimatedDocumentCount(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count items %s: %v", string(e), err)
|
||||
return 0, fmt.Errorf("failed to count items %s: %w", string(e), err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func GetItemByIGDBID[T any](e endpoint.EndpointName, id uint64) (*model.Item[T], error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var item model.Item[T]
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get item %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get item %s: %w", string(e), err)
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
@ -334,33 +206,35 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second+time.Duration(len(ids))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
type IdGetter interface {
|
||||
GetId() uint64
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), time.Duration(len(ids))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
res := make(map[uint64]*model.Item[T])
|
||||
for cursor.Next(ctx) {
|
||||
item := model.Item[T]{}
|
||||
err := cursor.Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to decode item %s: %w", string(e), err)
|
||||
}
|
||||
if v, ok := any(item.Item).(IdGetter); ok {
|
||||
res[v.GetId()] = &item
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to get id from item %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get id from item %s: %w", string(e), err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -368,31 +242,31 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
|
||||
}
|
||||
|
||||
func RemoveItemByID(e endpoint.EndpointName, id bson.ObjectID) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := coll.DeleteOne(ctx, bson.M{"_id": id})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove game: %v", err)
|
||||
return fmt.Errorf("failed to remove game: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RemoveItemsByID(e endpoint.EndpointName, ids []bson.ObjectID) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := coll.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove games: %v", err)
|
||||
return fmt.Errorf("failed to remove games: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -422,11 +296,11 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
|
||||
}},
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
cursor, err := coll.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to aggregate: %v", err)
|
||||
return fmt.Errorf("failed to aggregate: %w", err)
|
||||
}
|
||||
var results []struct {
|
||||
ID uint64 `bson:"_id"`
|
||||
@ -434,7 +308,7 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
|
||||
}
|
||||
err = cursor.All(ctx, &results)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get results: %v", err)
|
||||
return fmt.Errorf("failed to get results: %w", err)
|
||||
}
|
||||
|
||||
removedIds := make([]bson.ObjectID, 0, len(results))
|
||||
@ -445,59 +319,55 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
|
||||
|
||||
err = RemoveItemsByID(e, removedIds)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove duplicate games: %v", err)
|
||||
return fmt.Errorf("failed to remove duplicate games: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetItemsByIGDBGameID[T any](e endpoint.EndpointName, id uint64) ([]*model.Item[T], error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cursor, err := coll.Find(ctx, bson.M{"item.game.id": id})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var items []*model.Item[T]
|
||||
for cursor.Next(ctx) {
|
||||
item := model.Item[T]{}
|
||||
err := cursor.Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
|
||||
}
|
||||
items = append(items, &item)
|
||||
err = cursor.All(ctx, &items)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func GetItemsPagnated[T any](e endpoint.EndpointName, offset int64, limit int64) ([]*model.Item[T], error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
|
||||
defer cancel()
|
||||
cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
var items []*model.Item[T]
|
||||
for cursor.Next(ctx) {
|
||||
item := model.Item[T]{}
|
||||
err := cursor.Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
|
||||
}
|
||||
items = append(items, &item)
|
||||
err = cursor.All(ctx, &items)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
return items, nil
|
||||
|
||||
Reference in New Issue
Block a user