nite/igdb-database
1
Fork 0
mirror of https://github.com/bestnite/igdb-database.git synced 2025-06-16 09:53:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

u

Browse Source
This commit is contained in:
nite
2025-04-07 15:36:08 +10:00
parent 0214d95d3e
commit 4508b19f4b
5 changed files with 257 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
collector/store.go
View File

@ -33,9 +33,10 @@ func FetchAndStore[T any](
defer wg.Done() defer wg.Done()
defer func() { <-concurrence }() defer func() { <-concurrence }()
// items data from igdb
items, err := e.Paginated(uint64(i), 500) items, err := e.Paginated(uint64(i), 500)
if err != nil { if err != nil {
log.Printf("failed to get items: %v", err) log.Printf("failed to get items from igdb %s: %v", e.GetEndpointName(), err)
return return
} }
@ -48,14 +49,15 @@ func FetchAndStore[T any](
if v, ok := any(item).(IdGetter); ok { if v, ok := any(item).(IdGetter); ok {
ids = append(ids, v.GetId()) ids = append(ids, v.GetId())
} else { } else {
log.Printf("failed to get id from item: %v", err) log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err)
return return
} }
} }
// items data from database
data, err := db.GetItemsByIGDBIDs[T](e.GetEndpointName(), ids) data, err := db.GetItemsByIGDBIDs[T](e.GetEndpointName(), ids)
if err != nil { if err != nil {
log.Printf("failed to get items: %v", err) log.Printf("failed to get items from database %s: %v", e.GetEndpointName(), err)
return return
} }
@ -63,7 +65,7 @@ func FetchAndStore[T any](
for _, item := range items { for _, item := range items {
v, ok := any(item).(IdGetter) v, ok := any(item).(IdGetter)
if !ok { if !ok {
log.Printf("failed to get id from item: %v", err) log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err)
return return
} else { } else {
if data[v.GetId()] == nil { if data[v.GetId()] == nil {
@ -76,7 +78,7 @@ func FetchAndStore[T any](
} }
err = db.SaveItems(e.GetEndpointName(), newItems) err = db.SaveItems(e.GetEndpointName(), newItems)
if err != nil { if err != nil {
log.Printf("failed to save games: %v", err) log.Printf("failed to save games %s: %v", e.GetEndpointName(), err)
return return
} }

34
collector/webhook.go
View File

@ -2,6 +2,7 @@ package collector
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"igdb-database/config" "igdb-database/config"
"igdb-database/db" "igdb-database/db"
@ -94,6 +95,22 @@ func StartWebhookServer(client *igdb.Client) {
http.HandleFunc(webhook(client.Themes)) http.HandleFunc(webhook(client.Themes))
http.HandleFunc(webhook(client.Websites)) http.HandleFunc(webhook(client.Websites))
http.HandleFunc(webhook(client.WebsiteTypes)) http.HandleFunc(webhook(client.WebsiteTypes))
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200)
if _, err := w.Write([]byte("Hello World!")); err != nil {
log.Printf("failed to write response: %v", err)
}
})
serverStart := make(chan bool)
go func() {
defer close(serverStart)
log.Printf("starting webhook server on %s", config.C().Address)
err = http.ListenAndServe(config.C().Address, nil)
if err != nil {
log.Fatalf("failed to start webhook server: %v", err)
}
}()
enabledEndpoint := endpoint.AllEndpoints enabledEndpoint := endpoint.AllEndpoints
enabledEndpoint = slices.DeleteFunc(enabledEndpoint, func(e endpoint.EndpointName) bool { enabledEndpoint = slices.DeleteFunc(enabledEndpoint, func(e endpoint.EndpointName) bool {
@ -116,18 +133,7 @@ func StartWebhookServer(client *igdb.Client) {
log.Printf("all webhook registered") log.Printf("all webhook registered")
} }
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { <-serverStart
w.WriteHeader(200)
if _, err := w.Write([]byte("Hello World!")); err != nil {
log.Printf("failed to write response: %v", err)
}
})
log.Printf("starting webhook server on %s", config.C().Address)
err = http.ListenAndServe(config.C().Address, nil)
if err != nil {
log.Fatalf("failed to start webhook server: %v", err)
}
} }
func webhook[T any]( func webhook[T any](
@ -162,7 +168,7 @@ func webhook[T any](
return return
} }
oldItem, err := db.GetItemByIGDBID[T](e.GetEndpointName(), data.ID) oldItem, err := db.GetItemByIGDBID[T](e.GetEndpointName(), data.ID)
if err != nil && err != mongo.ErrNoDocuments { if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
log.Printf("failed to get %s: %v", e.GetEndpointName(), err) log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
return return
} }
@ -183,6 +189,7 @@ func webhook[T any](
if err == nil { if err == nil {
g.MId = game.MId g.MId = game.MId
_ = db.SaveGame(g) _ = db.SaveGame(g)
log.Printf("game %d aggregated", data.ID)
} }
} }
} }
@ -216,6 +223,7 @@ func webhook[T any](
log.Printf("failed to save game: %v", err) log.Printf("failed to save game: %v", err)
goto END goto END
} }
log.Printf("game %d aggregated", data.ID)
} }
END: END:

328
db/db.go
View File

@ -60,199 +60,71 @@ func (m *MongoDB) createIndex() {
ctx, cancel := context.WithTimeout(context.Background(), 3*60*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 3*60*time.Second)
defer cancel() defer cancel()
_, err := m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{ textIndexMap := map[endpoint.EndpointName]string{
Keys: bson.D{ endpoint.EPGames: "item.name",
{Key: "item.id", Value: 1}, endpoint.EPAlternativeNames: "item.name",
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
} }
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{ for e, idx := range textIndexMap {
Keys: bson.D{ _, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
{Key: "item.name", Value: "text"}, Keys: bson.D{
}, {Key: idx, Value: "text"},
}) },
if err != nil { })
log.Printf("failed to create index: %v", err) if err != nil {
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
}
} }
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{ indexMap := map[endpoint.EndpointName][]string{
Keys: bson.D{ endpoint.EPAlternativeNames: {"item.game.id"},
{Key: "item.name", Value: "text"}, endpoint.EPArtworks: {"item.game.id"},
}, endpoint.EPCollectionMemberships: {"item.game.id"},
}) endpoint.EPCovers: {"item.game.id"},
if err != nil { endpoint.EPExternalGames: {"item.game.id"},
log.Printf("failed to create index: %v", err) endpoint.EPGameEngines: {"item.game.id"},
endpoint.EPGameLocalizations: {"item.game.id"},
endpoint.EPGameVersions: {"item.game.id"},
endpoint.EPGameVersionFeatureValues: {"item.game.id"},
endpoint.EPGameVideos: {"item.game.id"},
endpoint.EPInvolvedCompanies: {"item.game.id"},
endpoint.EPLanguageSupports: {"item.game.id"},
endpoint.EPMultiplayerModes: {"item.game.id"},
endpoint.EPReleaseDates: {"item.game.id"},
endpoint.EPScreenshots: {"item.game.id"},
endpoint.EPWebsites: {"item.game.id"},
endpoint.EPGames: {"item.parent_game.id", "item.version_parent.id"},
} }
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{ for e, idxes := range indexMap {
Keys: bson.D{ for _, idx := range idxes {
{Key: "item.game.id", Value: 1}, _, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
}, Keys: bson.D{
}) {Key: idx, Value: 1},
if err != nil { },
log.Printf("failed to create index: %v", err) })
if err != nil {
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
}
}
} }
_, err = m.Collections[endpoint.EPArtworks].Indexes().CreateOne(ctx, mongo.IndexModel{ for _, e := range endpoint.AllEndpoints {
Keys: bson.D{ if e == endpoint.EPWebhooks || e == endpoint.EPSearch || e == endpoint.EPPopularityPrimitives {
{Key: "item.game.id", Value: 1}, continue
}, }
}) _, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
if err != nil { Keys: bson.D{
log.Printf("failed to create index: %v", err) {Key: "item.id", Value: 1},
} },
})
_, err = m.Collections[endpoint.EPCollectionMemberships].Indexes().CreateOne(ctx, mongo.IndexModel{ if err != nil {
Keys: bson.D{ log.Printf("failed to create index item.id for %s: %v", string(e), err)
{Key: "item.game.id", Value: 1}, }
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPCovers].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPExternalGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.parent_game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.version_parent.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameEngines].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameLocalizations].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameVersions].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameVersionFeatureValues].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPGameVideos].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPInvolvedCompanies].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPLanguageSupports].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPMultiplayerModes].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPReleaseDates].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPScreenshots].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
}
_, err = m.Collections[endpoint.EPWebsites].Indexes().CreateOne(ctx, mongo.IndexModel{
Keys: bson.D{
{Key: "item.game.id", Value: 1},
},
})
if err != nil {
log.Printf("failed to create index: %v", err)
} }
} }
func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error { func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if item.MId.IsZero() { if item.MId.IsZero() {
item.MId = bson.NewObjectID() item.MId = bson.NewObjectID()
} }
@ -265,17 +137,16 @@ func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
return fmt.Errorf("collection not found") return fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := coll.UpdateOne(ctx, filter, update, opts) _, err := coll.UpdateOne(ctx, filter, update, opts)
if err != nil { if err != nil {
return fmt.Errorf("failed to save item to %s: %v", string(e), err) return fmt.Errorf("failed to save item to %s: %w", string(e), err)
} }
return nil return nil
} }
func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error { func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
defer cancel()
var models []mongo.WriteModel var models []mongo.WriteModel
for _, item := range items { for _, item := range items {
@ -293,9 +164,11 @@ func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
return fmt.Errorf("collection not found") return fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
defer cancel()
_, err := coll.BulkWrite(ctx, models) _, err := coll.BulkWrite(ctx, models)
if err != nil { if err != nil {
return fmt.Errorf("failed to save items in bulk %s: %v", string(e), err) return fmt.Errorf("failed to save items in bulk %s: %w", string(e), err)
} }
return nil return nil
} }
@ -309,23 +182,22 @@ func CountItems(e endpoint.EndpointName) (int64, error) {
defer cancel() defer cancel()
count, err := coll.EstimatedDocumentCount(ctx) count, err := coll.EstimatedDocumentCount(ctx)
if err != nil { if err != nil {
return 0, fmt.Errorf("failed to count items %s: %v", string(e), err) return 0, fmt.Errorf("failed to count items %s: %w", string(e), err)
} }
return count, nil return count, nil
} }
func GetItemByIGDBID[T any](e endpoint.EndpointName, id uint64) (*model.Item[T], error) { func GetItemByIGDBID[T any](e endpoint.EndpointName, id uint64) (*model.Item[T], error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var item model.Item[T] var item model.Item[T]
coll := GetInstance().Collections[e] coll := GetInstance().Collections[e]
if coll == nil { if coll == nil {
return nil, fmt.Errorf("collection not found") return nil, fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item) err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get item %s: %v", string(e), err) return nil, fmt.Errorf("failed to get item %s: %w", string(e), err)
} }
return &item, nil return &item, nil
} }
@ -334,33 +206,35 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
if len(ids) == 0 { if len(ids) == 0 {
return nil, nil return nil, nil
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second+time.Duration(len(ids))*200*time.Millisecond)
defer cancel()
coll := GetInstance().Collections[e] coll := GetInstance().Collections[e]
if coll == nil { if coll == nil {
return nil, fmt.Errorf("collection not found") return nil, fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}}) cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}})
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err) return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
} }
type IdGetter interface { type IdGetter interface {
GetId() uint64 GetId() uint64
} }
ctx, cancel = context.WithTimeout(context.Background(), time.Duration(len(ids))*200*time.Millisecond)
defer cancel()
res := make(map[uint64]*model.Item[T]) res := make(map[uint64]*model.Item[T])
for cursor.Next(ctx) { for cursor.Next(ctx) {
item := model.Item[T]{} item := model.Item[T]{}
err := cursor.Decode(&item) err := cursor.Decode(&item)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err) return nil, fmt.Errorf("failed to decode item %s: %w", string(e), err)
} }
if v, ok := any(item.Item).(IdGetter); ok { if v, ok := any(item.Item).(IdGetter); ok {
res[v.GetId()] = &item res[v.GetId()] = &item
} else { } else {
return nil, fmt.Errorf("failed to get id from item %s: %v", string(e), err) return nil, fmt.Errorf("failed to get id from item %s: %w", string(e), err)
} }
} }
@ -368,31 +242,31 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
} }
func RemoveItemByID(e endpoint.EndpointName, id bson.ObjectID) error { func RemoveItemByID(e endpoint.EndpointName, id bson.ObjectID) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
coll := GetInstance().Collections[e] coll := GetInstance().Collections[e]
if coll == nil { if coll == nil {
return fmt.Errorf("collection not found") return fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := coll.DeleteOne(ctx, bson.M{"_id": id}) _, err := coll.DeleteOne(ctx, bson.M{"_id": id})
if err != nil { if err != nil {
return fmt.Errorf("failed to remove game: %v", err) return fmt.Errorf("failed to remove game: %w", err)
} }
return nil return nil
} }
func RemoveItemsByID(e endpoint.EndpointName, ids []bson.ObjectID) error { func RemoveItemsByID(e endpoint.EndpointName, ids []bson.ObjectID) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
coll := GetInstance().Collections[e] coll := GetInstance().Collections[e]
if coll == nil { if coll == nil {
return fmt.Errorf("collection not found") return fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := coll.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}}) _, err := coll.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
if err != nil { if err != nil {
return fmt.Errorf("failed to remove games: %v", err) return fmt.Errorf("failed to remove games: %w", err)
} }
return nil return nil
} }
@ -422,11 +296,11 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
}}, }},
}, },
} }
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel() defer cancel()
cursor, err := coll.Aggregate(ctx, pipeline) cursor, err := coll.Aggregate(ctx, pipeline)
if err != nil { if err != nil {
return fmt.Errorf("failed to aggregate: %v", err) return fmt.Errorf("failed to aggregate: %w", err)
} }
var results []struct { var results []struct {
ID uint64 `bson:"_id"` ID uint64 `bson:"_id"`
@ -434,7 +308,7 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
} }
err = cursor.All(ctx, &results) err = cursor.All(ctx, &results)
if err != nil { if err != nil {
return fmt.Errorf("failed to get results: %v", err) return fmt.Errorf("failed to get results: %w", err)
} }
removedIds := make([]bson.ObjectID, 0, len(results)) removedIds := make([]bson.ObjectID, 0, len(results))
@ -445,59 +319,55 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
err = RemoveItemsByID(e, removedIds) err = RemoveItemsByID(e, removedIds)
if err != nil { if err != nil {
return fmt.Errorf("failed to remove duplicate games: %v", err) return fmt.Errorf("failed to remove duplicate games: %w", err)
} }
return nil return nil
} }
func GetItemsByIGDBGameID[T any](e endpoint.EndpointName, id uint64) ([]*model.Item[T], error) { func GetItemsByIGDBGameID[T any](e endpoint.EndpointName, id uint64) ([]*model.Item[T], error) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
coll := GetInstance().Collections[e] coll := GetInstance().Collections[e]
if coll == nil { if coll == nil {
return nil, fmt.Errorf("collection not found") return nil, fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cursor, err := coll.Find(ctx, bson.M{"item.game.id": id}) cursor, err := coll.Find(ctx, bson.M{"item.game.id": id})
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err) return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
} }
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var items []*model.Item[T] var items []*model.Item[T]
for cursor.Next(ctx) { err = cursor.All(ctx, &items)
item := model.Item[T]{} if err != nil {
err := cursor.Decode(&item) return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
if err != nil {
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
}
items = append(items, &item)
} }
return items, nil return items, nil
} }
func GetItemsPagnated[T any](e endpoint.EndpointName, offset int64, limit int64) ([]*model.Item[T], error) { func GetItemsPagnated[T any](e endpoint.EndpointName, offset int64, limit int64) ([]*model.Item[T], error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
defer cancel()
coll := GetInstance().Collections[e] coll := GetInstance().Collections[e]
if coll == nil { if coll == nil {
return nil, fmt.Errorf("collection not found") return nil, fmt.Errorf("collection not found")
} }
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
defer cancel()
cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}})) cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}}))
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err) return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
} }
var items []*model.Item[T] var items []*model.Item[T]
for cursor.Next(ctx) { err = cursor.All(ctx, &items)
item := model.Item[T]{} if err != nil {
err := cursor.Decode(&item) return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
if err != nil {
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
}
items = append(items, &item)
} }
return items, nil return items, nil

29
db/game.go
View File

@ -2,6 +2,7 @@ package db
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"igdb-database/model" "igdb-database/model"
"time" "time"
@ -9,28 +10,28 @@ import (
"github.com/bestnite/go-igdb/endpoint" "github.com/bestnite/go-igdb/endpoint"
pb "github.com/bestnite/go-igdb/proto" pb "github.com/bestnite/go-igdb/proto"
"go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/bson"
"go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options" "go.mongodb.org/mongo-driver/v2/mongo/options"
) )
func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) { func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond)
defer cancel()
ids := make([]uint64, 0, len(games)) ids := make([]uint64, 0, len(games))
for _, game := range games { for _, game := range games {
ids = append(ids, game.Id) ids = append(ids, game.Id)
} }
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond)
defer cancel()
cursor, err := GetInstance().GameCollection.Find(ctx, bson.M{"id": bson.M{"$in": ids}}) cursor, err := GetInstance().GameCollection.Find(ctx, bson.M{"id": bson.M{"$in": ids}})
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get games: %v", err) return nil, fmt.Errorf("failed to get games: %w", err)
} }
res := make(map[uint64]bool, len(games)) res := make(map[uint64]bool, len(games))
g := []*model.Game{} g := []*model.Game{}
err = cursor.All(ctx, &g) err = cursor.All(ctx, &g)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get games: %v", err) return nil, fmt.Errorf("failed to get games: %w", err)
} }
for _, game := range g { for _, game := range g {
res[game.Id] = true res[game.Id] = true
@ -40,8 +41,6 @@ func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
} }
func SaveGame(game *model.Game) error { func SaveGame(game *model.Game) error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if game.MId.IsZero() { if game.MId.IsZero() {
game.MId = bson.NewObjectID() game.MId = bson.NewObjectID()
} }
@ -49,6 +48,8 @@ func SaveGame(game *model.Game) error {
update := bson.M{"$set": game} update := bson.M{"$set": game}
opts := options.UpdateOne().SetUpsert(true) opts := options.UpdateOne().SetUpsert(true)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_, err := GetInstance().GameCollection.UpdateOne(ctx, filter, update, opts) _, err := GetInstance().GameCollection.UpdateOne(ctx, filter, update, opts)
if err != nil { if err != nil {
return err return err
@ -112,10 +113,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
if game.Cover != nil { if game.Cover != nil {
coverId := game.Cover.Id coverId := game.Cover.Id
cover, err := GetItemByIGDBID[pb.Cover](endpoint.EPCovers, coverId) cover, err := GetItemByIGDBID[pb.Cover](endpoint.EPCovers, coverId)
if err != nil { if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return nil, err return nil, err
} }
res.Cover = cover.Item if cover != nil {
res.Cover = cover.Item
}
} }
res.CreatedAt = game.CreatedAt res.CreatedAt = game.CreatedAt
@ -152,10 +155,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
if game.Franchise != nil { if game.Franchise != nil {
franchiseId := game.Franchise.Id franchiseId := game.Franchise.Id
franchise, err := GetItemByIGDBID[pb.Franchise](endpoint.EPFranchises, franchiseId) franchise, err := GetItemByIGDBID[pb.Franchise](endpoint.EPFranchises, franchiseId)
if err != nil { if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
return nil, err return nil, err
} }
res.Franchise = franchise.Item if franchise != nil {
res.Franchise = franchise.Item
}
} }
franchiseIds := make([]uint64, 0, len(game.Franchises)) franchiseIds := make([]uint64, 0, len(game.Franchises))
@ -471,7 +476,7 @@ func GetGameByIGDBID(id uint64) (*model.Game, error) {
var game model.Game var game model.Game
err := GetInstance().GameCollection.FindOne(ctx, bson.M{"id": id}).Decode(&game) err := GetInstance().GameCollection.FindOne(ctx, bson.M{"id": id}).Decode(&game)
if err != nil { if err != nil {
return nil, fmt.Errorf("failed to get game: %v", err) return nil, fmt.Errorf("failed to get game: %w", err)
} }
return &game, nil return &game, nil
} }

188
main.go
View File

@ -1,6 +1,8 @@
package main package main
import ( import (
"errors"
"flag"
"igdb-database/collector" "igdb-database/collector"
"igdb-database/config" "igdb-database/config"
"igdb-database/db" "igdb-database/db"
@ -11,11 +13,121 @@ import (
"github.com/bestnite/go-igdb" "github.com/bestnite/go-igdb"
"github.com/bestnite/go-igdb/endpoint" "github.com/bestnite/go-igdb/endpoint"
pb "github.com/bestnite/go-igdb/proto" pb "github.com/bestnite/go-igdb/proto"
"go.mongodb.org/mongo-driver/v2/mongo"
)
var (
enableAggregate = flag.Bool("aggregate", false, "aggregate games")
enableFetch = flag.Bool("fetch", false, "fetch data")
enableReFetch = flag.Bool("re-fetch", false, "re fetch data even if collection is not empty")
enableReAggregate = flag.Bool("re-aggregate", false, "re aggregate games even if game_details is not empty")
enableWebhook = flag.Bool("webhook", true, "start webhook server")
) )
func main() { func main() {
flag.Parse()
client := igdb.New(config.C().Twitch.ClientID, config.C().Twitch.ClientSecret) client := igdb.New(config.C().Twitch.ClientID, config.C().Twitch.ClientSecret)
if *enableFetch || *enableReFetch {
log.Printf("fetching data")
allFetchAndStore(client)
log.Printf("data fetched")
}
if *enableAggregate || *enableReAggregate {
log.Printf("aggregating games")
aggregateGames()
log.Printf("games aggregated")
}
if *enableWebhook {
log.Printf("starting webhook server")
collector.StartWebhookServer(client)
}
}
func aggregateGames() {
total, err := db.CountItems(endpoint.EPGames)
if err != nil {
log.Fatalf("failed to count games: %v", err)
}
finished := int64(0)
wg := sync.WaitGroup{}
concurrenceNum := 10
taskOneLoop := int64(500)
concurrence := make(chan struct{}, concurrenceNum)
defer close(concurrence)
for i := int64(0); i < total; i += taskOneLoop {
concurrence <- struct{}{}
wg.Add(1)
go func(i int64) {
defer func() { <-concurrence }()
defer wg.Done()
items, err := db.GetItemsPagnated[pb.Game](endpoint.EPGames, i, taskOneLoop)
if err != nil {
log.Fatalf("failed to get games: %v", err)
}
games := make([]*pb.Game, 0, len(items))
for _, item := range items {
games = append(games, item.Item)
}
isAggregated := make(map[uint64]bool, len(games))
if !*enableReAggregate {
isAggregated, err = db.IsGamesAggregated(games)
if err != nil {
log.Fatalf("failed to check if games are aggregated: %v", err)
}
} else {
for _, game := range games {
isAggregated[game.Id] = false
}
}
for _, item := range items {
if isAggregated[item.Item.Id] {
p := atomic.AddInt64(&finished, 1)
log.Printf("game aggregated %d/%d", p, total)
continue
}
game, err := db.ConvertGame(item.Item)
if err != nil {
log.Fatalf("failed to convert game: %v", err)
}
oldGame, err := db.GetGameByIGDBID(item.Item.Id)
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
log.Fatalf("failed to get game: %v", err)
}
if oldGame != nil {
game.MId = oldGame.MId
}
err = db.SaveGame(game)
if err != nil {
log.Fatalf("failed to save game: %v", err)
}
p := atomic.AddInt64(&finished, 1)
log.Printf("game aggregated %d/%d", p, total)
}
}(i)
}
wg.Wait()
}
func fetchAndStore[T any](
e endpoint.EntityEndpoint[T],
) {
if count, err := db.CountItems(e.GetEndpointName()); (err == nil && count == 0) || *enableReFetch {
collector.FetchAndStore(e)
} else if err != nil {
log.Printf("failed to count items: %v", err)
}
}
func allFetchAndStore(client *igdb.Client) {
fetchAndStore(client.AgeRatingCategories) fetchAndStore(client.AgeRatingCategories)
fetchAndStore(client.AgeRatingContentDescriptions) fetchAndStore(client.AgeRatingContentDescriptions)
fetchAndStore(client.AgeRatingContentDescriptionsV2) fetchAndStore(client.AgeRatingContentDescriptionsV2)
@ -50,7 +162,6 @@ func main() {
fetchAndStore(client.GameLocalizations) fetchAndStore(client.GameLocalizations)
fetchAndStore(client.GameModes) fetchAndStore(client.GameModes)
fetchAndStore(client.GameReleaseFormats) fetchAndStore(client.GameReleaseFormats)
fetchAndStore(client.Games)
fetchAndStore(client.GameStatuses) fetchAndStore(client.GameStatuses)
fetchAndStore(client.GameTimeToBeats) fetchAndStore(client.GameTimeToBeats)
fetchAndStore(client.GameTypes) fetchAndStore(client.GameTypes)
@ -86,78 +197,5 @@ func main() {
fetchAndStore(client.Websites) fetchAndStore(client.Websites)
fetchAndStore(client.WebsiteTypes) fetchAndStore(client.WebsiteTypes)
log.Printf("aggregating games") fetchAndStore(client.Games)
aggregateGames()
log.Printf("games aggregated")
log.Printf("starting webhook server")
collector.StartWebhookServer(client)
}
func aggregateGames() {
total, err := db.CountItems(endpoint.EPGames)
if err != nil {
log.Fatalf("failed to count games: %v", err)
}
finished := int64(0)
wg := sync.WaitGroup{}
concurrenceNum := 10
taskOneLoop := int64(500)
concurrence := make(chan struct{}, concurrenceNum)
defer close(concurrence)
for i := int64(0); i < total; i += taskOneLoop {
concurrence <- struct{}{}
wg.Add(1)
go func(i int64) {
defer func() { <-concurrence }()
defer wg.Done()
items, err := db.GetItemsPagnated[pb.Game](endpoint.EPGames, i, taskOneLoop)
if err != nil {
log.Fatalf("failed to get games: %v", err)
}
games := make([]*pb.Game, 0, len(items))
for _, item := range items {
games = append(games, item.Item)
}
isAggregated, err := db.IsGamesAggregated(games)
if err != nil {
log.Fatalf("failed to check if games are aggregated: %v", err)
}
for _, item := range items {
if isAggregated[item.Item.Id] {
p := atomic.AddInt64(&finished, 1)
log.Printf("game aggregated %d/%d", p, total)
continue
}
if err != nil {
log.Fatalf("failed to check if game is aggregated: %v", err)
}
game, err := db.ConvertGame(item.Item)
if err != nil {
log.Fatalf("failed to convert game: %v", err)
}
err = db.SaveGame(game)
if err != nil {
log.Fatalf("failed to save game: %v", err)
}
p := atomic.AddInt64(&finished, 1)
log.Printf("game aggregated %d/%d", p, total)
}
}(i)
}
wg.Wait()
}
func fetchAndStore[T any](
e endpoint.EntityEndpoint[T],
) {
if count, err := db.CountItems(e.GetEndpointName()); err == nil && count == 0 {
collector.FetchAndStore(e)
} else if err != nil {
log.Printf("failed to count items: %v", err)
}
} }