mirror of
https://github.com/bestnite/igdb-database.git
synced 2025-04-26 21:25:54 +08:00
u
This commit is contained in:
parent
0214d95d3e
commit
4508b19f4b
@ -33,9 +33,10 @@ func FetchAndStore[T any](
|
||||
defer wg.Done()
|
||||
defer func() { <-concurrence }()
|
||||
|
||||
// items data from igdb
|
||||
items, err := e.Paginated(uint64(i), 500)
|
||||
if err != nil {
|
||||
log.Printf("failed to get items: %v", err)
|
||||
log.Printf("failed to get items from igdb %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
|
||||
@ -48,14 +49,15 @@ func FetchAndStore[T any](
|
||||
if v, ok := any(item).(IdGetter); ok {
|
||||
ids = append(ids, v.GetId())
|
||||
} else {
|
||||
log.Printf("failed to get id from item: %v", err)
|
||||
log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// items data from database
|
||||
data, err := db.GetItemsByIGDBIDs[T](e.GetEndpointName(), ids)
|
||||
if err != nil {
|
||||
log.Printf("failed to get items: %v", err)
|
||||
log.Printf("failed to get items from database %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
|
||||
@ -63,7 +65,7 @@ func FetchAndStore[T any](
|
||||
for _, item := range items {
|
||||
v, ok := any(item).(IdGetter)
|
||||
if !ok {
|
||||
log.Printf("failed to get id from item: %v", err)
|
||||
log.Printf("failed to get id from item %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
} else {
|
||||
if data[v.GetId()] == nil {
|
||||
@ -76,7 +78,7 @@ func FetchAndStore[T any](
|
||||
}
|
||||
err = db.SaveItems(e.GetEndpointName(), newItems)
|
||||
if err != nil {
|
||||
log.Printf("failed to save games: %v", err)
|
||||
log.Printf("failed to save games %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package collector
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"igdb-database/config"
|
||||
"igdb-database/db"
|
||||
@ -94,6 +95,22 @@ func StartWebhookServer(client *igdb.Client) {
|
||||
http.HandleFunc(webhook(client.Themes))
|
||||
http.HandleFunc(webhook(client.Websites))
|
||||
http.HandleFunc(webhook(client.WebsiteTypes))
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
if _, err := w.Write([]byte("Hello World!")); err != nil {
|
||||
log.Printf("failed to write response: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
serverStart := make(chan bool)
|
||||
go func() {
|
||||
defer close(serverStart)
|
||||
log.Printf("starting webhook server on %s", config.C().Address)
|
||||
err = http.ListenAndServe(config.C().Address, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to start webhook server: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
enabledEndpoint := endpoint.AllEndpoints
|
||||
enabledEndpoint = slices.DeleteFunc(enabledEndpoint, func(e endpoint.EndpointName) bool {
|
||||
@ -116,18 +133,7 @@ func StartWebhookServer(client *igdb.Client) {
|
||||
log.Printf("all webhook registered")
|
||||
}
|
||||
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
if _, err := w.Write([]byte("Hello World!")); err != nil {
|
||||
log.Printf("failed to write response: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
log.Printf("starting webhook server on %s", config.C().Address)
|
||||
err = http.ListenAndServe(config.C().Address, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to start webhook server: %v", err)
|
||||
}
|
||||
<-serverStart
|
||||
}
|
||||
|
||||
func webhook[T any](
|
||||
@ -162,7 +168,7 @@ func webhook[T any](
|
||||
return
|
||||
}
|
||||
oldItem, err := db.GetItemByIGDBID[T](e.GetEndpointName(), data.ID)
|
||||
if err != nil && err != mongo.ErrNoDocuments {
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
@ -183,6 +189,7 @@ func webhook[T any](
|
||||
if err == nil {
|
||||
g.MId = game.MId
|
||||
_ = db.SaveGame(g)
|
||||
log.Printf("game %d aggregated", data.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -216,6 +223,7 @@ func webhook[T any](
|
||||
log.Printf("failed to save game: %v", err)
|
||||
goto END
|
||||
}
|
||||
log.Printf("game %d aggregated", data.ID)
|
||||
}
|
||||
|
||||
END:
|
||||
|
||||
328
db/db.go
328
db/db.go
@ -60,199 +60,71 @@ func (m *MongoDB) createIndex() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
textIndexMap := map[endpoint.EndpointName]string{
|
||||
endpoint.EPGames: "item.name",
|
||||
endpoint.EPAlternativeNames: "item.name",
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.name", Value: "text"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
for e, idx := range textIndexMap {
|
||||
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: idx, Value: "text"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.name", Value: "text"},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
indexMap := map[endpoint.EndpointName][]string{
|
||||
endpoint.EPAlternativeNames: {"item.game.id"},
|
||||
endpoint.EPArtworks: {"item.game.id"},
|
||||
endpoint.EPCollectionMemberships: {"item.game.id"},
|
||||
endpoint.EPCovers: {"item.game.id"},
|
||||
endpoint.EPExternalGames: {"item.game.id"},
|
||||
endpoint.EPGameEngines: {"item.game.id"},
|
||||
endpoint.EPGameLocalizations: {"item.game.id"},
|
||||
endpoint.EPGameVersions: {"item.game.id"},
|
||||
endpoint.EPGameVersionFeatureValues: {"item.game.id"},
|
||||
endpoint.EPGameVideos: {"item.game.id"},
|
||||
endpoint.EPInvolvedCompanies: {"item.game.id"},
|
||||
endpoint.EPLanguageSupports: {"item.game.id"},
|
||||
endpoint.EPMultiplayerModes: {"item.game.id"},
|
||||
endpoint.EPReleaseDates: {"item.game.id"},
|
||||
endpoint.EPScreenshots: {"item.game.id"},
|
||||
endpoint.EPWebsites: {"item.game.id"},
|
||||
endpoint.EPGames: {"item.parent_game.id", "item.version_parent.id"},
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPAlternativeNames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
for e, idxes := range indexMap {
|
||||
for _, idx := range idxes {
|
||||
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: idx, Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index %s for %s: %v", idx, string(e), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPArtworks].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPCollectionMemberships].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPCovers].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPExternalGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.parent_game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGames].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.version_parent.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameEngines].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameLocalizations].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameVersions].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameVersionFeatureValues].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPGameVideos].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPInvolvedCompanies].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPLanguageSupports].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPMultiplayerModes].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPReleaseDates].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPScreenshots].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
}
|
||||
|
||||
_, err = m.Collections[endpoint.EPWebsites].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.game.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index: %v", err)
|
||||
for _, e := range endpoint.AllEndpoints {
|
||||
if e == endpoint.EPWebhooks || e == endpoint.EPSearch || e == endpoint.EPPopularityPrimitives {
|
||||
continue
|
||||
}
|
||||
_, err := m.Collections[e].Indexes().CreateOne(ctx, mongo.IndexModel{
|
||||
Keys: bson.D{
|
||||
{Key: "item.id", Value: 1},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("failed to create index item.id for %s: %v", string(e), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if item.MId.IsZero() {
|
||||
item.MId = bson.NewObjectID()
|
||||
}
|
||||
@ -265,17 +137,16 @@ func SaveItem[T any](e endpoint.EndpointName, item *model.Item[T]) error {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := coll.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save item to %s: %v", string(e), err)
|
||||
return fmt.Errorf("failed to save item to %s: %w", string(e), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
var models []mongo.WriteModel
|
||||
|
||||
for _, item := range items {
|
||||
@ -293,9 +164,11 @@ func SaveItems[T any](e endpoint.EndpointName, items []*model.Item[T]) error {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(items))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := coll.BulkWrite(ctx, models)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to save items in bulk %s: %v", string(e), err)
|
||||
return fmt.Errorf("failed to save items in bulk %s: %w", string(e), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -309,23 +182,22 @@ func CountItems(e endpoint.EndpointName) (int64, error) {
|
||||
defer cancel()
|
||||
count, err := coll.EstimatedDocumentCount(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count items %s: %v", string(e), err)
|
||||
return 0, fmt.Errorf("failed to count items %s: %w", string(e), err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func GetItemByIGDBID[T any](e endpoint.EndpointName, id uint64) (*model.Item[T], error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var item model.Item[T]
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
err := coll.FindOne(ctx, bson.M{"item.id": id}).Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get item %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get item %s: %w", string(e), err)
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
@ -334,33 +206,35 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second+time.Duration(len(ids))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
cursor, err := coll.Find(ctx, bson.M{"item.id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
type IdGetter interface {
|
||||
GetId() uint64
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), time.Duration(len(ids))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
res := make(map[uint64]*model.Item[T])
|
||||
for cursor.Next(ctx) {
|
||||
item := model.Item[T]{}
|
||||
err := cursor.Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to decode item %s: %w", string(e), err)
|
||||
}
|
||||
if v, ok := any(item.Item).(IdGetter); ok {
|
||||
res[v.GetId()] = &item
|
||||
} else {
|
||||
return nil, fmt.Errorf("failed to get id from item %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get id from item %s: %w", string(e), err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -368,31 +242,31 @@ func GetItemsByIGDBIDs[T any](e endpoint.EndpointName, ids []uint64) (map[uint64
|
||||
}
|
||||
|
||||
func RemoveItemByID(e endpoint.EndpointName, id bson.ObjectID) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := coll.DeleteOne(ctx, bson.M{"_id": id})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove game: %v", err)
|
||||
return fmt.Errorf("failed to remove game: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func RemoveItemsByID(e endpoint.EndpointName, ids []bson.ObjectID) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := coll.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove games: %v", err)
|
||||
return fmt.Errorf("failed to remove games: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -422,11 +296,11 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
|
||||
}},
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
cursor, err := coll.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to aggregate: %v", err)
|
||||
return fmt.Errorf("failed to aggregate: %w", err)
|
||||
}
|
||||
var results []struct {
|
||||
ID uint64 `bson:"_id"`
|
||||
@ -434,7 +308,7 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
|
||||
}
|
||||
err = cursor.All(ctx, &results)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get results: %v", err)
|
||||
return fmt.Errorf("failed to get results: %w", err)
|
||||
}
|
||||
|
||||
removedIds := make([]bson.ObjectID, 0, len(results))
|
||||
@ -445,59 +319,55 @@ func RemoveDuplicateItems(e endpoint.EndpointName) error {
|
||||
|
||||
err = RemoveItemsByID(e, removedIds)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to remove duplicate games: %v", err)
|
||||
return fmt.Errorf("failed to remove duplicate games: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetItemsByIGDBGameID[T any](e endpoint.EndpointName, id uint64) ([]*model.Item[T], error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cursor, err := coll.Find(ctx, bson.M{"item.game.id": id})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
var items []*model.Item[T]
|
||||
for cursor.Next(ctx) {
|
||||
item := model.Item[T]{}
|
||||
err := cursor.Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
|
||||
}
|
||||
items = append(items, &item)
|
||||
err = cursor.All(ctx, &items)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func GetItemsPagnated[T any](e endpoint.EndpointName, offset int64, limit int64) ([]*model.Item[T], error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
coll := GetInstance().Collections[e]
|
||||
if coll == nil {
|
||||
return nil, fmt.Errorf("collection not found")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(limit)*200*time.Millisecond)
|
||||
defer cancel()
|
||||
cursor, err := coll.Find(ctx, bson.M{}, options.Find().SetSkip(offset).SetLimit(limit).SetSort(bson.D{{Key: "item.id", Value: 1}}))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get items %s: %v", string(e), err)
|
||||
return nil, fmt.Errorf("failed to get items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
var items []*model.Item[T]
|
||||
for cursor.Next(ctx) {
|
||||
item := model.Item[T]{}
|
||||
err := cursor.Decode(&item)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode item %s: %v", string(e), err)
|
||||
}
|
||||
items = append(items, &item)
|
||||
err = cursor.All(ctx, &items)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode items %s: %w", string(e), err)
|
||||
}
|
||||
|
||||
return items, nil
|
||||
|
||||
29
db/game.go
29
db/game.go
@ -2,6 +2,7 @@ package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"igdb-database/model"
|
||||
"time"
|
||||
@ -9,28 +10,28 @@ import (
|
||||
"github.com/bestnite/go-igdb/endpoint"
|
||||
pb "github.com/bestnite/go-igdb/proto"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
)
|
||||
|
||||
func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
ids := make([]uint64, 0, len(games))
|
||||
for _, game := range games {
|
||||
ids = append(ids, game.Id)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(len(games))*200*time.Millisecond)
|
||||
defer cancel()
|
||||
cursor, err := GetInstance().GameCollection.Find(ctx, bson.M{"id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get games: %v", err)
|
||||
return nil, fmt.Errorf("failed to get games: %w", err)
|
||||
}
|
||||
|
||||
res := make(map[uint64]bool, len(games))
|
||||
g := []*model.Game{}
|
||||
err = cursor.All(ctx, &g)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get games: %v", err)
|
||||
return nil, fmt.Errorf("failed to get games: %w", err)
|
||||
}
|
||||
for _, game := range g {
|
||||
res[game.Id] = true
|
||||
@ -40,8 +41,6 @@ func IsGamesAggregated(games []*pb.Game) (map[uint64]bool, error) {
|
||||
}
|
||||
|
||||
func SaveGame(game *model.Game) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if game.MId.IsZero() {
|
||||
game.MId = bson.NewObjectID()
|
||||
}
|
||||
@ -49,6 +48,8 @@ func SaveGame(game *model.Game) error {
|
||||
update := bson.M{"$set": game}
|
||||
opts := options.UpdateOne().SetUpsert(true)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := GetInstance().GameCollection.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -112,10 +113,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
|
||||
if game.Cover != nil {
|
||||
coverId := game.Cover.Id
|
||||
cover, err := GetItemByIGDBID[pb.Cover](endpoint.EPCovers, coverId)
|
||||
if err != nil {
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, err
|
||||
}
|
||||
res.Cover = cover.Item
|
||||
if cover != nil {
|
||||
res.Cover = cover.Item
|
||||
}
|
||||
}
|
||||
|
||||
res.CreatedAt = game.CreatedAt
|
||||
@ -152,10 +155,12 @@ func ConvertGame(game *pb.Game) (*model.Game, error) {
|
||||
if game.Franchise != nil {
|
||||
franchiseId := game.Franchise.Id
|
||||
franchise, err := GetItemByIGDBID[pb.Franchise](endpoint.EPFranchises, franchiseId)
|
||||
if err != nil {
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
return nil, err
|
||||
}
|
||||
res.Franchise = franchise.Item
|
||||
if franchise != nil {
|
||||
res.Franchise = franchise.Item
|
||||
}
|
||||
}
|
||||
|
||||
franchiseIds := make([]uint64, 0, len(game.Franchises))
|
||||
@ -471,7 +476,7 @@ func GetGameByIGDBID(id uint64) (*model.Game, error) {
|
||||
var game model.Game
|
||||
err := GetInstance().GameCollection.FindOne(ctx, bson.M{"id": id}).Decode(&game)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get game: %v", err)
|
||||
return nil, fmt.Errorf("failed to get game: %w", err)
|
||||
}
|
||||
return &game, nil
|
||||
}
|
||||
|
||||
188
main.go
188
main.go
@ -1,6 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"flag"
|
||||
"igdb-database/collector"
|
||||
"igdb-database/config"
|
||||
"igdb-database/db"
|
||||
@ -11,11 +13,121 @@ import (
|
||||
"github.com/bestnite/go-igdb"
|
||||
"github.com/bestnite/go-igdb/endpoint"
|
||||
pb "github.com/bestnite/go-igdb/proto"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
)
|
||||
|
||||
var (
|
||||
enableAggregate = flag.Bool("aggregate", false, "aggregate games")
|
||||
enableFetch = flag.Bool("fetch", false, "fetch data")
|
||||
enableReFetch = flag.Bool("re-fetch", false, "re fetch data even if collection is not empty")
|
||||
enableReAggregate = flag.Bool("re-aggregate", false, "re aggregate games even if game_details is not empty")
|
||||
enableWebhook = flag.Bool("webhook", true, "start webhook server")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
client := igdb.New(config.C().Twitch.ClientID, config.C().Twitch.ClientSecret)
|
||||
|
||||
if *enableFetch || *enableReFetch {
|
||||
log.Printf("fetching data")
|
||||
allFetchAndStore(client)
|
||||
log.Printf("data fetched")
|
||||
}
|
||||
|
||||
if *enableAggregate || *enableReAggregate {
|
||||
log.Printf("aggregating games")
|
||||
aggregateGames()
|
||||
log.Printf("games aggregated")
|
||||
}
|
||||
|
||||
if *enableWebhook {
|
||||
log.Printf("starting webhook server")
|
||||
collector.StartWebhookServer(client)
|
||||
}
|
||||
}
|
||||
|
||||
func aggregateGames() {
|
||||
total, err := db.CountItems(endpoint.EPGames)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to count games: %v", err)
|
||||
}
|
||||
|
||||
finished := int64(0)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
concurrenceNum := 10
|
||||
taskOneLoop := int64(500)
|
||||
|
||||
concurrence := make(chan struct{}, concurrenceNum)
|
||||
defer close(concurrence)
|
||||
for i := int64(0); i < total; i += taskOneLoop {
|
||||
concurrence <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(i int64) {
|
||||
defer func() { <-concurrence }()
|
||||
defer wg.Done()
|
||||
items, err := db.GetItemsPagnated[pb.Game](endpoint.EPGames, i, taskOneLoop)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to get games: %v", err)
|
||||
}
|
||||
games := make([]*pb.Game, 0, len(items))
|
||||
for _, item := range items {
|
||||
games = append(games, item.Item)
|
||||
}
|
||||
isAggregated := make(map[uint64]bool, len(games))
|
||||
if !*enableReAggregate {
|
||||
isAggregated, err = db.IsGamesAggregated(games)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to check if games are aggregated: %v", err)
|
||||
}
|
||||
} else {
|
||||
for _, game := range games {
|
||||
isAggregated[game.Id] = false
|
||||
}
|
||||
}
|
||||
for _, item := range items {
|
||||
if isAggregated[item.Item.Id] {
|
||||
p := atomic.AddInt64(&finished, 1)
|
||||
log.Printf("game aggregated %d/%d", p, total)
|
||||
continue
|
||||
}
|
||||
|
||||
game, err := db.ConvertGame(item.Item)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to convert game: %v", err)
|
||||
}
|
||||
oldGame, err := db.GetGameByIGDBID(item.Item.Id)
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
log.Fatalf("failed to get game: %v", err)
|
||||
}
|
||||
if oldGame != nil {
|
||||
game.MId = oldGame.MId
|
||||
}
|
||||
|
||||
err = db.SaveGame(game)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to save game: %v", err)
|
||||
}
|
||||
p := atomic.AddInt64(&finished, 1)
|
||||
log.Printf("game aggregated %d/%d", p, total)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func fetchAndStore[T any](
|
||||
e endpoint.EntityEndpoint[T],
|
||||
) {
|
||||
if count, err := db.CountItems(e.GetEndpointName()); (err == nil && count == 0) || *enableReFetch {
|
||||
collector.FetchAndStore(e)
|
||||
} else if err != nil {
|
||||
log.Printf("failed to count items: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func allFetchAndStore(client *igdb.Client) {
|
||||
fetchAndStore(client.AgeRatingCategories)
|
||||
fetchAndStore(client.AgeRatingContentDescriptions)
|
||||
fetchAndStore(client.AgeRatingContentDescriptionsV2)
|
||||
@ -50,7 +162,6 @@ func main() {
|
||||
fetchAndStore(client.GameLocalizations)
|
||||
fetchAndStore(client.GameModes)
|
||||
fetchAndStore(client.GameReleaseFormats)
|
||||
fetchAndStore(client.Games)
|
||||
fetchAndStore(client.GameStatuses)
|
||||
fetchAndStore(client.GameTimeToBeats)
|
||||
fetchAndStore(client.GameTypes)
|
||||
@ -86,78 +197,5 @@ func main() {
|
||||
fetchAndStore(client.Websites)
|
||||
fetchAndStore(client.WebsiteTypes)
|
||||
|
||||
log.Printf("aggregating games")
|
||||
aggregateGames()
|
||||
log.Printf("games aggregated")
|
||||
|
||||
log.Printf("starting webhook server")
|
||||
collector.StartWebhookServer(client)
|
||||
}
|
||||
|
||||
func aggregateGames() {
|
||||
total, err := db.CountItems(endpoint.EPGames)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to count games: %v", err)
|
||||
}
|
||||
|
||||
finished := int64(0)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
concurrenceNum := 10
|
||||
taskOneLoop := int64(500)
|
||||
|
||||
concurrence := make(chan struct{}, concurrenceNum)
|
||||
defer close(concurrence)
|
||||
for i := int64(0); i < total; i += taskOneLoop {
|
||||
concurrence <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(i int64) {
|
||||
defer func() { <-concurrence }()
|
||||
defer wg.Done()
|
||||
items, err := db.GetItemsPagnated[pb.Game](endpoint.EPGames, i, taskOneLoop)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to get games: %v", err)
|
||||
}
|
||||
games := make([]*pb.Game, 0, len(items))
|
||||
for _, item := range items {
|
||||
games = append(games, item.Item)
|
||||
}
|
||||
isAggregated, err := db.IsGamesAggregated(games)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to check if games are aggregated: %v", err)
|
||||
}
|
||||
for _, item := range items {
|
||||
if isAggregated[item.Item.Id] {
|
||||
p := atomic.AddInt64(&finished, 1)
|
||||
log.Printf("game aggregated %d/%d", p, total)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("failed to check if game is aggregated: %v", err)
|
||||
}
|
||||
|
||||
game, err := db.ConvertGame(item.Item)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to convert game: %v", err)
|
||||
}
|
||||
err = db.SaveGame(game)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to save game: %v", err)
|
||||
}
|
||||
p := atomic.AddInt64(&finished, 1)
|
||||
log.Printf("game aggregated %d/%d", p, total)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func fetchAndStore[T any](
|
||||
e endpoint.EntityEndpoint[T],
|
||||
) {
|
||||
if count, err := db.CountItems(e.GetEndpointName()); err == nil && count == 0 {
|
||||
collector.FetchAndStore(e)
|
||||
} else if err != nil {
|
||||
log.Printf("failed to count items: %v", err)
|
||||
}
|
||||
fetchAndStore(client.Games)
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user