mirror of
https://github.com/bestnite/igdb-database.git
synced 2025-04-26 21:25:54 +08:00
u
This commit is contained in:
parent
95f46cc7e7
commit
5bd2627d5e
@ -166,85 +166,15 @@ func webhook[T any](
|
||||
log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
oldItem, err := db.GetItemById[T](e.GetEndpointName(), data.ID)
|
||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
||||
log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, ok := any(e).(*endpoint.Games); ok {
|
||||
if oldItem != nil {
|
||||
// oldGame update
|
||||
oldGame := any(oldItem).(*pb.Game)
|
||||
// compare themes and genres
|
||||
game := any(item).(*pb.Game)
|
||||
|
||||
oldGameThemeIds := make([]uint64, 0, len(oldGame.Themes))
|
||||
for _, t := range oldGame.Themes {
|
||||
oldGameThemeIds = append(oldGameThemeIds, t.Id)
|
||||
}
|
||||
gameThemeIds := make([]uint64, 0, len(game.Themes))
|
||||
for _, t := range game.Themes {
|
||||
gameThemeIds = append(gameThemeIds, t.Id)
|
||||
}
|
||||
if !slices.Equal(oldGameThemeIds, gameThemeIds) {
|
||||
for _, t := range oldGameThemeIds {
|
||||
if !slices.Contains(gameThemeIds, t) {
|
||||
_ = db.MinusThemeCount(t)
|
||||
}
|
||||
}
|
||||
for _, t := range gameThemeIds {
|
||||
if !slices.Contains(oldGameThemeIds, t) {
|
||||
_ = db.AddThemeCount(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
oldGameGenreIds := make([]uint64, 0, len(oldGame.Genres))
|
||||
for _, t := range oldGame.Genres {
|
||||
oldGameGenreIds = append(oldGameGenreIds, t.Id)
|
||||
}
|
||||
gameGenreIds := make([]uint64, 0, len(game.Genres))
|
||||
for _, t := range game.Genres {
|
||||
gameGenreIds = append(gameGenreIds, t.Id)
|
||||
}
|
||||
if !slices.Equal(oldGameGenreIds, gameGenreIds) {
|
||||
for _, t := range oldGameGenreIds {
|
||||
if !slices.Contains(gameGenreIds, t) {
|
||||
_ = db.MinusGenreCount(t)
|
||||
}
|
||||
}
|
||||
for _, t := range gameGenreIds {
|
||||
if !slices.Contains(oldGameGenreIds, t) {
|
||||
_ = db.AddGenreCount(t)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
g, err := db.ConvertGame(game)
|
||||
if err != nil {
|
||||
log.Printf("failed to convert game: %v", err)
|
||||
} else {
|
||||
_ = db.SaveGame(g)
|
||||
log.Printf("game %d aggregated", data.ID)
|
||||
}
|
||||
|
||||
game := any(item).(*pb.Game)
|
||||
g, err := db.ConvertGame(game)
|
||||
if err != nil {
|
||||
log.Printf("failed to convert game: %v", err)
|
||||
} else {
|
||||
// new game
|
||||
game := any(item).(*pb.Game)
|
||||
for _, t := range game.Themes {
|
||||
_ = db.AddThemeCount(t.Id)
|
||||
}
|
||||
for _, t := range game.Genres {
|
||||
_ = db.AddGenreCount(t.Id)
|
||||
}
|
||||
g, err := db.ConvertGame(game)
|
||||
if err != nil {
|
||||
log.Printf("failed to convert game: %v", err)
|
||||
} else {
|
||||
_ = db.SaveGame(g)
|
||||
log.Printf("game %d aggregated", data.ID)
|
||||
}
|
||||
_ = db.SaveGame(g)
|
||||
log.Printf("game %d aggregated", data.ID)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
137
db/count.go
137
db/count.go
@ -1,137 +0,0 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/bestnite/go-igdb/endpoint"
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
)
|
||||
|
||||
type Count struct {
|
||||
Theme uint64 `json:"theme,omitempty"`
|
||||
Genre uint64 `json:"genre,omitempty"`
|
||||
Count int64 `json:"count,omitempty"`
|
||||
}
|
||||
|
||||
func GetCountByThemeId(themeId uint64) (*Count, error) {
|
||||
var count Count
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
err := GetInstance().CountCollection.FindOne(ctx, bson.M{"theme": themeId}).Decode(&count)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get count: %w", err)
|
||||
}
|
||||
return &count, nil
|
||||
}
|
||||
|
||||
func GetCountByGenreId(genreId uint64) (*Count, error) {
|
||||
var count Count
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
err := GetInstance().CountCollection.FindOne(ctx, bson.M{"genre": genreId}).Decode(&count)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get count: %w", err)
|
||||
}
|
||||
return &count, nil
|
||||
}
|
||||
|
||||
func SaveCount(count *Count) error {
|
||||
var filter bson.M
|
||||
if count.Genre != 0 {
|
||||
filter = bson.M{"genre": count.Genre}
|
||||
}
|
||||
if count.Theme != 0 {
|
||||
filter = bson.M{"theme": count.Theme}
|
||||
}
|
||||
update := bson.M{"$set": count}
|
||||
opts := options.UpdateOne().SetUpsert(true)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func AddThemeCount(themeId uint64) error {
|
||||
filter := bson.M{"theme": themeId}
|
||||
update := bson.M{"$inc": bson.M{"count": 1}}
|
||||
opts := options.UpdateOne().SetUpsert(true)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func AddGenreCount(genreId uint64) error {
|
||||
filter := bson.M{"genre": genreId}
|
||||
update := bson.M{"$inc": bson.M{"count": 1}}
|
||||
opts := options.UpdateOne().SetUpsert(true)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MinusThemeCount(themeId uint64) error {
|
||||
filter := bson.M{"theme": themeId}
|
||||
update := bson.M{"$inc": bson.M{"count": -1}}
|
||||
opts := options.UpdateOne().SetUpsert(true)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MinusGenreCount(genreId uint64) error {
|
||||
filter := bson.M{"genre": genreId}
|
||||
update := bson.M{"$inc": bson.M{"count": -1}}
|
||||
opts := options.UpdateOne().SetUpsert(true)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CountTheme(themeId uint64) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
filter := bson.M{"themes.id": themeId}
|
||||
count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count theme: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
func CountGenre(genreId uint64) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
filter := bson.M{"genres.id": genreId}
|
||||
count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to count genre: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
59
main.go
59
main.go
@ -34,8 +34,6 @@ func main() {
|
||||
log.Printf("data fetched")
|
||||
}
|
||||
|
||||
Count()
|
||||
|
||||
if *enableAggregate || *enableReAggregate {
|
||||
log.Printf("aggregating games")
|
||||
aggregateGames()
|
||||
@ -118,63 +116,6 @@ func fetchAndStore[T any](
|
||||
}
|
||||
}
|
||||
|
||||
func Count() {
|
||||
ids, err := db.GetAllItemsIDs[pb.Theme](endpoint.EPThemes)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to get all items ids %s: %v", endpoint.EPThemes, err)
|
||||
}
|
||||
|
||||
concurrence := make(chan struct{}, 10)
|
||||
defer close(concurrence)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
for _, id := range ids {
|
||||
concurrence <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(id uint64) {
|
||||
defer func() {
|
||||
<-concurrence
|
||||
wg.Done()
|
||||
}()
|
||||
count, err := db.CountTheme(id)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to count theme: %v", err)
|
||||
}
|
||||
log.Printf("theme %d count: %d", id, count)
|
||||
err = db.SaveCount(&db.Count{Theme: id, Count: count})
|
||||
if err != nil {
|
||||
log.Fatalf("failed to save count: %v", err)
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
|
||||
ids, err = db.GetAllItemsIDs[pb.Genre](endpoint.EPGenres)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to get all items ids %s: %v", endpoint.EPGenres, err)
|
||||
}
|
||||
for _, id := range ids {
|
||||
concurrence <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(id uint64) {
|
||||
defer func() {
|
||||
<-concurrence
|
||||
wg.Done()
|
||||
}()
|
||||
count, err := db.CountGenre(id)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to count genre: %v", err)
|
||||
}
|
||||
log.Printf("genre %d count: %d", id, count)
|
||||
err = db.SaveCount(&db.Count{Genre: id, Count: count})
|
||||
if err != nil {
|
||||
log.Fatalf("failed to save count: %v", err)
|
||||
}
|
||||
}(id)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func allFetchAndStore(client *igdb.Client) {
|
||||
if *onlyRefetchGames {
|
||||
fetchAndStore(client.Games)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user