mirror of
https://github.com/bestnite/igdb-database.git
synced 2025-04-26 21:25:54 +08:00
u
This commit is contained in:
parent
95f46cc7e7
commit
5bd2627d5e
@ -166,85 +166,15 @@ func webhook[T any](
|
|||||||
log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
|
log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
oldItem, err := db.GetItemById[T](e.GetEndpointName(), data.ID)
|
|
||||||
if err != nil && !errors.Is(err, mongo.ErrNoDocuments) {
|
|
||||||
log.Printf("failed to get %s: %v", e.GetEndpointName(), err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, ok := any(e).(*endpoint.Games); ok {
|
if _, ok := any(e).(*endpoint.Games); ok {
|
||||||
if oldItem != nil {
|
game := any(item).(*pb.Game)
|
||||||
// oldGame update
|
g, err := db.ConvertGame(game)
|
||||||
oldGame := any(oldItem).(*pb.Game)
|
if err != nil {
|
||||||
// compare themes and genres
|
log.Printf("failed to convert game: %v", err)
|
||||||
game := any(item).(*pb.Game)
|
|
||||||
|
|
||||||
oldGameThemeIds := make([]uint64, 0, len(oldGame.Themes))
|
|
||||||
for _, t := range oldGame.Themes {
|
|
||||||
oldGameThemeIds = append(oldGameThemeIds, t.Id)
|
|
||||||
}
|
|
||||||
gameThemeIds := make([]uint64, 0, len(game.Themes))
|
|
||||||
for _, t := range game.Themes {
|
|
||||||
gameThemeIds = append(gameThemeIds, t.Id)
|
|
||||||
}
|
|
||||||
if !slices.Equal(oldGameThemeIds, gameThemeIds) {
|
|
||||||
for _, t := range oldGameThemeIds {
|
|
||||||
if !slices.Contains(gameThemeIds, t) {
|
|
||||||
_ = db.MinusThemeCount(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, t := range gameThemeIds {
|
|
||||||
if !slices.Contains(oldGameThemeIds, t) {
|
|
||||||
_ = db.AddThemeCount(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
oldGameGenreIds := make([]uint64, 0, len(oldGame.Genres))
|
|
||||||
for _, t := range oldGame.Genres {
|
|
||||||
oldGameGenreIds = append(oldGameGenreIds, t.Id)
|
|
||||||
}
|
|
||||||
gameGenreIds := make([]uint64, 0, len(game.Genres))
|
|
||||||
for _, t := range game.Genres {
|
|
||||||
gameGenreIds = append(gameGenreIds, t.Id)
|
|
||||||
}
|
|
||||||
if !slices.Equal(oldGameGenreIds, gameGenreIds) {
|
|
||||||
for _, t := range oldGameGenreIds {
|
|
||||||
if !slices.Contains(gameGenreIds, t) {
|
|
||||||
_ = db.MinusGenreCount(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, t := range gameGenreIds {
|
|
||||||
if !slices.Contains(oldGameGenreIds, t) {
|
|
||||||
_ = db.AddGenreCount(t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
g, err := db.ConvertGame(game)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("failed to convert game: %v", err)
|
|
||||||
} else {
|
|
||||||
_ = db.SaveGame(g)
|
|
||||||
log.Printf("game %d aggregated", data.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
// new game
|
_ = db.SaveGame(g)
|
||||||
game := any(item).(*pb.Game)
|
log.Printf("game %d aggregated", data.ID)
|
||||||
for _, t := range game.Themes {
|
|
||||||
_ = db.AddThemeCount(t.Id)
|
|
||||||
}
|
|
||||||
for _, t := range game.Genres {
|
|
||||||
_ = db.AddGenreCount(t.Id)
|
|
||||||
}
|
|
||||||
g, err := db.ConvertGame(game)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("failed to convert game: %v", err)
|
|
||||||
} else {
|
|
||||||
_ = db.SaveGame(g)
|
|
||||||
log.Printf("game %d aggregated", data.ID)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
137
db/count.go
137
db/count.go
@ -1,137 +0,0 @@
|
|||||||
package db
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/bestnite/go-igdb/endpoint"
|
|
||||||
"go.mongodb.org/mongo-driver/v2/bson"
|
|
||||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
|
||||||
)
|
|
||||||
|
|
||||||
type Count struct {
|
|
||||||
Theme uint64 `json:"theme,omitempty"`
|
|
||||||
Genre uint64 `json:"genre,omitempty"`
|
|
||||||
Count int64 `json:"count,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetCountByThemeId(themeId uint64) (*Count, error) {
|
|
||||||
var count Count
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
err := GetInstance().CountCollection.FindOne(ctx, bson.M{"theme": themeId}).Decode(&count)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to get count: %w", err)
|
|
||||||
}
|
|
||||||
return &count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GetCountByGenreId(genreId uint64) (*Count, error) {
|
|
||||||
var count Count
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
err := GetInstance().CountCollection.FindOne(ctx, bson.M{"genre": genreId}).Decode(&count)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to get count: %w", err)
|
|
||||||
}
|
|
||||||
return &count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func SaveCount(count *Count) error {
|
|
||||||
var filter bson.M
|
|
||||||
if count.Genre != 0 {
|
|
||||||
filter = bson.M{"genre": count.Genre}
|
|
||||||
}
|
|
||||||
if count.Theme != 0 {
|
|
||||||
filter = bson.M{"theme": count.Theme}
|
|
||||||
}
|
|
||||||
update := bson.M{"$set": count}
|
|
||||||
opts := options.UpdateOne().SetUpsert(true)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func AddThemeCount(themeId uint64) error {
|
|
||||||
filter := bson.M{"theme": themeId}
|
|
||||||
update := bson.M{"$inc": bson.M{"count": 1}}
|
|
||||||
opts := options.UpdateOne().SetUpsert(true)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func AddGenreCount(genreId uint64) error {
|
|
||||||
filter := bson.M{"genre": genreId}
|
|
||||||
update := bson.M{"$inc": bson.M{"count": 1}}
|
|
||||||
opts := options.UpdateOne().SetUpsert(true)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func MinusThemeCount(themeId uint64) error {
|
|
||||||
filter := bson.M{"theme": themeId}
|
|
||||||
update := bson.M{"$inc": bson.M{"count": -1}}
|
|
||||||
opts := options.UpdateOne().SetUpsert(true)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func MinusGenreCount(genreId uint64) error {
|
|
||||||
filter := bson.M{"genre": genreId}
|
|
||||||
update := bson.M{"$inc": bson.M{"count": -1}}
|
|
||||||
opts := options.UpdateOne().SetUpsert(true)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
_, err := GetInstance().CountCollection.UpdateOne(ctx, filter, update, opts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func CountTheme(themeId uint64) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
filter := bson.M{"themes.id": themeId}
|
|
||||||
count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("failed to count theme: %w", err)
|
|
||||||
}
|
|
||||||
return count, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func CountGenre(genreId uint64) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
filter := bson.M{"genres.id": genreId}
|
|
||||||
count, err := GetInstance().Collections[endpoint.EPGames].CountDocuments(ctx, filter)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("failed to count genre: %w", err)
|
|
||||||
}
|
|
||||||
return count, nil
|
|
||||||
}
|
|
||||||
59
main.go
59
main.go
@ -34,8 +34,6 @@ func main() {
|
|||||||
log.Printf("data fetched")
|
log.Printf("data fetched")
|
||||||
}
|
}
|
||||||
|
|
||||||
Count()
|
|
||||||
|
|
||||||
if *enableAggregate || *enableReAggregate {
|
if *enableAggregate || *enableReAggregate {
|
||||||
log.Printf("aggregating games")
|
log.Printf("aggregating games")
|
||||||
aggregateGames()
|
aggregateGames()
|
||||||
@ -118,63 +116,6 @@ func fetchAndStore[T any](
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func Count() {
|
|
||||||
ids, err := db.GetAllItemsIDs[pb.Theme](endpoint.EPThemes)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to get all items ids %s: %v", endpoint.EPThemes, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
concurrence := make(chan struct{}, 10)
|
|
||||||
defer close(concurrence)
|
|
||||||
wg := sync.WaitGroup{}
|
|
||||||
|
|
||||||
for _, id := range ids {
|
|
||||||
concurrence <- struct{}{}
|
|
||||||
wg.Add(1)
|
|
||||||
go func(id uint64) {
|
|
||||||
defer func() {
|
|
||||||
<-concurrence
|
|
||||||
wg.Done()
|
|
||||||
}()
|
|
||||||
count, err := db.CountTheme(id)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to count theme: %v", err)
|
|
||||||
}
|
|
||||||
log.Printf("theme %d count: %d", id, count)
|
|
||||||
err = db.SaveCount(&db.Count{Theme: id, Count: count})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to save count: %v", err)
|
|
||||||
}
|
|
||||||
}(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
ids, err = db.GetAllItemsIDs[pb.Genre](endpoint.EPGenres)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to get all items ids %s: %v", endpoint.EPGenres, err)
|
|
||||||
}
|
|
||||||
for _, id := range ids {
|
|
||||||
concurrence <- struct{}{}
|
|
||||||
wg.Add(1)
|
|
||||||
go func(id uint64) {
|
|
||||||
defer func() {
|
|
||||||
<-concurrence
|
|
||||||
wg.Done()
|
|
||||||
}()
|
|
||||||
count, err := db.CountGenre(id)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to count genre: %v", err)
|
|
||||||
}
|
|
||||||
log.Printf("genre %d count: %d", id, count)
|
|
||||||
err = db.SaveCount(&db.Count{Genre: id, Count: count})
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("failed to save count: %v", err)
|
|
||||||
}
|
|
||||||
}(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func allFetchAndStore(client *igdb.Client) {
|
func allFetchAndStore(client *igdb.Client) {
|
||||||
if *onlyRefetchGames {
|
if *onlyRefetchGames {
|
||||||
fetchAndStore(client.Games)
|
fetchAndStore(client.Games)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user