igdb-database/main.go

package main

import (
	"igdb-database/collector"
	"igdb-database/config"
	"igdb-database/db"
	"log"
	"sync"
	"sync/atomic"

	"github.com/bestnite/go-igdb"
	"github.com/bestnite/go-igdb/endpoint"
	pb "github.com/bestnite/go-igdb/proto"
)

func main() {
	client := igdb.New(config.C().Twitch.ClientID, config.C().Twitch.ClientSecret)

	fetchAndStore(client.AgeRatingCategories)
	fetchAndStore(client.AgeRatingContentDescriptions)
	fetchAndStore(client.AgeRatingContentDescriptionsV2)
	fetchAndStore(client.AgeRatingOrganizations)
	fetchAndStore(client.AgeRatings)
	fetchAndStore(client.AlternativeNames)
	fetchAndStore(client.Artworks)
	fetchAndStore(client.CharacterGenders)
	fetchAndStore(client.CharacterMugShots)
	fetchAndStore(client.Characters)
	fetchAndStore(client.CharacterSpecies)
	fetchAndStore(client.CollectionMemberships)
	fetchAndStore(client.CollectionMembershipTypes)
	fetchAndStore(client.CollectionRelations)
	fetchAndStore(client.CollectionRelationTypes)
	fetchAndStore(client.Collections)
	fetchAndStore(client.CollectionTypes)
	fetchAndStore(client.Companies)
	fetchAndStore(client.CompanyLogos)
	fetchAndStore(client.CompanyStatuses)
	fetchAndStore(client.CompanyWebsites)
	fetchAndStore(client.Covers)
	fetchAndStore(client.DateFormats)
	fetchAndStore(client.EventLogos)
	fetchAndStore(client.EventNetworks)
	fetchAndStore(client.Events)
	fetchAndStore(client.ExternalGames)
	fetchAndStore(client.ExternalGameSources)
	fetchAndStore(client.Franchises)
	fetchAndStore(client.GameEngineLogos)
	fetchAndStore(client.GameEngines)
	fetchAndStore(client.GameLocalizations)
	fetchAndStore(client.GameModes)
	fetchAndStore(client.GameReleaseFormats)
	fetchAndStore(client.Games)
	fetchAndStore(client.GameStatuses)
	fetchAndStore(client.GameTimeToBeats)
	fetchAndStore(client.GameTypes)
	fetchAndStore(client.GameVersionFeatures)
	fetchAndStore(client.GameVersionFeatureValues)
	fetchAndStore(client.GameVersions)
	fetchAndStore(client.GameVideos)
	fetchAndStore(client.Genres)
	fetchAndStore(client.InvolvedCompanies)
	fetchAndStore(client.Keywords)
	fetchAndStore(client.Languages)
	fetchAndStore(client.LanguageSupports)
	fetchAndStore(client.LanguageSupportTypes)
	fetchAndStore(client.MultiplayerModes)
	fetchAndStore(client.NetworkTypes)
	fetchAndStore(client.PlatformFamilies)
	fetchAndStore(client.PlatformLogos)
	fetchAndStore(client.Platforms)
	fetchAndStore(client.PlatformTypes)
	fetchAndStore(client.PlatformVersionCompanies)
	fetchAndStore(client.PlatformVersionReleaseDates)
	fetchAndStore(client.PlatformVersions)
	fetchAndStore(client.PlatformWebsites)
	fetchAndStore(client.PlayerPerspectives)
	fetchAndStore(client.PopularityPrimitives)
	fetchAndStore(client.PopularityTypes)
	fetchAndStore(client.Regions)
	fetchAndStore(client.ReleaseDateRegions)
	fetchAndStore(client.ReleaseDates)
	fetchAndStore(client.ReleaseDateStatuses)
	fetchAndStore(client.Screenshots)
	fetchAndStore(client.Themes)
	fetchAndStore(client.Websites)
	fetchAndStore(client.WebsiteTypes)

	log.Printf("aggregating games")
	aggregateGames()
	log.Printf("games aggregated")

	log.Printf("starting webhook server")
	collector.StartWebhookServer(client)
}

func aggregateGames() {
	total, err := db.CountItems(endpoint.EPGames)
	if err != nil {
		log.Fatalf("failed to count games: %v", err)
	}

	finished := int64(0)
	wg := sync.WaitGroup{}

	concurrenceNum := 10
	taskOneLoop := int64(500)

	concurrence := make(chan struct{}, concurrenceNum)
	defer close(concurrence)
	for i := int64(0); i < total; i += taskOneLoop {
		concurrence <- struct{}{}
		wg.Add(1)
		go func(i int64) {
			defer func() { <-concurrence }()
			defer wg.Done()
			items, err := db.GetItemsPagnated[pb.Game](endpoint.EPGames, i, taskOneLoop)
			if err != nil {
				log.Fatalf("failed to get games: %v", err)
			}
			games := make([]*pb.Game, 0, len(items))
			for _, item := range items {
				games = append(games, item.Item)
			}
			isAggregated, err := db.IsGamesAggregated(games)
			if err != nil {
				log.Fatalf("failed to check if games are aggregated: %v", err)
			}
			for _, item := range items {
				if isAggregated[item.Item.Id] {
					p := atomic.AddInt64(&finished, 1)
					log.Printf("game aggregated %d/%d", p, total)
					continue
				}
				if err != nil {
					log.Fatalf("failed to check if game is aggregated: %v", err)
				}

				game, err := db.ConvertGame(item.Item)
				if err != nil {
					log.Fatalf("failed to convert game: %v", err)
				}
				err = db.SaveGame(game)
				if err != nil {
					log.Fatalf("failed to save game: %v", err)
				}
				p := atomic.AddInt64(&finished, 1)
				log.Printf("game aggregated %d/%d", p, total)
			}
		}(i)
	}
	wg.Wait()
}

func fetchAndStore[T any](
	e endpoint.EntityEndpoint[T],
) {
	if count, err := db.CountItems(e.GetEndpointName()); err == nil && count == 0 {
		collector.FetchAndStore(e)
	} else if err != nil {
		log.Printf("failed to count items: %v", err)
	}
}