start

collector/store.go

@@ -0,0 +1,85 @@
+package collector
+
+import (
+	"igdb-database/db"
+	"igdb-database/model"
+	"log"
+	"math"
+	"sync"
+	"sync/atomic"
+
+	"github.com/bestnite/go-igdb/endpoint"
+)
+
+func FetchAndStore[T any](
+	e endpoint.EntityEndpoint[T],
+) {
+	total, err := e.GetLastOneId()
+	if err != nil {
+		log.Fatalf("failed to get %s length: %v", e.GetEndpointName(), err)
+	}
+	log.Printf("%s length: %d", e.GetEndpointName(), total)
+	wg := sync.WaitGroup{}
+	concurrence := make(chan struct{}, 3)
+	defer close(concurrence)
+
+	totalSteps := int(math.Ceil(float64(total) / 500))
+	finished := int32(0)
+
+	for i := 0; i < int(total); i += 500 {
+		wg.Add(1)
+		concurrence <- struct{}{}
+		go func(i int) {
+			defer wg.Done()
+			defer func() { <-concurrence }()
+
+			items, err := e.Paginated(uint64(i), 500)
+			if err != nil {
+				log.Printf("failed to get items: %v", err)
+				return
+			}
+
+			type IdGetter interface {
+				GetId() uint64
+			}
+
+			ids := make([]uint64, 0, len(items))
+			for _, item := range items {
+				if v, ok := any(item).(IdGetter); ok {
+					ids = append(ids, v.GetId())
+				} else {
+					log.Printf("failed to get id from item: %v", err)
+					return
+				}
+			}
+
+			data, err := db.GetItemsByIGDBIDs[T](e.GetEndpointName(), ids)
+			if err != nil {
+				log.Printf("failed to get items: %v", err)
+				return
+			}
+
+			newItems := make([]*model.Item[T], 0, len(items))
+			for _, item := range items {
+				v, ok := any(item).(IdGetter)
+				if !ok {
+					log.Printf("failed to get id from item: %v", err)
+					return
+				} else {
+					n := model.NewItem(item)
+					n.MId = data[v.GetId()].MId
+					newItems = append(newItems, n)
+				}
+			}
+			err = db.SaveItems(e.GetEndpointName(), newItems)
+			if err != nil {
+				log.Printf("failed to save games: %v", err)
+				return
+			}
+
+			cur := atomic.AddInt32(&finished, 1)
+			log.Printf("%s finished: %d/%d", e.GetEndpointName(), cur, totalSteps)
+		}(i)
+	}
+	wg.Wait()
+}