Compare commits
No commits in common. "main" and "v0.0.6-alpha.1" have entirely different histories.
main
...
v0.0.6-alp
1
.gitignore
vendored
1
.gitignore
vendored
@ -8,4 +8,3 @@ docs
|
||||
deploy.sh
|
||||
config.json
|
||||
organize.json
|
||||
cmd/test.go
|
@ -9,10 +9,11 @@ pcgamedb is a powerful command-line tool designed to scrape and manage repack ga
|
||||
- Fitgirl
|
||||
- KaOSKrew
|
||||
- DODI
|
||||
- FreeGOG
|
||||
- ~~FreeGOG~~
|
||||
- GOGGames
|
||||
- OnlineFix
|
||||
- Xatab
|
||||
- ~~ARMGDDN~~
|
||||
- SteamRIP
|
||||
- Chovka
|
||||
|
||||
|
9
cache/redis.go
vendored
9
cache/redis.go
vendored
@ -16,6 +16,9 @@ var cache *redis.Client
|
||||
var mutx = &sync.RWMutex{}
|
||||
|
||||
func connect() {
|
||||
if !config.Config.RedisAvaliable {
|
||||
return
|
||||
}
|
||||
cache = redis.NewClient(&redis.Options{
|
||||
Addr: fmt.Sprintf("%s:%d", config.Config.Redis.Host, config.Config.Redis.Port),
|
||||
Password: config.Config.Redis.Password,
|
||||
@ -66,14 +69,14 @@ func Get(key string) (string, bool) {
|
||||
return value, true
|
||||
}
|
||||
|
||||
func Set(key string, value interface{}) error {
|
||||
func Add(key string, value interface{}) error {
|
||||
CheckConnect()
|
||||
ctx := context.Background()
|
||||
cmd := cache.Set(ctx, key, value, 0)
|
||||
cmd := cache.Set(ctx, key, value, 7*24*time.Hour)
|
||||
return cmd.Err()
|
||||
}
|
||||
|
||||
func SetWithExpire(key string, value interface{}, expire time.Duration) error {
|
||||
func AddWithExpire(key string, value interface{}, expire time.Duration) error {
|
||||
CheckConnect()
|
||||
ctx := context.Background()
|
||||
cmd := cache.Set(ctx, key, value, expire)
|
||||
|
@ -4,9 +4,9 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/log"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
138
cmd/format.go
Normal file
138
cmd/format.go
Normal file
@ -0,0 +1,138 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var formatCmd = &cobra.Command{
|
||||
Use: "format",
|
||||
Short: "Format game downloads name by formatter",
|
||||
Long: "Format game downloads name by formatter",
|
||||
Run: formatRun,
|
||||
}
|
||||
|
||||
type FormatCommandConfig struct {
|
||||
Source string
|
||||
}
|
||||
|
||||
var formatCmdCfg FormatCommandConfig
|
||||
|
||||
func init() {
|
||||
formatCmd.Flags().StringVarP(&formatCmdCfg.Source, "source", "s", "", "source to fix (fitgirl/dodi/kaoskrew/freegog/xatab/onlinefix/armgddn)")
|
||||
RootCmd.AddCommand(formatCmd)
|
||||
}
|
||||
|
||||
func formatRun(cmd *cobra.Command, args []string) {
|
||||
formatCmdCfg.Source = strings.ToLower(formatCmdCfg.Source)
|
||||
switch formatCmdCfg.Source {
|
||||
case "dodi":
|
||||
items, err := db.GetDODIGameItems()
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
oldName := item.Name
|
||||
item.Name = crawler.DODIFormatter(item.RawName)
|
||||
if oldName != item.Name {
|
||||
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
|
||||
err := db.SaveGameItem(item)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to update item", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "kaoskrew":
|
||||
items, err := db.GetKaOsKrewGameItems()
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
oldName := item.Name
|
||||
item.Name = crawler.KaOsKrewFormatter(item.RawName)
|
||||
if oldName != item.Name {
|
||||
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
|
||||
err := db.SaveGameItem(item)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to update item", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "freegog":
|
||||
items, err := db.GetFreeGOGGameItems()
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
oldName := item.Name
|
||||
item.Name = crawler.FreeGOGFormatter(item.RawName)
|
||||
if oldName != item.Name {
|
||||
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
|
||||
err := db.SaveGameItem(item)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to update item", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "xatab":
|
||||
items, err := db.GetXatabGameItems()
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
oldName := item.Name
|
||||
item.Name = crawler.XatabFormatter(item.RawName)
|
||||
if oldName != item.Name {
|
||||
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
|
||||
err := db.SaveGameItem(item)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to update item", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "onlinefix":
|
||||
items, err := db.GetOnlineFixGameItems()
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
oldName := item.Name
|
||||
item.Name = crawler.OnlineFixFormatter(item.RawName)
|
||||
if oldName != item.Name {
|
||||
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
|
||||
err := db.SaveGameItem(item)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to update item", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
case "armgddn":
|
||||
items, err := db.GetARMGDDNGameItems()
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
oldName := item.Name
|
||||
item.Name = crawler.ARMGDDNFormatter(item.RawName)
|
||||
if oldName != item.Name {
|
||||
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
|
||||
err := db.SaveGameItem(item)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to update item", zap.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,10 +1,10 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/log"
|
||||
"github.com/spf13/cobra"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type importCommandConfig struct {
|
||||
|
@ -33,11 +33,16 @@ func organizeRun(cmd *cobra.Command, args []string) {
|
||||
log.Logger.Error("Failed to get games", zap.Error(err))
|
||||
}
|
||||
for _, game := range games {
|
||||
err := crawler.OrganizeGameItem(game)
|
||||
gameInfo, err := crawler.OrganizeGameItem(game)
|
||||
if err == nil {
|
||||
err = db.SaveGameInfo(gameInfo)
|
||||
if err != nil {
|
||||
log.Logger.Error("failed to organize game item", zap.String("name", game.Name), zap.Error(err))
|
||||
log.Logger.Error("Failed to save game info", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
log.Logger.Info("game item organized", zap.String("name", game.Name))
|
||||
log.Logger.Info("Organized game", zap.String("name", game.Name))
|
||||
} else {
|
||||
log.Logger.Error("Failed to organize game", zap.String("name", game.Name))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -38,7 +38,7 @@ func init() {
|
||||
}
|
||||
|
||||
func addRun(cmd *cobra.Command, args []string) {
|
||||
var c []*ManualCommandConfig
|
||||
c := []*ManualCommandConfig{}
|
||||
if manualCmdCfg.Config != "" {
|
||||
data, err := os.ReadFile(manualCmdCfg.Config)
|
||||
if err != nil {
|
||||
|
@ -1,9 +1,9 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"go.uber.org/zap"
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/log"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
66
cmd/task.go
66
cmd/task.go
@ -10,63 +10,31 @@ import (
|
||||
)
|
||||
|
||||
type taskCommandConfig struct {
|
||||
Cron string
|
||||
Now bool
|
||||
Crawl bool
|
||||
CrawlCron string
|
||||
}
|
||||
|
||||
var taskCommandCfg taskCommandConfig
|
||||
|
||||
var crawlTaskCmd = &cobra.Command{
|
||||
Use: "crawl",
|
||||
Long: "Start crawl task",
|
||||
Short: "Start crawl task",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if taskCommandCfg.Now {
|
||||
task.Crawl(log.Logger)
|
||||
}
|
||||
c := cron.New()
|
||||
_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.Crawl(log.Logger) })
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to add task", zap.Error(err))
|
||||
}
|
||||
c.Start()
|
||||
select {}
|
||||
},
|
||||
}
|
||||
|
||||
var updateTaskCmd = &cobra.Command{
|
||||
Use: "update",
|
||||
Long: "Start update outdated game infos task",
|
||||
Short: "Start update outdated game infos task",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if taskCommandCfg.Now {
|
||||
task.UpdateOutdatedGameInfos(log.Logger)
|
||||
}
|
||||
c := cron.New()
|
||||
_, err := c.AddFunc(taskCommandCfg.Cron, func() { task.UpdateOutdatedGameInfos(log.Logger) })
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to add task", zap.Error(err))
|
||||
}
|
||||
c.Start()
|
||||
select {}
|
||||
},
|
||||
}
|
||||
var taskCmdCfg taskCommandConfig
|
||||
|
||||
var taskCmd = &cobra.Command{
|
||||
Use: "task",
|
||||
Long: "Start task",
|
||||
Short: "Start task",
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
if taskCmdCfg.Crawl {
|
||||
task.Crawl(log.Logger)
|
||||
c := cron.New()
|
||||
_, err := c.AddFunc(taskCmdCfg.CrawlCron, func() { task.Crawl(log.Logger) })
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to add task", zap.Error(err))
|
||||
}
|
||||
c.Start()
|
||||
select {}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
crawlTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
|
||||
crawlTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
|
||||
|
||||
updateTaskCmd.Flags().StringVar(&taskCommandCfg.Cron, "cron", "0 */3 * * *", "cron expression")
|
||||
updateTaskCmd.Flags().BoolVar(&taskCommandCfg.Now, "now", false, "run task immediately")
|
||||
|
||||
taskCmd.AddCommand(crawlTaskCmd)
|
||||
taskCmd.AddCommand(updateTaskCmd)
|
||||
|
||||
taskCmd.Flags().BoolVar(&taskCmdCfg.Crawl, "crawl", false, "enable auto crawl")
|
||||
taskCmd.Flags().StringVar(&taskCmdCfg.CrawlCron, "crawl-cron", "0 */3 * * *", "crawl cron expression")
|
||||
RootCmd.AddCommand(taskCmd)
|
||||
}
|
||||
|
57
cmd/update.go
Normal file
57
cmd/update.go
Normal file
@ -0,0 +1,57 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/log"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var updateCmd = &cobra.Command{
|
||||
Use: "update",
|
||||
Long: "Update game info by game data platform",
|
||||
Short: "Update game info by game data platform",
|
||||
Run: updateRun,
|
||||
}
|
||||
|
||||
type updateCommandConfig struct {
|
||||
PlatformID int
|
||||
Platform string
|
||||
ID string
|
||||
}
|
||||
|
||||
var updateCmdcfx updateCommandConfig
|
||||
|
||||
func init() {
|
||||
updateCmd.Flags().IntVarP(&updateCmdcfx.PlatformID, "platform-id", "p", 0, "platform id")
|
||||
updateCmd.Flags().StringVarP(&updateCmdcfx.Platform, "platform", "t", "", "platform")
|
||||
updateCmd.Flags().StringVarP(&updateCmdcfx.ID, "game-id", "i", "", "game info id")
|
||||
RootCmd.AddCommand(updateCmd)
|
||||
}
|
||||
|
||||
func updateRun(cmd *cobra.Command, args []string) {
|
||||
id, err := primitive.ObjectIDFromHex(updateCmdcfx.ID)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to parse game info id", zap.Error(err))
|
||||
return
|
||||
}
|
||||
oldInfo, err := db.GetGameInfoByID(id)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to get game info", zap.Error(err))
|
||||
return
|
||||
}
|
||||
newInfo, err := crawler.GenerateGameInfo(updateCmdcfx.Platform, updateCmdcfx.PlatformID)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to generate game info", zap.Error(err))
|
||||
return
|
||||
}
|
||||
newInfo.ID = id
|
||||
newInfo.GameIDs = oldInfo.GameIDs
|
||||
err = db.SaveGameInfo(newInfo)
|
||||
if err != nil {
|
||||
log.Logger.Error("Failed to save game info", zap.Error(err))
|
||||
}
|
||||
}
|
@ -20,7 +20,10 @@ type config struct {
|
||||
Twitch twitch `json:"twitch"`
|
||||
Webhooks webhooks `json:"webhooks"`
|
||||
CFClearanceScraper cfClearanceScraper `json:"cf_clearance_scraper"`
|
||||
DatabaseAvaliable bool
|
||||
OnlineFixAvaliable bool
|
||||
MegaAvaliable bool
|
||||
RedisAvaliable bool
|
||||
}
|
||||
|
||||
type cfClearanceScraper struct {
|
||||
@ -94,20 +97,11 @@ func init() {
|
||||
}
|
||||
}
|
||||
loadEnvVariables(&Config)
|
||||
Config.OnlineFixAvaliable = Config.OnlineFix.User != "" && Config.OnlineFix.Password != ""
|
||||
Config.RedisAvaliable = Config.Redis.Host != ""
|
||||
Config.DatabaseAvaliable = Config.Database.Database != "" && Config.Database.Host != ""
|
||||
|
||||
if Config.OnlineFix.User == "" || Config.OnlineFix.Password == "" {
|
||||
panic("Need OnlineFix User and Password")
|
||||
}
|
||||
if Config.Redis.Host == "" {
|
||||
panic("Need Redis Host")
|
||||
}
|
||||
if Config.Database.Database == "" || Config.Database.Host == "" {
|
||||
panic("Need Database Name and Host")
|
||||
}
|
||||
if Config.CFClearanceScraper.Url == "" {
|
||||
panic("Need CF Clearance Scraper URL")
|
||||
}
|
||||
if !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") {
|
||||
if Config.CFClearanceScraper.Url != "" && !strings.HasSuffix(Config.CFClearanceScraper.Url, "/cf-clearance-scraper") {
|
||||
Config.CFClearanceScraper.Url += "/cf-clearance-scraper"
|
||||
}
|
||||
}
|
||||
|
3
constant/graphql.go
Normal file
3
constant/graphql.go
Normal file
@ -0,0 +1,3 @@
|
||||
package constant
|
||||
|
||||
const EpicStoreSearchQuery = "query searchStoreQuery($allowCountries: String, $category: String, $count: Int, $country: String!, $keywords: String, $locale: String, $namespace: String, $itemNs: String, $sortBy: String, $sortDir: String, $start: Int, $tag: String, $releaseDate: String, $withPrice: Boolean = false, $withPromotions: Boolean = false) {\n Catalog {\n searchStore(allowCountries: $allowCountries, category: $category, count: $count, country: $country, keywords: $keywords, locale: $locale, namespace: $namespace, itemNs: $itemNs, sortBy: $sortBy, sortDir: $sortDir, releaseDate: $releaseDate, start: $start, tag: $tag) {\n elements {\n title\n id\n namespace\n description\n effectiveDate\n keyImages {\n type\n url\n }\n seller {\n id\n name\n }\n productSlug\n urlSlug\n url\n tags {\n id\n }\n items {\n id\n namespace\n }\n customAttributes {\n key\n value\n }\n categories {\n path\n }\n price(country: $country) @include(if: $withPrice) {\n totalPrice {\n discountPrice\n originalPrice\n voucherDiscount\n discount\n currencyCode\n currencyInfo {\n decimals\n }\n fmtPrice(locale: $locale) {\n originalPrice\n discountPrice\n intermediatePrice\n }\n }\n lineOffers {\n appliedRules {\n id\n endDate\n discountSetting {\n discountType\n }\n }\n }\n }\n promotions(category: $category) @include(if: $withPromotions) {\n promotionalOffers {\n promotionalOffers {\n startDate\n endDate\n discountSetting {\n discountType\n discountPercentage\n }\n }\n }\n upcomingPromotionalOffers {\n promotionalOffers {\n startDate\n endDate\n discountSetting {\n discountType\n discountPercentage\n }\n }\n }\n }\n }\n paging {\n count\n total\n }\n }\n }\n}\n"
|
@ -5,7 +5,7 @@ type language struct {
|
||||
NativeName string `json:"native_name"`
|
||||
}
|
||||
|
||||
var IGDBLanguages = map[int]language{
|
||||
var IGDBLanguages map[int]language = map[int]language{
|
||||
1: {
|
||||
Name: "Arabic",
|
||||
NativeName: "العربية",
|
||||
|
@ -4,9 +4,8 @@ const (
|
||||
C1337xBaseURL = "https://www.1337x.to"
|
||||
FreeGOGListURL = "https://freegogpcgames.com/a-z-games-list"
|
||||
GOGGamesBaseURL = "https://www.gog-games.to"
|
||||
GOGGamesURL = "https://www.gog-games.to/search?page=%v&search=&in_dev_filter=none&sort_by=last_update_desc"
|
||||
GOGGamesPageURL = "https://www.gog-games.to/game/%s"
|
||||
GOGGamesGameAPIURL = "https://www.gog-games.to/api/v1/games/%s"
|
||||
GOGGamesURL = "https://www.gog-games.to/search?page=%v&search=&is_new=false&is_updated=true&in_dev_filter=none&sort_by=last_update_desc"
|
||||
GOGGamesPageURL = "https://www.gog-games.to/games/%s"
|
||||
SteamSearchURL = "https://store.steampowered.com/search"
|
||||
SteamAppDetailURL = "https://store.steampowered.com/api/appdetails"
|
||||
SteamAllAppsURL = "https://api.steampowered.com/ISteamApps/GetAppList/v2/?format=json"
|
||||
@ -19,12 +18,10 @@ const (
|
||||
IGDBSearchURL = "https://api.igdb.com/v4/search"
|
||||
IGDBCompaniesURL = "https://api.igdb.com/v4/companies"
|
||||
IGDBWebsitesURL = "https://api.igdb.com/v4/websites"
|
||||
IGDBPopularityURL = "https://api.igdb.com/v4/popularity_primitives"
|
||||
TwitchAuthURL = "https://id.twitch.tv/oauth2/token"
|
||||
Steam250Top250URL = "https://steam250.com/top250"
|
||||
Steam250BestOfTheYearURL = "https://steam250.com/%v"
|
||||
Steam250WeekTop50URL = "https://steam250.com/7day"
|
||||
Steam250MonthTop50URL = "https://steam250.com/30day"
|
||||
Steam250MostPlayedURL = "https://steam250.com/most_played"
|
||||
FitGirlURL = "https://fitgirl-repacks.site/page/%v/"
|
||||
SteamRIPBaseURL = "https://steamrip.com"
|
||||
|
@ -21,33 +21,35 @@ type Formatter func(string) string
|
||||
|
||||
type s1337xCrawler struct {
|
||||
source string
|
||||
platform string
|
||||
formatter Formatter
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
func New1337xCrawler(source string, platform string, formatter Formatter, logger *zap.Logger) *s1337xCrawler {
|
||||
func New1337xCrawler(source string, formatter Formatter, logger *zap.Logger) *s1337xCrawler {
|
||||
return &s1337xCrawler{
|
||||
source: source,
|
||||
formatter: formatter,
|
||||
logger: logger,
|
||||
platform: platform,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *s1337xCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
var resp *utils.FetchResponse
|
||||
var doc *goquery.Document
|
||||
var err error
|
||||
requestUrl := fmt.Sprintf("%s/%s/%d/", constant.C1337xBaseURL, c.source, page)
|
||||
resp, err := utils.Request().Get(requestUrl)
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: requestUrl,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err = goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err = goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
trSelection := doc.Find("tbody>tr")
|
||||
var urls []string
|
||||
urls := []string{}
|
||||
trSelection.Each(func(i int, trNode *goquery.Selection) {
|
||||
nameSelection := trNode.Find(".name").First()
|
||||
if aNode := nameSelection.Find("a").Eq(1); aNode.Length() > 0 {
|
||||
@ -73,22 +75,30 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(URL)
|
||||
func (c *s1337xCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var item = &model.GameItem{}
|
||||
item.Url = URL
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
item.Url = url
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -98,7 +108,7 @@ func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
info[strings.TrimSpace(item.Find("strong").Text())] = strings.TrimSpace(item.Find("span").Text())
|
||||
})
|
||||
magnetRegex := regexp.MustCompile(`magnet:\?[^"]*`)
|
||||
magnetRegexRes := magnetRegex.FindStringSubmatch(string(resp.Body()))
|
||||
magnetRegexRes := magnetRegex.FindStringSubmatch(string(resp.Data))
|
||||
item.Size = info["Total size"]
|
||||
item.RawName = doc.Find("title").Text()
|
||||
item.RawName = strings.Replace(item.RawName, "Download ", "", 1)
|
||||
@ -106,7 +116,6 @@ func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
item.Name = c.formatter(item.RawName)
|
||||
item.Download = magnetRegexRes[0]
|
||||
item.Author = strings.Replace(c.source, "-torrents", "", -1)
|
||||
item.Platform = c.platform
|
||||
return item, nil
|
||||
}
|
||||
|
||||
@ -146,14 +155,18 @@ func (c *s1337xCrawler) CrawlAll() (res []*model.GameItem, err error) {
|
||||
}
|
||||
|
||||
func (c *s1337xCrawler) GetTotalPageNum() (int, error) {
|
||||
var resp *utils.FetchResponse
|
||||
var doc *goquery.Document
|
||||
var err error
|
||||
|
||||
requestUrl := fmt.Sprintf("%s/%s/%d/", constant.C1337xBaseURL, c.source, 1)
|
||||
resp, err := utils.Request().Get(requestUrl)
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: requestUrl,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
doc, _ = goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, _ = goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
selection := doc.Find(".last")
|
||||
pageStr, exist := selection.Find("a").Attr("href")
|
||||
if !exist {
|
||||
|
214
crawler/armgddn.go
Normal file
214
crawler/armgddn.go
Normal file
@ -0,0 +1,214 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"github.com/jlaffaye/ftp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
ftpAddress = "72.21.17.26:13017"
|
||||
ftpUsername = "ARMGDDNGames"
|
||||
ftpPassword = "ARMGDDNGames"
|
||||
)
|
||||
|
||||
type GameData struct {
|
||||
NumberOfGame string `json:"Number of game"`
|
||||
AppID string `json:"appid"`
|
||||
FolderName string `json:"foldername"`
|
||||
}
|
||||
|
||||
type ARMGDDNCrawler struct {
|
||||
logger zap.Logger
|
||||
conn *ftp.ServerConn
|
||||
}
|
||||
|
||||
// Deprecated: ARMGDDN has changed resource distribution method
|
||||
func NewARMGDDNCrawler(logger *zap.Logger) *ARMGDDNCrawler {
|
||||
return &ARMGDDNCrawler{
|
||||
logger: *logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) connectFTP() error {
|
||||
var err error
|
||||
tlsConfig := &tls.Config{InsecureSkipVerify: true}
|
||||
c.conn, err = ftp.Dial(ftpAddress, ftp.DialWithTimeout(5*time.Second), ftp.DialWithExplicitTLS(tlsConfig))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = c.conn.Login(ftpUsername, ftpPassword); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) fetchAndParseFTPData(filePath string) ([]GameData, error) {
|
||||
r, err := c.conn.Retr(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
buf, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var data []GameData
|
||||
if err = json.Unmarshal(buf, &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameItem, error) {
|
||||
count := 0
|
||||
var res []*model.GameItem
|
||||
modTimeMap := make(map[string]time.Time)
|
||||
entries, err := c.conn.List(fmt.Sprintf("/%s", platform))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if entry.Type == ftp.EntryTypeFolder {
|
||||
modTimeMap[entry.Name] = entry.Time
|
||||
}
|
||||
}
|
||||
for _, v := range data {
|
||||
if count == num {
|
||||
break
|
||||
}
|
||||
path := fmt.Sprintf("/%s/%s", platform, v.FolderName)
|
||||
u := fmt.Sprintf("ARMGDDNGames/%s/%s", platform, v.NumberOfGame)
|
||||
modTime, ok := modTimeMap[v.FolderName]
|
||||
if !ok {
|
||||
c.logger.Warn("mod time not found", zap.String("url", u))
|
||||
continue
|
||||
}
|
||||
updateFlag := fmt.Sprintf("ARMGDDNGames/%s/%s/%s", platform, v.NumberOfGame, modTime.UTC().String())
|
||||
if db.IsARMGDDNCrawled(updateFlag) {
|
||||
continue
|
||||
}
|
||||
c.logger.Info("Crawling", zap.String("url", u))
|
||||
walker := c.conn.Walk(path)
|
||||
size := uint64(0)
|
||||
for walker.Next() {
|
||||
if walker.Stat().Type == ftp.EntryTypeFile {
|
||||
fileSize, err := c.conn.FileSize(walker.Path())
|
||||
if err != nil {
|
||||
c.logger.Warn("file size error", zap.Error(err))
|
||||
break
|
||||
}
|
||||
size += uint64(fileSize)
|
||||
}
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(u)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
item.Url = u
|
||||
item.Name = ARMGDDNFormatter(v.FolderName)
|
||||
item.UpdateFlag = updateFlag
|
||||
item.Size = utils.BytesToSize(size)
|
||||
item.RawName = v.FolderName
|
||||
item.Author = "ARMGDDN"
|
||||
item.Download = fmt.Sprintf("ftpes://%s:%s@%s/%s/%s", ftpUsername, ftpPassword, ftpAddress, platform, url.QueryEscape(v.FolderName))
|
||||
if err := db.SaveGameItem(item); err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
count++
|
||||
var id int
|
||||
var info *model.GameInfo
|
||||
if v.AppID != "NONSTEAM" {
|
||||
id, err = strconv.Atoi(v.AppID)
|
||||
if err != nil {
|
||||
c.logger.Warn("strconv error", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
info, err = OrganizeGameItemWithSteam(id, item)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
info, err = OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("save game info error", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func ARMGDDNFormatter(name string) string {
|
||||
cleanedName := strings.ReplaceAll(strings.TrimSpace(name), "-ARMGDDN", "")
|
||||
matchIndex := regexp.MustCompile(`v\d`).FindStringIndex(cleanedName)
|
||||
if matchIndex == nil {
|
||||
return cleanedName
|
||||
}
|
||||
return strings.TrimSpace(cleanedName[:matchIndex[0]])
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameItem, error) {
|
||||
return c.crawlPlatform("/PC/currentserverPC-FTP.json", "PC", num)
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameItem, error) {
|
||||
return c.crawlPlatform("/PCVR/currentserverPCVR-FTP.json", "PCVR", num)
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
num1 := num / 2
|
||||
num2 := num - num1
|
||||
if num == -1 {
|
||||
num1 = -1
|
||||
num2 = -1
|
||||
}
|
||||
res1, err := c.CrawlPC(num1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res2, err := c.CrawlPCVR(num2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(res1, res2...), nil
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
return c.Crawl(-1)
|
||||
}
|
||||
|
||||
func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameItem, error) {
|
||||
err := c.connectFTP()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = c.conn.Quit() }()
|
||||
|
||||
data, err := c.fetchAndParseFTPData(jsonFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c.crawlGames(data, platform, num)
|
||||
}
|
@ -30,34 +30,38 @@ func (c *ChovkaCrawler) Name() string {
|
||||
return "ChovkaCrawler"
|
||||
}
|
||||
|
||||
func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(URL)
|
||||
func (c *ChovkaCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(URL)
|
||||
item, err := db.GetGameItemByUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Url = URL
|
||||
item.Url = url
|
||||
item.RawName = doc.Find(".inner-entry__title").First().Text()
|
||||
item.Name = ChovkaFormatter(item.RawName)
|
||||
item.Author = "Chovka"
|
||||
item.UpdateFlag = item.RawName
|
||||
item.Platform = "windows"
|
||||
downloadURL := doc.Find(".download-torrent").AttrOr("href", "")
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("failed to find download URL")
|
||||
return nil, errors.New("Failed to find download URL")
|
||||
}
|
||||
resp, err = utils.Request().SetHeader("Referer", URL).Get(downloadURL)
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Headers: map[string]string{"Referer": url},
|
||||
Url: downloadURL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
magnet, size, err := utils.ConvertTorrentToMagnet(resp.Body())
|
||||
magnet, size, err := utils.ConvertTorrentToMagnet(resp.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -67,16 +71,18 @@ func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(fmt.Sprintf(constant.RepackInfoURL, page))
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: fmt.Sprintf(constant.RepackInfoURL, page),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var urls []string
|
||||
var updateFlags []string
|
||||
urls := []string{}
|
||||
updateFlags := []string{}
|
||||
doc.Find(".entry").Each(func(i int, s *goquery.Selection) {
|
||||
u, exist := s.Find(".entry__title.h2 a").Attr("href")
|
||||
if !exist {
|
||||
@ -101,11 +107,15 @@ func (c *ChovkaCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
if err := db.SaveGameInfo(info); err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@ -139,11 +149,13 @@ func (c *ChovkaCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func (c *ChovkaCrawler) GetTotalPageNum() (int, error) {
|
||||
resp, err := utils.Request().Get(fmt.Sprintf(constant.RepackInfoURL, 1))
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: fmt.Sprintf(constant.RepackInfoURL, 1),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
@ -31,10 +31,10 @@ func BuildCrawlerMap(logger *zap.Logger) map[string]Crawler {
|
||||
"xatab": NewXatabCrawler(logger),
|
||||
"onlinefix": NewOnlineFixCrawler(logger),
|
||||
"steamrip": NewSteamRIPCrawler(logger),
|
||||
// "armgddn": NewARMGDDNCrawler(logger),
|
||||
"chovka": NewChovkaCrawler(logger),
|
||||
"goggames": NewGOGGamesCrawler(logger),
|
||||
"johncena141": NewJohncena141Crawler(logger),
|
||||
// "gnarly": NewGnarlyCrawler(logger),
|
||||
// "goggames": NewGOGGamesCrawler(logger),
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
@ -22,7 +22,6 @@ func NewDODICrawler(logger *zap.Logger) *DODICrawler {
|
||||
logger: logger,
|
||||
crawler: *New1337xCrawler(
|
||||
DODIName,
|
||||
"windows",
|
||||
DODIFormatter,
|
||||
logger,
|
||||
),
|
||||
|
@ -31,61 +31,64 @@ func (c *FitGirlCrawler) Name() string {
|
||||
return "FitGirlCrawler"
|
||||
}
|
||||
|
||||
func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(URL)
|
||||
func (c *FitGirlCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
titleElem := doc.Find("h3").First().Find("strong")
|
||||
if titleElem.Length() == 0 {
|
||||
return nil, errors.New("failed to find title")
|
||||
return nil, errors.New("Failed to find title")
|
||||
}
|
||||
rawTitle := titleElem.Text()
|
||||
titleElem.Children().Remove()
|
||||
title := strings.TrimSpace(titleElem.Text())
|
||||
sizeRegex := regexp.MustCompile(`Repack Size: <strong>(.*?)</strong>`)
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(string(resp.Body()))
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(sizeRegexRes) == 0 {
|
||||
return nil, errors.New("failed to find size")
|
||||
return nil, errors.New("Failed to find size")
|
||||
}
|
||||
size := sizeRegexRes[1]
|
||||
magnetRegex := regexp.MustCompile(`magnet:\?[^"]*`)
|
||||
magnetRegexRes := magnetRegex.FindStringSubmatch(string(resp.Body()))
|
||||
magnetRegexRes := magnetRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(magnetRegexRes) == 0 {
|
||||
return nil, errors.New("failed to find magnet")
|
||||
return nil, errors.New("Failed to find magnet")
|
||||
}
|
||||
magnet := magnetRegexRes[0]
|
||||
item, err := db.GetGameItemByUrl(URL)
|
||||
item, err := db.GetGameItemByUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Name = strings.TrimSpace(title)
|
||||
item.RawName = rawTitle
|
||||
item.Url = URL
|
||||
item.Url = url
|
||||
item.Size = size
|
||||
item.Author = "FitGirl"
|
||||
item.Download = magnet
|
||||
item.Platform = "windows"
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(fmt.Sprintf(constant.FitGirlURL, page))
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: fmt.Sprintf(constant.FitGirlURL, page),
|
||||
})
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to fetch", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to parse HTML", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
var urls []string
|
||||
var updateFlags []string //link+date
|
||||
urls := []string{}
|
||||
updateFlags := []string{} //link+date
|
||||
doc.Find("article").Each(func(i int, s *goquery.Selection) {
|
||||
u, exist1 := s.Find(".entry-title>a").First().Attr("href")
|
||||
d, exist2 := s.Find("time").First().Attr("datetime")
|
||||
@ -112,10 +115,16 @@ func (c *FitGirlCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@ -149,11 +158,13 @@ func (c *FitGirlCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func (c *FitGirlCrawler) GetTotalPageNum() (int, error) {
|
||||
resp, err := utils.Request().Get(fmt.Sprintf(constant.FitGirlURL, 1))
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: fmt.Sprintf(constant.FitGirlURL, 1),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
@ -3,21 +3,17 @@ package crawler
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"html"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/config"
|
||||
"pcgamedb/constant"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"git.nite07.com/nite/ccs"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
@ -32,57 +28,38 @@ func NewFreeGOGCrawler(logger *zap.Logger) *FreeGOGCrawler {
|
||||
}
|
||||
}
|
||||
|
||||
func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) {
|
||||
var session ccs.Session
|
||||
var err error
|
||||
if val, exist := cache.Get("freegog_waf_session"); exist {
|
||||
err := json.Unmarshal([]byte(val), &session)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
session, err = ccs.WAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jsonBytes, err := json.Marshal(session)
|
||||
if err == nil {
|
||||
_ = cache.SetWithExpire("freegog_waf_session", jsonBytes, 1*time.Hour)
|
||||
}
|
||||
}
|
||||
return &session, nil
|
||||
}
|
||||
|
||||
func (c *FreeGOGCrawler) Name() string {
|
||||
return "FreeGOG"
|
||||
}
|
||||
|
||||
func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
count := 0
|
||||
session, err := c.getSession()
|
||||
session, err := utils.CCSWAFSession(config.Config.CFClearanceScraper.Url, constant.FreeGOGListURL)
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to create session", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
resp, err := ccs.RequestWithWAFSession(http.MethodGet, constant.FreeGOGListURL, *session, nil)
|
||||
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
|
||||
Url: constant.FreeGOGListURL,
|
||||
}, session)
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to fetch", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader([]byte(resp.Body)))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to parse HTML", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var urls []string
|
||||
var updateFlags []string //rawName+link
|
||||
urls := []string{}
|
||||
updateFlags := []string{} //rawName+link
|
||||
doc.Find(".items-outer li a").Each(func(i int, s *goquery.Selection) {
|
||||
urls = append(urls, s.AttrOr("href", ""))
|
||||
updateFlags = append(updateFlags, s.Text()+s.AttrOr("href", ""))
|
||||
})
|
||||
|
||||
var res []*model.GameItem
|
||||
res := []*model.GameItem{}
|
||||
for i, u := range urls {
|
||||
if count == num {
|
||||
break
|
||||
@ -91,7 +68,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
continue
|
||||
}
|
||||
c.logger.Info("Crawling", zap.String("URL", u))
|
||||
item, err := c.CrawlByUrl(u)
|
||||
item, err := c.CrawlByUrl(u, session)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
@ -104,30 +81,34 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
}
|
||||
res = append(res, item)
|
||||
count++
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
session, err := c.getSession()
|
||||
func (c *FreeGOGCrawler) CrawlByUrl(url string, session *utils.WAFSession) (*model.GameItem, error) {
|
||||
resp, err := utils.FetchWithWAFSession(utils.FetchConfig{
|
||||
Url: url,
|
||||
}, session)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := ccs.RequestWithWAFSession(http.MethodGet, URL, *session, nil)
|
||||
item, err := db.GetGameItemByUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Url = URL
|
||||
item.Url = url
|
||||
rawTitleRegex := regexp.MustCompile(`(?i)<h1 class="entry-title">(.*?)</h1>`)
|
||||
rawTitleRegexRes := rawTitleRegex.FindStringSubmatch(string(resp.Body))
|
||||
rawTitleRegexRes := rawTitleRegex.FindStringSubmatch(string(resp.Data))
|
||||
rawName := ""
|
||||
if len(rawTitleRegexRes) > 1 {
|
||||
rawName = html.UnescapeString(rawTitleRegexRes[1])
|
||||
@ -137,12 +118,12 @@ func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
}
|
||||
item.Name = FreeGOGFormatter(item.RawName)
|
||||
sizeRegex := regexp.MustCompile(`(?i)>Size:\s?(.*?)<`)
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(string(resp.Body))
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(sizeRegexRes) > 1 {
|
||||
item.Size = sizeRegexRes[1]
|
||||
}
|
||||
magnetRegex := regexp.MustCompile(`<a class="download-btn" href="https://gdl.freegogpcgames.xyz/download-gen\.php\?url=(.*?)"`)
|
||||
magnetRegexRes := magnetRegex.FindStringSubmatch(string(resp.Body))
|
||||
magnetRegexRes := magnetRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(magnetRegexRes) > 1 {
|
||||
magnet, err := base64.StdEncoding.DecodeString(magnetRegexRes[1])
|
||||
if err != nil {
|
||||
@ -150,10 +131,9 @@ func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
}
|
||||
item.Download = string(magnet)
|
||||
} else {
|
||||
return nil, errors.New("failed to find magnet link")
|
||||
return nil, errors.New("Failed to find magnet link")
|
||||
}
|
||||
item.Author = "FreeGOG"
|
||||
item.Platform = "windows"
|
||||
return item, nil
|
||||
}
|
||||
|
||||
|
@ -9,7 +9,6 @@ import (
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
@ -23,33 +22,33 @@ func GenerateGameInfo(platform string, id int) (*model.GameInfo, error) {
|
||||
case "igdb":
|
||||
return GenerateIGDBGameInfo(id)
|
||||
default:
|
||||
return nil, errors.New("invalid ID type")
|
||||
return nil, errors.New("Invalid ID type")
|
||||
}
|
||||
}
|
||||
|
||||
// OrganizeGameItem Organize game item and save game info to database
|
||||
func OrganizeGameItem(game *model.GameItem) error {
|
||||
hasOriganized, _ := db.HasGameItemOrganized(game.ID)
|
||||
if hasOriganized {
|
||||
return nil
|
||||
}
|
||||
|
||||
item, err := OrganizeGameItemWithIGDB(game)
|
||||
func OrganizeGameItem(game *model.GameItem) (*model.GameInfo, error) {
|
||||
item, err := OrganizeGameItemWithIGDB(0, game)
|
||||
if err == nil {
|
||||
if item.SteamID == 0 {
|
||||
// get steam id from igdb
|
||||
steamID, err := GetSteamIDByIGDBID(item.IGDBID)
|
||||
steamID, err := GetSteamIDByIGDBIDCache(item.IGDBID)
|
||||
if err == nil {
|
||||
item.SteamID = steamID
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
err = db.SaveGameInfo(item)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
item, err = OrganizeGameItemWithSteam(0, game)
|
||||
if err == nil {
|
||||
if item.IGDBID == 0 {
|
||||
igdbID, err := GetIGDBIDBySteamIDCache(item.SteamID)
|
||||
if err == nil {
|
||||
item.IGDBID = igdbID
|
||||
}
|
||||
return err
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func AddGameInfoManually(gameID primitive.ObjectID, platform string, plateformID int) (*model.GameInfo, error) {
|
||||
@ -65,7 +64,7 @@ func AddGameInfoManually(gameID primitive.ObjectID, platform string, plateformID
|
||||
func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platformID int) (*model.GameInfo, error) {
|
||||
info, err := db.GetGameInfoByPlatformID(platform, platformID)
|
||||
if err != nil {
|
||||
if errors.Is(err, mongo.ErrNoDocuments) {
|
||||
if err == mongo.ErrNoDocuments {
|
||||
info, err = AddGameInfoManually(gameID, platform, platformID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -81,13 +80,13 @@ func OrganizeGameItemManually(gameID primitive.ObjectID, platform string, platfo
|
||||
return nil, err
|
||||
}
|
||||
if platform == "igdb" {
|
||||
steamID, err := GetSteamIDByIGDBID(platformID)
|
||||
steamID, err := GetSteamIDByIGDBIDCache(platformID)
|
||||
if err == nil {
|
||||
info.SteamID = steamID
|
||||
}
|
||||
}
|
||||
if platform == "steam" {
|
||||
igdbID, err := GetIGDBIDBySteamAppID(platformID)
|
||||
igdbID, err := GetIGDBIDBySteamIDCache(platformID)
|
||||
if err == nil {
|
||||
info.IGDBID = igdbID
|
||||
}
|
||||
@ -99,7 +98,7 @@ func FormatName(name string) string {
|
||||
name = regexp.MustCompile(`(?i)[\w’'-]+\s(Edition|Vision|Collection|Bundle|Pack|Deluxe)`).ReplaceAllString(name, " ")
|
||||
name = regexp.MustCompile(`(?i)GOTY`).ReplaceAllString(name, "")
|
||||
name = regexp.MustCompile(`(?i)nsw for pc`).ReplaceAllString(name, "")
|
||||
name = regexp.MustCompile(`\([^)]+\)`).ReplaceAllString(name, "")
|
||||
name = regexp.MustCompile(`\([^\)]+\)`).ReplaceAllString(name, "")
|
||||
name = regexp.MustCompile(`\s+`).ReplaceAllString(name, " ")
|
||||
name = strings.Replace(name, ": Remastered", "", -1)
|
||||
name = strings.Replace(name, ": Remaster", "", -1)
|
||||
@ -116,7 +115,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
|
||||
for _, info := range infos {
|
||||
changed := false
|
||||
if info.IGDBID != 0 && info.SteamID == 0 {
|
||||
steamID, err := GetSteamIDByIGDBID(info.IGDBID)
|
||||
steamID, err := GetSteamIDByIGDBIDCache(info.IGDBID)
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
if err != nil {
|
||||
continue
|
||||
@ -125,7 +124,7 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
|
||||
changed = true
|
||||
}
|
||||
if info.SteamID != 0 && info.IGDBID == 0 {
|
||||
igdbID, err := GetIGDBIDBySteamAppID(info.SteamID)
|
||||
igdbID, err := GetIGDBIDBySteamIDCache(info.SteamID)
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
if err != nil {
|
||||
continue
|
||||
@ -134,38 +133,9 @@ func SupplementPlatformIDToGameInfo(logger *zap.Logger) error {
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
logger.Info("supp", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
|
||||
logger.Info("Supplemented platform id for game info", zap.String("name", info.Name), zap.Int("igdb", int(info.IGDBID)), zap.Int("steam", int(info.SteamID)))
|
||||
_ = db.SaveGameInfo(info)
|
||||
} else {
|
||||
logger.Info("skip", zap.String("name", info.Name), zap.Int("igdb", info.IGDBID), zap.Int("steam", info.SteamID))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateGameInfo(num int) (chan *model.GameInfo, error) {
|
||||
infos, err := db.GetOutdatedGameInfos(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
updateChan := make(chan *model.GameInfo)
|
||||
|
||||
go func() {
|
||||
for _, info := range infos {
|
||||
if info.IGDBID != 0 {
|
||||
newInfo, err := GenerateIGDBGameInfo(info.IGDBID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
db.MergeGameInfo(info, newInfo)
|
||||
err = db.SaveGameInfo(newInfo)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
updateChan <- newInfo
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return updateChan, nil
|
||||
}
|
||||
|
106
crawler/gnarly.go
Normal file
106
crawler/gnarly.go
Normal file
@ -0,0 +1,106 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"pcgamedb/constant"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
type GnarlyCrawler struct {
|
||||
logger *zap.Logger
|
||||
}
|
||||
|
||||
func NewGnarlyCrawler(logger *zap.Logger) *GnarlyCrawler {
|
||||
return &GnarlyCrawler{
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *GnarlyCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
var res []*model.GameItem
|
||||
count := 0
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.GnarlyURL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sizeRegex := regexp.MustCompile(`\[(\d+)\s(GB|MB)\]`)
|
||||
pElementHtml := make([]string, 0)
|
||||
doc.Find("p").Each(func(i int, s *goquery.Selection) {
|
||||
pElementHtml = append(pElementHtml, s.Text())
|
||||
})
|
||||
for _, s := range pElementHtml {
|
||||
if strings.Contains(s, "https://bin.0xfc.de/") {
|
||||
lines := strings.Split(s, "\n")
|
||||
for i := 0; i < len(lines); i++ {
|
||||
if strings.Contains(lines[i], "[Gnarly Repacks]") {
|
||||
i++
|
||||
if strings.Contains(lines[i], "https://bin.0xfc.de/") {
|
||||
if count == num {
|
||||
return res, nil
|
||||
}
|
||||
if db.IsGnarlyCrawled(lines[i-1]) {
|
||||
continue
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(lines[i])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(lines[i])
|
||||
if len(sizeRegexRes) == 3 {
|
||||
item.Size = sizeRegexRes[1] + " " + sizeRegexRes[2]
|
||||
}
|
||||
c.logger.Info("Crawling", zap.String("Name", lines[i-1]))
|
||||
item.RawName = lines[i-1]
|
||||
item.Url = constant.GnarlyURL
|
||||
item.Author = "Gnarly"
|
||||
item.Name = GnarlyFormatter(item.RawName)
|
||||
download, err := utils.DecryptPrivateBin(lines[i], "gnarly")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
item.Download = download
|
||||
item.UpdateFlag = item.RawName
|
||||
res = append(res, item)
|
||||
count++
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save game info", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *GnarlyCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
return c.Crawl(-1)
|
||||
}
|
||||
|
||||
var parenthesesRegex = regexp.MustCompile(`\(([^)]+)\)`)
|
||||
|
||||
func GnarlyFormatter(name string) string {
|
||||
name = name[:strings.Index(name, " [Gnarly Repacks]")]
|
||||
name = parenthesesRegex.ReplaceAllString(name, "")
|
||||
return strings.TrimSpace(name)
|
||||
}
|
@ -3,7 +3,6 @@ package crawler
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -13,7 +12,6 @@ import (
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"git.nite07.com/nite/ccs"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@ -31,57 +29,37 @@ func (c *GOGGamesCrawler) Name() string {
|
||||
return "GOGGamesCrawler"
|
||||
}
|
||||
|
||||
// URL is api url, like https://www.gog-games.to/api/v1/games/%s
|
||||
func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
if !strings.HasPrefix(URL, "https://www.gog-games.to/game/") {
|
||||
return nil, fmt.Errorf("invalid url")
|
||||
}
|
||||
_, slug := path.Split(URL)
|
||||
|
||||
apiUrl := fmt.Sprintf(constant.GOGGamesGameAPIURL, slug)
|
||||
|
||||
token, err := ccs.TurnstileToken(config.Config.CFClearanceScraper.Url, apiUrl, "0x4AAAAAAAfOlgvCKbOdW1zc")
|
||||
token, err := utils.CCSTurnstileToken(config.Config.CFClearanceScraper.Url, URL, "0x4AAAAAAAfOlgvCKbOdW1zc")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := utils.Request().SetHeader("cf-turnstile-response", token).Get(apiUrl)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: URL,
|
||||
Headers: map[string]string{
|
||||
"cf-turnstile-response": token,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := gameResult{}
|
||||
err = json.Unmarshal(resp.Body(), &data)
|
||||
err = json.Unmarshal(resp.Data, &data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
name := data.Title
|
||||
|
||||
// find download links
|
||||
fileHosters := []string{
|
||||
"gofile",
|
||||
"fileditch",
|
||||
"qiwi",
|
||||
"filesfm",
|
||||
"pixeldrain",
|
||||
"1fichier",
|
||||
}
|
||||
links := make([]string, 0)
|
||||
for _, h := range fileHosters {
|
||||
if value, exist := data.Links.Game[h]; exist {
|
||||
for _, link := range value.Links {
|
||||
for _, link := range data.Links.Game.Gofile.Links {
|
||||
links = append(links, link.Link)
|
||||
}
|
||||
if len(data.Links.Patch.Gofile.Links) > 0 {
|
||||
for _, link := range data.Links.Patch.Gofile.Links {
|
||||
links = append(links, link.Link)
|
||||
}
|
||||
}
|
||||
if value, exist := data.Links.Patch[h]; exist {
|
||||
for _, link := range value.Links {
|
||||
links = append(links, link.Link)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(links) == 0 {
|
||||
return nil, fmt.Errorf("no download link found")
|
||||
}
|
||||
|
||||
size := uint64(0)
|
||||
for _, file := range data.Files.Game {
|
||||
s, _ := utils.SizeToBytes(file.Size)
|
||||
@ -98,47 +76,52 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
item.Url = URL
|
||||
item.Size = utils.BytesToSize(size)
|
||||
item.Author = "GOGGames"
|
||||
item.Platform = "windows"
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(fmt.Sprintf(constant.GOGGamesURL, page))
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: fmt.Sprintf(constant.GOGGamesURL, page),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data := searchResult{}
|
||||
err = json.Unmarshal(resp.Body(), &data)
|
||||
err = json.Unmarshal(resp.Data, &data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
urls := make([]string, 0)
|
||||
var updateFlags []string //link+date
|
||||
updateFlags := []string{} //link+date
|
||||
for _, item := range data.Data {
|
||||
urls = append(urls, fmt.Sprintf(constant.GOGGamesPageURL, item.Slug))
|
||||
updateFlags = append(updateFlags, fmt.Sprintf("%s%s", item.GogURL, item.LastUpdate))
|
||||
}
|
||||
res := make([]*model.GameItem, 0)
|
||||
for i, u := range urls {
|
||||
c.logger.Info("Crawling", zap.String("URL", u))
|
||||
if db.IsGameCrawled(updateFlags[i], "GOGGames") {
|
||||
continue
|
||||
}
|
||||
c.logger.Info("Crawling", zap.String("URL", u))
|
||||
item, err := c.CrawlByUrl(u)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to crawl", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
item.UpdateFlag = updateFlags[i]
|
||||
if err := db.SaveGameItem(item); err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
if err := db.SaveGameInfo(info); err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@ -172,12 +155,14 @@ func (c *GOGGamesCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func (c *GOGGamesCrawler) GetTotalPageNum() (int, error) {
|
||||
resp, err := utils.Request().Get(fmt.Sprintf(constant.GOGGamesURL, 1))
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: fmt.Sprintf(constant.GOGGamesURL, 1),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
data := searchResult{}
|
||||
err = json.Unmarshal(resp.Body(), &data)
|
||||
err = json.Unmarshal(resp.Data, &data)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
@ -276,21 +261,73 @@ type gameResult struct {
|
||||
} `json:"links"`
|
||||
} `json:"gofile"`
|
||||
} `json:"goodie"`
|
||||
Game map[string]struct {
|
||||
Game struct {
|
||||
OneFichier struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"1fichier"`
|
||||
Vikingfile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"vikingfile"`
|
||||
Pixeldrain struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"pixeldrain"`
|
||||
Gofile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"gofile"`
|
||||
} `json:"game"`
|
||||
Patch map[string]struct {
|
||||
Patch struct {
|
||||
OneFichier struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"1fichier"`
|
||||
Vikingfile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"vikingfile"`
|
||||
Pixeldrain struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"pixeldrain"`
|
||||
Gofile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Links []struct {
|
||||
Label string `json:"label"`
|
||||
Link string `json:"link"`
|
||||
} `json:"links"`
|
||||
} `json:"gofile"`
|
||||
} `json:"patch"`
|
||||
} `json:"links"`
|
||||
Files struct {
|
||||
|
626
crawler/igdb.go
626
crawler/igdb.go
@ -1,15 +1,14 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/config"
|
||||
@ -17,233 +16,73 @@ import (
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
type twitchToken struct {
|
||||
}
|
||||
var TwitchToken string
|
||||
|
||||
var token = twitchToken{}
|
||||
|
||||
func (t *twitchToken) getToken() (string, error) {
|
||||
if val, exist := cache.Get("twitch_token"); exist {
|
||||
return val, nil
|
||||
}
|
||||
token, expires, err := loginTwitch()
|
||||
func _GetIGDBID(name string) (int, error) {
|
||||
var err error
|
||||
if TwitchToken == "" {
|
||||
TwitchToken, err = LoginTwitch()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to login twitch: %w", err)
|
||||
return 0, fmt.Errorf("failed to login twitch: %w", err)
|
||||
}
|
||||
_ = cache.SetWithExpire("twitch_token", token, expires)
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func loginTwitch() (string, time.Duration, error) {
|
||||
baseURL, _ := url.Parse(constant.TwitchAuthURL)
|
||||
params := url.Values{}
|
||||
params.Add("client_id", config.Config.Twitch.ClientID)
|
||||
params.Add("client_secret", config.Config.Twitch.ClientSecret)
|
||||
params.Add("grant_type", "client_credentials")
|
||||
baseURL.RawQuery = params.Encode()
|
||||
resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
data := struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
TokenType string `json:"token_type"`
|
||||
}{}
|
||||
err = json.Unmarshal(resp.Body(), &data)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
|
||||
}
|
||||
|
||||
func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
|
||||
t, err := token.getToken()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBSearchURL,
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + t,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
}).Post(URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
},
|
||||
Data: fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name),
|
||||
Method: "POST",
|
||||
})
|
||||
if string(resp.Data) == "[]" {
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBSearchURL,
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
Data: fmt.Sprintf(`search "%s"; fields *; limit 50;`, name),
|
||||
Method: "POST",
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func getIGDBID(name string) (int, error) {
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if string(resp.Body()) == "[]" {
|
||||
resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
var data model.IGDBSearches
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
|
||||
}
|
||||
if len(data) == 1 {
|
||||
return GetIGDBAppParent(data[0].Game)
|
||||
return data[0].Game, nil
|
||||
}
|
||||
maxSimilairty := 0.0
|
||||
maxSimilairtyIndex := 0
|
||||
for i, item := range data {
|
||||
for _, item := range data {
|
||||
if strings.EqualFold(item.Name, name) {
|
||||
return item.Game, nil
|
||||
}
|
||||
if sim := utils.Similarity(name, item.Name); sim >= 0.8 {
|
||||
if sim > maxSimilairty {
|
||||
maxSimilairty = sim
|
||||
maxSimilairtyIndex = i
|
||||
if utils.Similarity(name, item.Name) >= 0.8 {
|
||||
return item.Game, nil
|
||||
}
|
||||
}
|
||||
detail, err := GetIGDBAppDetail(item.Game)
|
||||
detail, err := GetIGDBAppDetailCache(item.Game)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, alternativeNames := range detail.AlternativeNames {
|
||||
if sim := utils.Similarity(alternativeNames.Name, name); sim >= 0.8 {
|
||||
if sim > maxSimilairty {
|
||||
maxSimilairty = sim
|
||||
maxSimilairtyIndex = i
|
||||
if utils.Similarity(alternativeNames.Name, name) >= 0.8 {
|
||||
return item.Game, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if maxSimilairty >= 0.8 {
|
||||
return GetIGDBAppParent(data[maxSimilairtyIndex].Game)
|
||||
}
|
||||
return 0, fmt.Errorf("IGDB ID not found: %s", name)
|
||||
}
|
||||
|
||||
func getIGDBIDBySteamSearch(name string) (int, error) {
|
||||
baseURL, _ := url.Parse(constant.SteamSearchURL)
|
||||
params := url.Values{}
|
||||
params.Add("term", name)
|
||||
baseURL.RawQuery = params.Encode()
|
||||
|
||||
resp, err := utils.Request().Get(baseURL.String())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type searchResult struct {
|
||||
ID int
|
||||
Type string
|
||||
Name string
|
||||
}
|
||||
var items []searchResult
|
||||
doc.Find(".search_result_row").Each(func(i int, s *goquery.Selection) {
|
||||
if itemKey, exist := s.Attr("data-ds-itemkey"); exist {
|
||||
if strings.HasPrefix(itemKey, "App_") {
|
||||
id, err := strconv.Atoi(itemKey[4:])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
name := s.Find(".title").Text()
|
||||
items = append(items, searchResult{
|
||||
ID: id,
|
||||
Type: "App",
|
||||
Name: name,
|
||||
})
|
||||
}
|
||||
if strings.HasPrefix(itemKey, "Bundle_") {
|
||||
id, err := strconv.Atoi(itemKey[7:])
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
name := s.Find(".title").Text()
|
||||
items = append(items, searchResult{
|
||||
ID: id,
|
||||
Type: "Bundle",
|
||||
Name: name,
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
maxSim := 0.0
|
||||
var maxSimItem searchResult
|
||||
for _, item := range items {
|
||||
if strings.EqualFold(strings.TrimSpace(item.Name), strings.TrimSpace(name)) {
|
||||
maxSimItem = item
|
||||
break
|
||||
} else {
|
||||
sim := utils.Similarity(item.Name, name)
|
||||
if sim >= 0.8 && sim > maxSim {
|
||||
maxSim = sim
|
||||
maxSimItem = item
|
||||
}
|
||||
}
|
||||
}
|
||||
if maxSim != 0 {
|
||||
if maxSimItem.Type == "App" {
|
||||
return GetIGDBIDBySteamAppID(maxSimItem.ID)
|
||||
}
|
||||
if maxSimItem.Type == "Bundle" {
|
||||
return GetIGDBIDBySteamBundleID(maxSimItem.ID)
|
||||
}
|
||||
}
|
||||
return 0, fmt.Errorf("steam ID not found: %s", name)
|
||||
}
|
||||
|
||||
// GetIGDBAppParent returns the parent of the game, if no parent return itself
|
||||
func GetIGDBAppParent(id int) (int, error) {
|
||||
key := fmt.Sprintf("igdb_parent:%d", id)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
id, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
detail, err := GetIGDBAppDetail(id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
hasParent := false
|
||||
for detail.VersionParent != 0 {
|
||||
hasParent = true
|
||||
detail, err = GetIGDBAppDetail(detail.VersionParent)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
if hasParent {
|
||||
return detail.ID, nil
|
||||
}
|
||||
|
||||
_ = cache.Set(key, id)
|
||||
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// GetIGDBID returns the IGDB ID of the game, try directly IGDB api first, then steam search
|
||||
func GetIGDBID(name string) (int, error) {
|
||||
key := fmt.Sprintf("igdb_id:%s", name)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
return strconv.Atoi(val)
|
||||
}
|
||||
name1 := name
|
||||
name2 := FormatName(name)
|
||||
names := []string{name1}
|
||||
@ -251,40 +90,61 @@ func GetIGDBID(name string) (int, error) {
|
||||
names = append(names, name2)
|
||||
}
|
||||
for _, name := range names {
|
||||
id, err := getIGDBID(name)
|
||||
id, err := _GetIGDBID(name)
|
||||
if err == nil {
|
||||
_ = cache.Set(key, id)
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
for _, name := range names {
|
||||
id, err := getIGDBIDBySteamSearch(name)
|
||||
if err == nil {
|
||||
_ = cache.Set(key, id)
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("IGDB ID not found")
|
||||
}
|
||||
|
||||
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
|
||||
key := fmt.Sprintf("igdb_game:%v", id)
|
||||
func GetIGDBIDCache(name string) (int, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("igdb_id:%s", name)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
var data model.IGDBGameDetail
|
||||
if err := json.Unmarshal([]byte(val), &data); err != nil {
|
||||
id, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
} else {
|
||||
id, err := GetIGDBID(name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_ = cache.Add(key, id)
|
||||
return id, nil
|
||||
}
|
||||
} else {
|
||||
return GetIGDBID(name)
|
||||
}
|
||||
}
|
||||
|
||||
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
|
||||
var err error
|
||||
if TwitchToken == "" {
|
||||
TwitchToken, err = LoginTwitch()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &data, nil
|
||||
}
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id = %v;fields *,alternative_names.*,language_supports.*,screenshots.*,cover.*,involved_companies.*,involved_companies.*,game_engines.*,game_modes.*,genres.*,player_perspectives.*,release_dates.*,videos.*,websites.*,platforms.*,themes.*,collections.*;`, id))
|
||||
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBGameURL,
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
Data: fmt.Sprintf(`where id=%v ;fields *,alternative_names.name,language_supports.language,language_supports.language_support_type,screenshots.url,cover.url,involved_companies.company,involved_companies.developer,involved_companies.publisher;`, id),
|
||||
Method: "POST",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var data model.IGDBGameDetails
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
@ -293,45 +153,120 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
|
||||
if data[0].Name == "" {
|
||||
return GetIGDBAppDetail(id)
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(data[0])
|
||||
if err == nil {
|
||||
_ = cache.Set(key, string(jsonBytes))
|
||||
}
|
||||
|
||||
return data[0], nil
|
||||
}
|
||||
|
||||
func GetIGDBCompany(id int) (string, error) {
|
||||
key := fmt.Sprintf("igdb_companies:%v", id)
|
||||
func GetIGDBAppDetailCache(id int) (*model.IGDBGameDetail, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("igdb_game:%v", id)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
return val, nil
|
||||
var data model.IGDBGameDetail
|
||||
if err := json.Unmarshal([]byte(val), &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &data, nil
|
||||
} else {
|
||||
data, err := GetIGDBAppDetail(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataBytes, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = cache.Add(key, dataBytes)
|
||||
return data, nil
|
||||
}
|
||||
} else {
|
||||
return GetIGDBAppDetail(id)
|
||||
}
|
||||
}
|
||||
|
||||
func LoginTwitch() (string, error) {
|
||||
baseURL, _ := url.Parse(constant.TwitchAuthURL)
|
||||
params := url.Values{}
|
||||
params.Add("client_id", config.Config.Twitch.ClientID)
|
||||
params.Add("client_secret", config.Config.Twitch.ClientSecret)
|
||||
params.Add("grant_type", "client_credentials")
|
||||
baseURL.RawQuery = params.Encode()
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: baseURL.String(),
|
||||
Method: "POST",
|
||||
Headers: map[string]string{
|
||||
"User-Agent": "",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data := struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
}{}
|
||||
err = json.Unmarshal(resp.Data, &data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return data.AccessToken, nil
|
||||
}
|
||||
|
||||
func GetIGDBCompany(id int) (string, error) {
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBCompaniesURL, fmt.Sprintf(`where id=%v; fields *;`, id))
|
||||
if TwitchToken == "" {
|
||||
TwitchToken, err = LoginTwitch()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBCompaniesURL,
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
Data: fmt.Sprintf(`where id=%v; fields *;`, id),
|
||||
Method: "POST",
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var data model.IGDBCompanies
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return "", errors.New("not found")
|
||||
return "", errors.New("Not found")
|
||||
}
|
||||
if data[0].Name == "" {
|
||||
return GetIGDBCompany(id)
|
||||
}
|
||||
|
||||
_ = cache.Set(key, data[0].Name)
|
||||
|
||||
return data[0].Name, nil
|
||||
}
|
||||
|
||||
func GetIGDBCompanyCache(id int) (string, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("igdb_companies:%v", id)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
return val, nil
|
||||
} else {
|
||||
data, err := GetIGDBCompany(id)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
_ = cache.Add(key, data)
|
||||
return data, nil
|
||||
}
|
||||
} else {
|
||||
return GetIGDBCompany(id)
|
||||
}
|
||||
}
|
||||
|
||||
func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
|
||||
item := &model.GameInfo{}
|
||||
detail, err := GetIGDBAppDetail(id)
|
||||
detail, err := GetIGDBAppDetailCache(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -360,7 +295,7 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
|
||||
|
||||
for _, company := range detail.InvolvedCompanies {
|
||||
if company.Developer || company.Publisher {
|
||||
companyName, err := GetIGDBCompany(company.Company)
|
||||
companyName, err := GetIGDBCompanyCache(company.Company)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@ -373,66 +308,18 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
|
||||
}
|
||||
}
|
||||
|
||||
item.GameEngines = make([]string, 0)
|
||||
for _, engine := range detail.GameEngines {
|
||||
item.GameEngines = append(item.GameEngines, engine.Name)
|
||||
}
|
||||
|
||||
item.GameModes = make([]string, 0)
|
||||
for _, mode := range detail.GameModes {
|
||||
item.GameModes = append(item.GameModes, mode.Name)
|
||||
}
|
||||
|
||||
item.Genres = make([]string, 0)
|
||||
for _, genre := range detail.Genres {
|
||||
item.Genres = append(item.Genres, genre.Name)
|
||||
}
|
||||
|
||||
item.Themes = make([]string, 0)
|
||||
for _, theme := range detail.Themes {
|
||||
item.Themes = append(item.Themes, theme.Name)
|
||||
}
|
||||
|
||||
item.Platforms = make([]string, 0)
|
||||
for _, platform := range detail.Platforms {
|
||||
item.Platforms = append(item.Platforms, platform.Name)
|
||||
}
|
||||
|
||||
item.PlayerPerspectives = make([]string, 0)
|
||||
for _, perspective := range detail.PlayerPerspectives {
|
||||
item.PlayerPerspectives = append(item.PlayerPerspectives, perspective.Name)
|
||||
}
|
||||
|
||||
item.SimilarGames = detail.SimilarGames
|
||||
|
||||
item.Videos = make([]string, 0)
|
||||
for _, video := range detail.Videos {
|
||||
item.Videos = append(item.Videos, fmt.Sprintf("https://www.youtube.com/watch?v=%s", video.VideoID))
|
||||
}
|
||||
|
||||
item.Websites = make([]string, 0)
|
||||
for _, website := range detail.Websites {
|
||||
item.Websites = append(item.Websites, website.URL)
|
||||
}
|
||||
|
||||
item.Collections = make([]model.GameCollection, 0)
|
||||
|
||||
for _, collection := range detail.Collections {
|
||||
item.Collections = append(item.Collections, model.GameCollection{
|
||||
Games: collection.Games,
|
||||
Name: collection.Name,
|
||||
})
|
||||
}
|
||||
|
||||
return item, nil
|
||||
}
|
||||
|
||||
// OrganizeGameItemWithIGDB Will add GameItem.ID to the newly added GameInfo.GameIDs
|
||||
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
|
||||
id, err := GetIGDBID(game.Name)
|
||||
// id=0, means search id by name
|
||||
func OrganizeGameItemWithIGDB(id int, game *model.GameItem) (*model.GameInfo, error) {
|
||||
var err error
|
||||
if id == 0 {
|
||||
id, err = GetIGDBIDCache(game.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
d, err := db.GetGameInfoByPlatformID("igdb", id)
|
||||
if err == nil {
|
||||
d.GameIDs = append(d.GameIDs, game.ID)
|
||||
@ -448,92 +335,147 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func GetIGDBIDBySteamAppID(id int) (int, error) {
|
||||
key := fmt.Sprintf("igdb_id_by_steam_app_id:%v", id)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
return strconv.Atoi(val)
|
||||
}
|
||||
func GetIGDBIDBySteamID(id int) (int, error) {
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id))
|
||||
if TwitchToken == "" {
|
||||
TwitchToken, err = LoginTwitch()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBWebsitesURL,
|
||||
Method: "POST",
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
Data: fmt.Sprintf(`where url = "https://store.steampowered.com/app/%v" | url = "https://store.steampowered.com/app/%v/"*; fields *; limit 500;`, id, id),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var data []struct {
|
||||
Game int `json:"game"`
|
||||
}
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return 0, errors.New("not found")
|
||||
return 0, errors.New("Not found")
|
||||
}
|
||||
if data[0].Game == 0 {
|
||||
return GetIGDBIDBySteamAppID(id)
|
||||
return GetIGDBIDBySteamID(id)
|
||||
}
|
||||
|
||||
_ = cache.Set(key, strconv.Itoa(data[0].Game))
|
||||
|
||||
return GetIGDBAppParent(data[0].Game)
|
||||
return data[0].Game, nil
|
||||
}
|
||||
|
||||
func GetIGDBIDBySteamBundleID(id int) (int, error) {
|
||||
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%v", id)
|
||||
func GetIGDBIDBySteamIDCache(id int) (int, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("igdb_id_by_steam_id:%v", id)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
return strconv.Atoi(val)
|
||||
}
|
||||
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%v" | url = "https://store.steampowered.com/bundle/%v/"*; fields *; limit 500;`, id, id))
|
||||
} else {
|
||||
data, err := GetIGDBIDBySteamID(id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_ = cache.Add(key, strconv.Itoa(data))
|
||||
return data, nil
|
||||
}
|
||||
} else {
|
||||
return GetIGDBIDBySteamID(id)
|
||||
}
|
||||
}
|
||||
|
||||
func GetIGDBIDsBySteamIDs(ids []int) (map[int]int, error) {
|
||||
var err error
|
||||
if TwitchToken == "" {
|
||||
TwitchToken, err = LoginTwitch()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
conditionBuilder := strings.Builder{}
|
||||
for _, id := range ids {
|
||||
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v" | `, id))
|
||||
conditionBuilder.WriteString(fmt.Sprintf(`url = "https://store.steampowered.com/app/%v/"* | `, id))
|
||||
}
|
||||
condition := strings.TrimSuffix(conditionBuilder.String(), " | ")
|
||||
respBody := fmt.Sprintf(`where %s; fields *; limit 500;`, condition)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBWebsitesURL,
|
||||
Method: "POST",
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
Data: respBody,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var data []struct {
|
||||
Game int `json:"game"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return 0, errors.New("not found")
|
||||
}
|
||||
if data[0].Game == 0 {
|
||||
return GetIGDBIDBySteamBundleID(id)
|
||||
}
|
||||
|
||||
_ = cache.Set(key, strconv.Itoa(data[0].Game))
|
||||
|
||||
return GetIGDBAppParent(data[0].Game)
|
||||
}
|
||||
|
||||
// GetIGDBPopularGameIDs get IGDB popular game IDs
|
||||
// popularity_type = 1 IGDB Visits: Game page visits on IGDB.com.
|
||||
// popularity_type = 2 IGDB Want to Play: Additions to IGDB.com users’ “Want to Play” lists.
|
||||
// popularity_type = 3 IGDB Playing: Additions to IGDB.com users’ “Playing” lists.
|
||||
// popularity_type = 4 IGDB Played: Additions to IGDB.com users’ “Played” lists.
|
||||
func GetIGDBPopularGameIDs(popularityType int, offset int, limit int) ([]int, error) {
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBPopularityURL, fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %v; offset %v; where popularity_type = %v;", limit, offset, popularityType))
|
||||
if err != nil {
|
||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type IgdbPopularity struct {
|
||||
GameID int `json:"game_id"`
|
||||
Value float64 `json:"value"`
|
||||
}
|
||||
var data []IgdbPopularity
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ret := make([]int, 0)
|
||||
ret := make(map[int]int)
|
||||
regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
|
||||
for _, d := range data {
|
||||
pid, err := GetIGDBAppParent(d.GameID)
|
||||
if err != nil {
|
||||
ret = append(ret, d.GameID)
|
||||
idStr := regex.FindStringSubmatch(d.Url)
|
||||
if len(idStr) < 2 {
|
||||
continue
|
||||
}
|
||||
ret = append(ret, pid)
|
||||
id, err := strconv.Atoi(idStr[1])
|
||||
if err == nil {
|
||||
ret[id] = d.Game
|
||||
}
|
||||
}
|
||||
for _, id := range ids {
|
||||
if _, ok := ret[id]; !ok {
|
||||
ret[id] = 0
|
||||
}
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func GetIGDBIDsBySteamIDsCache(ids []int) (map[int]int, error) {
|
||||
res := make(map[int]int)
|
||||
notExistIDs := make([]int, 0)
|
||||
if config.Config.RedisAvaliable {
|
||||
for _, steamID := range ids {
|
||||
key := fmt.Sprintf("igdb_id_by_steam_id:%v", steamID)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
igdbID, _ := strconv.Atoi(val)
|
||||
res[steamID] = igdbID
|
||||
} else {
|
||||
notExistIDs = append(notExistIDs, steamID)
|
||||
}
|
||||
}
|
||||
if len(res) == len(ids) {
|
||||
return res, nil
|
||||
}
|
||||
idMap, err := GetIGDBIDsBySteamIDs(notExistIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for steamID, igdbID := range idMap {
|
||||
res[steamID] = igdbID
|
||||
if igdbID != 0 {
|
||||
_ = cache.Add(fmt.Sprintf("igdb_id_by_steam_id:%v", steamID), igdbID)
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
} else {
|
||||
return GetIGDBIDsBySteamIDs(ids)
|
||||
}
|
||||
}
|
||||
|
@ -1,66 +0,0 @@
|
||||
package crawler
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"pcgamedb/model"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const Johncena141Name string = "johncena141-torrents"
|
||||
|
||||
type Johncena141Crawler struct {
|
||||
logger *zap.Logger
|
||||
crawler s1337xCrawler
|
||||
}
|
||||
|
||||
func NewJohncena141Crawler(logger *zap.Logger) *Johncena141Crawler {
|
||||
return &Johncena141Crawler{
|
||||
logger: logger,
|
||||
crawler: *New1337xCrawler(
|
||||
Johncena141Name,
|
||||
"linux",
|
||||
Johncena141Formatter,
|
||||
logger,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Johncena141Crawler) Name() string {
|
||||
return "Johncena141Crawler"
|
||||
}
|
||||
|
||||
func (c *Johncena141Crawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
return c.crawler.Crawl(page)
|
||||
}
|
||||
|
||||
func (c *Johncena141Crawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
return c.crawler.CrawlByUrl(url)
|
||||
}
|
||||
|
||||
func (c *Johncena141Crawler) CrawlMulti(pages []int) ([]*model.GameItem, error) {
|
||||
return c.crawler.CrawlMulti(pages)
|
||||
}
|
||||
|
||||
func (c *Johncena141Crawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
return c.crawler.CrawlAll()
|
||||
}
|
||||
|
||||
func (c *Johncena141Crawler) GetTotalPageNum() (int, error) {
|
||||
return c.crawler.GetTotalPageNum()
|
||||
}
|
||||
|
||||
var Johncena141Regexps = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\s{2,}`),
|
||||
regexp.MustCompile(`(?i)[\-\+]\s?[^:\-]*?\s(Edition|Bundle|Pack|Set|Remake|Collection)`),
|
||||
}
|
||||
|
||||
func Johncena141Formatter(name string) string {
|
||||
nameslice := strings.Split(name, " - ")
|
||||
name = nameslice[0]
|
||||
reg1 := regexp.MustCompile(`(?i)\(.*?\)`)
|
||||
name = reg1.ReplaceAllString(name, "")
|
||||
return strings.TrimSpace(name)
|
||||
}
|
@ -21,7 +21,6 @@ func NewKaOsKrewCrawler(logger *zap.Logger) *KaOsKrewCrawler {
|
||||
logger: logger,
|
||||
crawler: *New1337xCrawler(
|
||||
KaOsKrewName,
|
||||
"windows",
|
||||
KaOsKrewFormatter,
|
||||
logger,
|
||||
),
|
||||
|
@ -5,15 +5,12 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/config"
|
||||
"pcgamedb/constant"
|
||||
"pcgamedb/db"
|
||||
@ -26,11 +23,13 @@ import (
|
||||
|
||||
type OnlineFixCrawler struct {
|
||||
logger *zap.Logger
|
||||
cookies map[string]string
|
||||
}
|
||||
|
||||
func NewOnlineFixCrawler(logger *zap.Logger) *OnlineFixCrawler {
|
||||
return &OnlineFixCrawler{
|
||||
logger: logger,
|
||||
cookies: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
@ -39,24 +38,36 @@ func (c *OnlineFixCrawler) Name() string {
|
||||
}
|
||||
|
||||
func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
cookies, err := c.getCookies()
|
||||
if !config.Config.OnlineFixAvaliable {
|
||||
c.logger.Error("Need Online Fix account")
|
||||
return nil, errors.New("Online Fix is not available")
|
||||
}
|
||||
if len(c.cookies) == 0 {
|
||||
err := c.login()
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to login", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
requestURL := fmt.Sprintf("%s/page/%d/", constant.OnlineFixURL, page)
|
||||
resp, err := utils.Request().SetHeader("Referer", constant.OnlineFixURL).SetCookies(cookies).Get(requestURL)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: requestURL,
|
||||
Cookies: c.cookies,
|
||||
Headers: map[string]string{
|
||||
"Referer": constant.OnlineFixURL,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to fetch", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
body := utils.Windows1251ToUTF8(resp.Body())
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to parse HTML", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
var urls []string
|
||||
var updateFlags []string //link+date
|
||||
urls := []string{}
|
||||
updateFlags := []string{} //link+date
|
||||
doc.Find("article.news").Each(func(i int, s *goquery.Selection) {
|
||||
urls = append(urls, s.Find(".big-link").First().AttrOr("href", ""))
|
||||
updateFlags = append(
|
||||
@ -84,73 +95,96 @@ func (c *OnlineFixCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
cookies, err := c.getCookies()
|
||||
func (c *OnlineFixCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
if len(c.cookies) == 0 {
|
||||
err := c.login()
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to login", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
resp, err := utils.Request().SetHeaders(map[string]string{
|
||||
}
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
Cookies: c.cookies,
|
||||
Headers: map[string]string{
|
||||
"Referer": constant.OnlineFixURL,
|
||||
}).SetCookies(cookies).Get(URL)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := utils.Windows1251ToUTF8(resp.Body())
|
||||
titleRegex := regexp.MustCompile(`(?i)<h1.*?>(.*?)</h1>`)
|
||||
titleRegexRes := titleRegex.FindAllStringSubmatch(string(body), -1)
|
||||
titleRegexRes := titleRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
if len(titleRegexRes) == 0 {
|
||||
return nil, errors.New("failed to find title")
|
||||
return nil, errors.New("Failed to find title")
|
||||
}
|
||||
downloadRegex := regexp.MustCompile(`(?i)<a[^>]+\bhref="([^"]+)"[^>]+>(Скачать Torrent|Скачать торрент)</a>`)
|
||||
downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(body), -1)
|
||||
downloadRegex := regexp.MustCompile(`(?i)<a[^>]*\bhref="([^"]+)"[^>]*>(Скачать Torrent|Скачать торрент)</a>`)
|
||||
downloadRegexRes := downloadRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
if len(downloadRegexRes) == 0 {
|
||||
return nil, errors.New("failed to find download button")
|
||||
return nil, errors.New("Failed to find download button")
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(URL)
|
||||
item, err := db.GetGameItemByUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.RawName = titleRegexRes[0][1]
|
||||
item.Name = OnlineFixFormatter(item.RawName)
|
||||
item.Url = URL
|
||||
item.Url = url
|
||||
item.Author = "OnlineFix"
|
||||
item.Size = "0"
|
||||
resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1])
|
||||
body = utils.Windows1251ToUTF8(resp.Body())
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: downloadRegexRes[0][1],
|
||||
Cookies: c.cookies,
|
||||
Headers: map[string]string{
|
||||
"Referer": url,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.Contains(downloadRegexRes[0][1], "uploads.online-fix.me") {
|
||||
magnetRegex := regexp.MustCompile(`(?i)"(.*?).torrent"`)
|
||||
magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(body), -1)
|
||||
magnetRegexRes := magnetRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
if len(magnetRegexRes) == 0 {
|
||||
return nil, errors.New("failed to find magnet")
|
||||
return nil, errors.New("Failed to find magnet")
|
||||
}
|
||||
resp, err = utils.Request().SetHeader("Referer", URL).SetCookies(cookies).Get(downloadRegexRes[0][1] + strings.Trim(magnetRegexRes[0][0], "\""))
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: downloadRegexRes[0][1] + strings.Trim(magnetRegexRes[0][0], "\""),
|
||||
Cookies: c.cookies,
|
||||
Headers: map[string]string{
|
||||
"Referer": url,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(resp.Body())
|
||||
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(resp.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if strings.Contains(downloadRegexRes[0][1], "online-fix.me/ext") {
|
||||
if strings.Contains(string(body), "mega.nz") {
|
||||
if strings.Contains(string(resp.Data), "mega.nz") {
|
||||
if !config.Config.MegaAvaliable {
|
||||
return nil, errors.New("mega is not avaliable")
|
||||
return nil, errors.New("Mega is not avaliable")
|
||||
}
|
||||
megaRegex := regexp.MustCompile(`(?i)location.href=\\'([^\\']*)\\'`)
|
||||
megaRegexRes := megaRegex.FindAllStringSubmatch(string(body), -1)
|
||||
megaRegexRes := megaRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
if len(megaRegexRes) == 0 {
|
||||
return nil, errors.New("failed to find download link")
|
||||
return nil, errors.New("Failed to find download link")
|
||||
}
|
||||
path, files, err := utils.MegaDownload(megaRegexRes[0][1], "torrent")
|
||||
if err != nil {
|
||||
@ -173,12 +207,11 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
}
|
||||
_ = os.RemoveAll(path)
|
||||
} else {
|
||||
return nil, errors.New("failed to find download link")
|
||||
return nil, errors.New("Failed to find download link")
|
||||
}
|
||||
} else {
|
||||
return nil, errors.New("failed to find download link")
|
||||
return nil, errors.New("Failed to find download link")
|
||||
}
|
||||
item.Platform = "windows"
|
||||
return item, nil
|
||||
}
|
||||
|
||||
@ -211,12 +244,17 @@ func (c *OnlineFixCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func (c *OnlineFixCrawler) GetTotalPageNum() (int, error) {
|
||||
resp, err := utils.Request().SetHeader("Referer", constant.OnlineFixURL).Get(constant.OnlineFixURL)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.OnlineFixURL,
|
||||
Headers: map[string]string{
|
||||
"Referer": constant.OnlineFixURL,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
pageRegex := regexp.MustCompile(`(?i)<a href="https://online-fix.me/page/(\d+)/">.*?</a>`)
|
||||
pageRegexRes := pageRegex.FindAllStringSubmatch(string(resp.Body()), -1)
|
||||
pageRegexRes := pageRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
if len(pageRegexRes) == 0 {
|
||||
return 0, err
|
||||
}
|
||||
@ -232,48 +270,48 @@ type csrf struct {
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
func (c *OnlineFixCrawler) getCookies() ([]*http.Cookie, error) {
|
||||
val, exists := cache.Get("onlinefix_cookies")
|
||||
if exists {
|
||||
var cookies []*http.Cookie
|
||||
if err := json.Unmarshal([]byte(val), &cookies); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cookies, nil
|
||||
}
|
||||
|
||||
resp, err := utils.Request().SetHeaders(map[string]string{
|
||||
func (c *OnlineFixCrawler) login() error {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.OnlineFixCSRFURL,
|
||||
Headers: map[string]string{
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Referer": constant.OnlineFixURL,
|
||||
}).Get(constant.OnlineFixCSRFURL)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
var csrf csrf
|
||||
if err = json.Unmarshal(resp.Body(), &csrf); err != nil {
|
||||
return nil, err
|
||||
if err = json.Unmarshal(resp.Data, &csrf); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cookies := resp.Cookies()
|
||||
|
||||
for _, cookie := range resp.Cookie {
|
||||
c.cookies[cookie.Name] = cookie.Value
|
||||
}
|
||||
params := url.Values{}
|
||||
params.Add("login_name", config.Config.OnlineFix.User)
|
||||
params.Add("login_password", config.Config.OnlineFix.Password)
|
||||
params.Add(csrf.Field, csrf.Value)
|
||||
params.Add("login", "submit")
|
||||
resp, err = utils.Request().SetHeaders(map[string]string{
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.OnlineFixURL,
|
||||
Method: "POST",
|
||||
Cookies: c.cookies,
|
||||
Headers: map[string]string{
|
||||
"Origin": constant.OnlineFixURL,
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
"Referer": constant.OnlineFixURL,
|
||||
}).SetCookies(cookies).SetBody(params.Encode()).Post(constant.OnlineFixURL)
|
||||
},
|
||||
Data: params,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return err
|
||||
}
|
||||
cookies = resp.Cookies()
|
||||
jsonBytes, _ := json.Marshal(cookies)
|
||||
_ = cache.SetWithExpire("onlinefix_cookies", string(jsonBytes), time.Hour)
|
||||
|
||||
return cookies, nil
|
||||
for _, cookie := range resp.Cookie {
|
||||
c.cookies[cookie.Name] = cookie.Value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func OnlineFixFormatter(name string) string {
|
||||
|
258
crawler/steam.go
258
crawler/steam.go
@ -10,12 +10,127 @@ import (
|
||||
"strings"
|
||||
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/config"
|
||||
"pcgamedb/constant"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
)
|
||||
|
||||
func _GetSteamID(name string) (int, error) {
|
||||
baseURL, _ := url.Parse(constant.SteamSearchURL)
|
||||
params := url.Values{}
|
||||
params.Add("term", name)
|
||||
baseURL.RawQuery = params.Encode()
|
||||
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: baseURL.String(),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
idRegex := regexp.MustCompile(`data-ds-appid="(.*?)"`)
|
||||
nameRegex := regexp.MustCompile(`<span class="title">(.*?)</span>`)
|
||||
idRegexRes := idRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
nameRegexRes := nameRegex.FindAllStringSubmatch(string(resp.Data), -1)
|
||||
|
||||
if len(idRegexRes) == 0 {
|
||||
return 0, fmt.Errorf("Steam ID not found: %s", name)
|
||||
}
|
||||
|
||||
maxSim := 0.0
|
||||
maxSimID := 0
|
||||
for i, id := range idRegexRes {
|
||||
idStr := id[1]
|
||||
nameStr := nameRegexRes[i][1]
|
||||
if index := strings.Index(idStr, ","); index != -1 {
|
||||
idStr = idStr[:index]
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(nameStr), strings.TrimSpace(name)) {
|
||||
return strconv.Atoi(idStr)
|
||||
} else {
|
||||
sim := utils.Similarity(nameStr, name)
|
||||
if sim >= 0.8 && sim > maxSim {
|
||||
maxSim = sim
|
||||
maxSimID, _ = strconv.Atoi(idStr)
|
||||
}
|
||||
}
|
||||
}
|
||||
if maxSimID != 0 {
|
||||
return maxSimID, nil
|
||||
}
|
||||
return 0, fmt.Errorf("Steam ID not found: %s", name)
|
||||
}
|
||||
|
||||
func GetSteamID(name string) (int, error) {
|
||||
name1 := name
|
||||
name2 := FormatName(name)
|
||||
names := []string{name1}
|
||||
if name1 != name2 {
|
||||
names = append(names, name2)
|
||||
}
|
||||
for _, n := range names {
|
||||
id, err := _GetSteamID(n)
|
||||
if err == nil {
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("Steam ID not found")
|
||||
}
|
||||
|
||||
func GetSteamIDCache(name string) (int, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("steam_id:%s", name)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
id, err := strconv.Atoi(val)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
} else {
|
||||
id, err := GetSteamID(name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_ = cache.Add(key, id)
|
||||
return id, nil
|
||||
}
|
||||
} else {
|
||||
return GetSteamID(name)
|
||||
}
|
||||
}
|
||||
|
||||
func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
|
||||
baseURL, _ := url.Parse(constant.SteamAppDetailURL)
|
||||
params := url.Values{}
|
||||
params.Add("appids", strconv.Itoa(id))
|
||||
// params.Add("l", "schinese")
|
||||
baseURL.RawQuery = params.Encode()
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: baseURL.String(),
|
||||
Headers: map[string]string{
|
||||
"User-Agent": "",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var detail map[string]*model.SteamAppDetail
|
||||
if err = json.Unmarshal(resp.Data, &detail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := detail[strconv.Itoa(id)]; !ok {
|
||||
return nil, fmt.Errorf("Steam App not found: %d", id)
|
||||
}
|
||||
if detail[strconv.Itoa(id)] == nil {
|
||||
return nil, fmt.Errorf("Steam App not found: %d", id)
|
||||
}
|
||||
return detail[strconv.Itoa(id)], nil
|
||||
}
|
||||
|
||||
func GetSteamAppDetailCache(id int) (*model.SteamAppDetail, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("steam_game:%d", id)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
@ -24,41 +139,26 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
|
||||
return nil, err
|
||||
}
|
||||
return &detail, nil
|
||||
}
|
||||
|
||||
baseURL, _ := url.Parse(constant.SteamAppDetailURL)
|
||||
params := url.Values{}
|
||||
params.Add("appids", strconv.Itoa(id))
|
||||
// params.Add("l", "schinese")
|
||||
baseURL.RawQuery = params.Encode()
|
||||
resp, err := utils.Request().SetHeaders(map[string]string{
|
||||
"User-Agent": "",
|
||||
}).Get(baseURL.String())
|
||||
} else {
|
||||
data, err := GetSteamAppDetail(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var detail map[string]*model.SteamAppDetail
|
||||
if err = json.Unmarshal(resp.Body(), &detail); err != nil {
|
||||
dataBytes, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, ok := detail[strconv.Itoa(id)]; !ok {
|
||||
return nil, fmt.Errorf("steam App not found: %d", id)
|
||||
_ = cache.Add(key, dataBytes)
|
||||
return data, nil
|
||||
}
|
||||
if detail[strconv.Itoa(id)] == nil {
|
||||
return nil, fmt.Errorf("steam App not found: %d", id)
|
||||
} else {
|
||||
return GetSteamAppDetail(id)
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(detail[strconv.Itoa(id)])
|
||||
if err == nil {
|
||||
_ = cache.Set(key, string(jsonBytes))
|
||||
}
|
||||
|
||||
return detail[strconv.Itoa(id)], nil
|
||||
}
|
||||
|
||||
func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
|
||||
item := &model.GameInfo{}
|
||||
detail, err := GetSteamAppDetail(id)
|
||||
detail, err := GetSteamAppDetailCache(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -68,7 +168,7 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
|
||||
item.Cover = fmt.Sprintf("https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/%v/library_600x900_2x.jpg", id)
|
||||
item.Developers = detail.Data.Developers
|
||||
item.Publishers = detail.Data.Publishers
|
||||
var screenshots []string
|
||||
screenshots := []string{}
|
||||
for _, screenshot := range detail.Data.Screenshots {
|
||||
screenshots = append(screenshots, screenshot.PathFull)
|
||||
}
|
||||
@ -76,7 +176,80 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func OrganizeGameItemWithSteam(id int, game *model.GameItem) (*model.GameInfo, error) {
|
||||
var err error
|
||||
if id == 0 {
|
||||
id, err = GetSteamIDCache(game.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
d, err := db.GetGameInfoByPlatformID("steam", id)
|
||||
if err == nil {
|
||||
d.GameIDs = append(d.GameIDs, game.ID)
|
||||
d.GameIDs = utils.Unique(d.GameIDs)
|
||||
return d, nil
|
||||
}
|
||||
detail, err := GenerateGameInfo("steam", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
detail.GameIDs = append(detail.GameIDs, game.ID)
|
||||
detail.GameIDs = utils.Unique(detail.GameIDs)
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
|
||||
var err error
|
||||
if TwitchToken == "" {
|
||||
TwitchToken, err = LoginTwitch()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.IGDBWebsitesURL,
|
||||
Method: "POST",
|
||||
Headers: map[string]string{
|
||||
"Client-ID": config.Config.Twitch.ClientID,
|
||||
"Authorization": "Bearer " + TwitchToken,
|
||||
"User-Agent": "",
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
Data: fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID),
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var data []struct {
|
||||
Game int `json:"game"`
|
||||
Url string `json:"url"`
|
||||
}
|
||||
if err = json.Unmarshal(resp.Data, &data); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return 0, errors.New("Not found")
|
||||
}
|
||||
for _, v := range data {
|
||||
if strings.HasPrefix(v.Url, "https://store.steampowered.com/app/") {
|
||||
regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
|
||||
idStr := regex.FindStringSubmatch(v.Url)
|
||||
if len(idStr) < 2 {
|
||||
return 0, errors.New("Failed parse")
|
||||
}
|
||||
steamID, err := strconv.Atoi(idStr[1])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return steamID, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("Not found")
|
||||
}
|
||||
|
||||
func GetSteamIDByIGDBIDCache(IGDBID int) (int, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("steam_game:%d", IGDBID)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
@ -85,37 +258,16 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
var err error
|
||||
resp, err := igdbRequest(constant.IGDBWebsitesURL, fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID))
|
||||
} else {
|
||||
id, err := GetSteamIDByIGDBID(IGDBID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var data []struct {
|
||||
Game int `json:"game"`
|
||||
Url string `json:"url"`
|
||||
dataBytes := strconv.Itoa(id)
|
||||
_ = cache.Add(key, dataBytes)
|
||||
return id, nil
|
||||
}
|
||||
if err = json.Unmarshal(resp.Body(), &data); err != nil {
|
||||
return 0, err
|
||||
} else {
|
||||
return GetSteamIDByIGDBID(IGDBID)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
return 0, errors.New("not found")
|
||||
}
|
||||
for _, v := range data {
|
||||
if strings.HasPrefix(v.Url, "https://store.steampowered.com/app/") {
|
||||
regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
|
||||
idStr := regex.FindStringSubmatch(v.Url)
|
||||
if len(idStr) < 2 {
|
||||
return 0, errors.New("failed parse")
|
||||
}
|
||||
steamID, err := strconv.Atoi(idStr[1])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
_ = cache.Set(key, strconv.Itoa(steamID))
|
||||
return steamID, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("not found")
|
||||
}
|
||||
|
@ -4,36 +4,28 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/db"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/config"
|
||||
"pcgamedb/constant"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
func GetSteam250(URL string) ([]*model.GameInfo, error) {
|
||||
key := "steam250:" + url.QueryEscape(URL)
|
||||
if val, ok := cache.Get(key); ok {
|
||||
var infos []*model.GameInfo
|
||||
err := json.Unmarshal([]byte(val), &infos)
|
||||
func GetSteam250(url string) ([]*model.GameInfo, error) {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
resp, err := utils.Request().Get(URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -51,38 +43,81 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
|
||||
rank = append(rank, item)
|
||||
steamIDs = append(steamIDs, item.SteamID)
|
||||
})
|
||||
infos, err := db.GetGameInfosByPlatformIDs("steam", steamIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var res []*model.GameInfo
|
||||
count := 0
|
||||
for _, steamID := range steamIDs {
|
||||
if count >= 10 {
|
||||
break
|
||||
}
|
||||
if len(infos) > 10 {
|
||||
return infos[:10], nil
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(infos)
|
||||
info, err := db.GetGameInfoByPlatformID("steam", steamID)
|
||||
if err == nil {
|
||||
_ = cache.SetWithExpire(key, string(jsonBytes), 12*time.Hour)
|
||||
res = append(res, info)
|
||||
count++
|
||||
continue
|
||||
}
|
||||
|
||||
return infos, nil
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func GetSteam250Top250() ([]*model.GameInfo, error) {
|
||||
return GetSteam250(constant.Steam250Top250URL)
|
||||
}
|
||||
|
||||
func GetSteam250Top250Cache() ([]*model.GameInfo, error) {
|
||||
return GetSteam250Cache("top250", GetSteam250Top250)
|
||||
}
|
||||
|
||||
func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) {
|
||||
return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, time.Now().UTC().Year()))
|
||||
}
|
||||
|
||||
func GetSteam250BestOfTheYearCache() ([]*model.GameInfo, error) {
|
||||
return GetSteam250Cache(fmt.Sprintf("bestoftheyear:%v", time.Now().UTC().Year()), GetSteam250BestOfTheYear)
|
||||
}
|
||||
|
||||
func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
|
||||
return GetSteam250(constant.Steam250WeekTop50URL)
|
||||
}
|
||||
|
||||
func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
|
||||
return GetSteam250(constant.Steam250MonthTop50URL)
|
||||
func GetSteam250WeekTop50Cache() ([]*model.GameInfo, error) {
|
||||
return GetSteam250Cache("weektop50", GetSteam250WeekTop50)
|
||||
}
|
||||
|
||||
func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
|
||||
return GetSteam250(constant.Steam250MostPlayedURL)
|
||||
}
|
||||
|
||||
func GetSteam250MostPlayedCache() ([]*model.GameInfo, error) {
|
||||
return GetSteam250Cache("mostplayed", GetSteam250MostPlayed)
|
||||
}
|
||||
|
||||
func GetSteam250Cache(k string, f func() ([]*model.GameInfo, error)) ([]*model.GameInfo, error) {
|
||||
if config.Config.RedisAvaliable {
|
||||
key := k
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
var res []*model.GameInfo
|
||||
err := json.Unmarshal([]byte(val), &res)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
} else {
|
||||
data, err := f()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataBytes, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return data, nil
|
||||
}
|
||||
err = cache.AddWithExpire(key, dataBytes, 12*time.Hour)
|
||||
if err != nil {
|
||||
return data, nil
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
} else {
|
||||
return f()
|
||||
}
|
||||
}
|
||||
|
@ -30,52 +30,53 @@ func (c *SteamRIPCrawler) Name() string {
|
||||
return "SteamRIPCrawler"
|
||||
}
|
||||
|
||||
func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(URL)
|
||||
func (c *SteamRIPCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(URL)
|
||||
item, err := db.GetGameItemByUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.RawName = strings.TrimSpace(doc.Find(".entry-title").First().Text())
|
||||
item.Name = SteamRIPFormatter(item.RawName)
|
||||
item.Url = URL
|
||||
item.Url = url
|
||||
item.Author = "SteamRIP"
|
||||
item.Platform = "windows"
|
||||
sizeRegex := regexp.MustCompile(`(?i)<li><strong>Game Size:\s?</strong>(.*?)</li>`)
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(string(resp.Body()))
|
||||
sizeRegexRes := sizeRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(sizeRegexRes) != 0 {
|
||||
item.Size = strings.TrimSpace(sizeRegexRes[1])
|
||||
} else {
|
||||
item.Size = "unknown"
|
||||
}
|
||||
megadbRegex := regexp.MustCompile(`(?i)(?:https?:)?(//megadb\.net/[^"]+)`)
|
||||
megadbRegexRes := megadbRegex.FindStringSubmatch(string(resp.Body()))
|
||||
megadbRegexRes := megadbRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(megadbRegexRes) != 0 {
|
||||
item.Download = fmt.Sprintf("https:%s", megadbRegexRes[1])
|
||||
}
|
||||
if item.Download == "" {
|
||||
gofileRegex := regexp.MustCompile(`(?i)(?:https?:)?(//gofile\.io/d/[^"]+)`)
|
||||
gofileRegexRes := gofileRegex.FindStringSubmatch(string(resp.Body()))
|
||||
gofileRegexRes := gofileRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(gofileRegexRes) != 0 {
|
||||
item.Download = fmt.Sprintf("https:%s", gofileRegexRes[1])
|
||||
}
|
||||
}
|
||||
if item.Download == "" {
|
||||
filecryptRegex := regexp.MustCompile(`(?i)(?:https?:)?(//filecrypt\.co/Container/[^"]+)`)
|
||||
filecryptRegexRes := filecryptRegex.FindStringSubmatch(string(resp.Body()))
|
||||
filecryptRegexRes := filecryptRegex.FindStringSubmatch(string(resp.Data))
|
||||
if len(filecryptRegexRes) != 0 {
|
||||
item.Download = fmt.Sprintf("https:%s", filecryptRegexRes[1])
|
||||
}
|
||||
}
|
||||
if item.Download == "" {
|
||||
return nil, errors.New("failed to find download link")
|
||||
return nil, errors.New("Failed to find download link")
|
||||
}
|
||||
|
||||
return item, nil
|
||||
@ -83,17 +84,19 @@ func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
|
||||
func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
count := 0
|
||||
resp, err := utils.Request().Get(constant.SteamRIPGameListURL)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.SteamRIPGameListURL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var items []*model.GameItem
|
||||
var urls []string
|
||||
var updateFlags []string // title
|
||||
urls := []string{}
|
||||
updateFlags := []string{} // title
|
||||
doc.Find(".az-list-item>a").Each(func(i int, s *goquery.Selection) {
|
||||
u, exist := s.Attr("href")
|
||||
if !exist {
|
||||
@ -122,10 +125,16 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
|
||||
}
|
||||
items = append(items, item)
|
||||
count++
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
@ -33,18 +33,20 @@ func (c *XatabCrawler) Name() string {
|
||||
|
||||
func (c *XatabCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
requestURL := fmt.Sprintf("%s/page/%v", constant.XatabBaseURL, page)
|
||||
resp, err := utils.Request().Get(requestURL)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: requestURL,
|
||||
})
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to fetch", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to parse HTML", zap.Error(err))
|
||||
return nil, err
|
||||
}
|
||||
var urls []string
|
||||
var updateFlags []string // title
|
||||
urls := []string{}
|
||||
updateFlags := []string{} // title
|
||||
doc.Find(".entry").Each(func(i int, s *goquery.Selection) {
|
||||
u, exist := s.Find(".entry__title.h2 a").Attr("href")
|
||||
if !exist {
|
||||
@ -70,42 +72,52 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameItem, error) {
|
||||
continue
|
||||
}
|
||||
res = append(res, item)
|
||||
if err := OrganizeGameItem(item); err != nil {
|
||||
info, err := OrganizeGameItem(item)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to organize", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
err = db.SaveGameInfo(info)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
|
||||
resp, err := utils.Request().Get(URL)
|
||||
func (c *XatabCrawler) CrawlByUrl(url string) (*model.GameItem, error) {
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: url,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item, err := db.GetGameItemByUrl(URL)
|
||||
item, err := db.GetGameItemByUrl(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
item.Url = URL
|
||||
item.Url = url
|
||||
item.RawName = doc.Find(".inner-entry__title").First().Text()
|
||||
item.Name = XatabFormatter(item.RawName)
|
||||
item.Author = "Xatab"
|
||||
item.UpdateFlag = item.RawName
|
||||
item.Platform = "windows"
|
||||
downloadURL := doc.Find("#download>a").First().AttrOr("href", "")
|
||||
if downloadURL == "" {
|
||||
return nil, errors.New("failed to find download URL")
|
||||
return nil, errors.New("Failed to find download URL")
|
||||
}
|
||||
resp, err = utils.Request().SetHeaders(map[string]string{"Referer": URL}).Get(downloadURL)
|
||||
resp, err = utils.Fetch(utils.FetchConfig{
|
||||
Headers: map[string]string{"Referer": url},
|
||||
Url: downloadURL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
magnet, size, err := utils.ConvertTorrentToMagnet(resp.Body())
|
||||
magnet, size, err := utils.ConvertTorrentToMagnet(resp.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -150,11 +162,13 @@ func (c *XatabCrawler) CrawlAll() ([]*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func (c *XatabCrawler) GetTotalPageNum() (int, error) {
|
||||
resp, err := utils.Request().Get(constant.XatabBaseURL)
|
||||
resp, err := utils.Fetch(utils.FetchConfig{
|
||||
Url: constant.XatabBaseURL,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
@ -41,15 +41,6 @@ func (c *CustomCollection) UpdateOne(ctx context.Context, filter interface{}, up
|
||||
return c.coll.UpdateOne(ctx, filter, update, opts...)
|
||||
}
|
||||
|
||||
func (c *CustomCollection) UpdateMany(ctx context.Context, filter interface{}, update interface{},
|
||||
opts ...*options.UpdateOptions) (*mongo.UpdateResult, error) {
|
||||
CheckConnect()
|
||||
if c.coll == nil {
|
||||
c.coll = mongoDB.Database(config.Config.Database.Database).Collection(c.collName)
|
||||
}
|
||||
return c.coll.UpdateMany(ctx, filter, update, opts...)
|
||||
}
|
||||
|
||||
func (c *CustomCollection) Aggregate(ctx context.Context, pipeline interface{},
|
||||
opts ...*options.AggregateOptions) (*mongo.Cursor, error) {
|
||||
CheckConnect()
|
||||
@ -103,12 +94,3 @@ func (c *CustomCollection) InsertMany(ctx context.Context, documents []interface
|
||||
}
|
||||
return c.coll.InsertMany(ctx, documents, opts...)
|
||||
}
|
||||
|
||||
func (c *CustomCollection) BulkWrite(ctx context.Context, models []mongo.WriteModel,
|
||||
opts ...*options.BulkWriteOptions) (*mongo.BulkWriteResult, error) {
|
||||
CheckConnect()
|
||||
if c.coll == nil {
|
||||
c.coll = mongoDB.Database(config.Config.Database.Database).Collection(c.collName)
|
||||
}
|
||||
return c.coll.BulkWrite(ctx, models, opts...)
|
||||
}
|
||||
|
4
db/db.go
4
db/db.go
@ -32,6 +32,10 @@ var (
|
||||
)
|
||||
|
||||
func connect() {
|
||||
if !config.Config.DatabaseAvaliable {
|
||||
log.Logger.Panic("Missing database configuration information")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
clientOptions := options.Client().ApplyURI(fmt.Sprintf(
|
||||
|
@ -5,15 +5,15 @@ import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
"pcgamedb/model"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
func Export() ([]byte, []byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
var infos []model.GameInfo
|
||||
var games []model.GameItem
|
||||
infos := []model.GameInfo{}
|
||||
games := []model.GameItem{}
|
||||
cursor, err := GameInfoCollection.Find(ctx, bson.M{})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
|
353
db/game.go
353
db/game.go
@ -5,12 +5,13 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"pcgamedb/utils"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pcgamedb/cache"
|
||||
"pcgamedb/config"
|
||||
"pcgamedb/model"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
@ -33,9 +34,7 @@ func GetGameItemsByAuthor(regex string) ([]*model.GameItem, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func(cursor *mongo.Cursor, ctx context.Context) {
|
||||
_ = cursor.Close(ctx)
|
||||
}(cursor, ctx)
|
||||
defer cursor.Close(ctx)
|
||||
if cursor.Err() != nil {
|
||||
return nil, cursor.Err()
|
||||
}
|
||||
@ -62,9 +61,7 @@ func GetGameItemsByAuthorPagination(regex string, page int, pageSize int) ([]*mo
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer func(cursor *mongo.Cursor, ctx context.Context) {
|
||||
_ = cursor.Close(ctx)
|
||||
}(cursor, ctx)
|
||||
defer cursor.Close(ctx)
|
||||
if cursor.Err() != nil {
|
||||
return nil, 0, cursor.Err()
|
||||
}
|
||||
@ -131,34 +128,6 @@ func SaveGameItem(item *model.GameItem) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveGameItems(items []*model.GameItem) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
operations := make([]mongo.WriteModel, len(items))
|
||||
for i, item := range items {
|
||||
if item.ID.IsZero() {
|
||||
item.ID = primitive.NewObjectID()
|
||||
}
|
||||
if item.CreatedAt.IsZero() {
|
||||
item.CreatedAt = time.Now()
|
||||
}
|
||||
item.UpdatedAt = time.Now()
|
||||
item.Size = strings.Replace(item.Size, "gb", "GB", -1)
|
||||
item.Size = strings.Replace(item.Size, "mb", "MB", -1)
|
||||
operations[i] = mongo.NewUpdateOneModel().
|
||||
SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
|
||||
SetUpdate(bson.D{{Key: "$set", Value: item}}).
|
||||
SetUpsert(true)
|
||||
}
|
||||
|
||||
opts := options.BulkWrite().SetOrdered(false)
|
||||
_, err := GameItemCollection.BulkWrite(ctx, operations, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveGameInfo(item *model.GameInfo) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@ -179,44 +148,15 @@ func SaveGameInfo(item *model.GameInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveGameInfos(items []*model.GameInfo) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
operations := make([]mongo.WriteModel, len(items))
|
||||
for i, item := range items {
|
||||
if item.ID.IsZero() {
|
||||
item.ID = primitive.NewObjectID()
|
||||
}
|
||||
if item.CreatedAt.IsZero() {
|
||||
item.CreatedAt = time.Now()
|
||||
}
|
||||
item.UpdatedAt = time.Now()
|
||||
operations[i] = mongo.NewUpdateOneModel().
|
||||
SetFilter(bson.D{{Key: "_id", Value: item.ID}}).
|
||||
SetUpdate(bson.D{{Key: "$set", Value: item}}).
|
||||
SetUpsert(true)
|
||||
}
|
||||
|
||||
opts := options.BulkWrite().SetOrdered(false)
|
||||
_, err := GameInfoCollection.BulkWrite(ctx, operations, opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetAllGameItems() ([]*model.GameItem, error) {
|
||||
var items []*model.GameItem
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
cursor, err := GameItemCollection.Find(ctx, bson.D{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func(cursor *mongo.Cursor, ctx context.Context) {
|
||||
_ = cursor.Close(ctx)
|
||||
}(cursor, ctx)
|
||||
defer cursor.Close(ctx)
|
||||
for cursor.Next(ctx) {
|
||||
var game model.GameItem
|
||||
if err = cursor.Decode(&game); err != nil {
|
||||
@ -258,9 +198,6 @@ func GetGameItemByID(id primitive.ObjectID) (*model.GameItem, error) {
|
||||
}
|
||||
|
||||
func GetGameItemsByIDs(ids []primitive.ObjectID) ([]*model.GameItem, error) {
|
||||
if len(ids) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var items []*model.GameItem
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@ -268,9 +205,7 @@ func GetGameItemsByIDs(ids []primitive.ObjectID) ([]*model.GameItem, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func(cursor *mongo.Cursor, ctx context.Context) {
|
||||
_ = cursor.Close(ctx)
|
||||
}(cursor, ctx)
|
||||
defer cursor.Close(ctx)
|
||||
for cursor.Next(ctx) {
|
||||
var game model.GameItem
|
||||
if err = cursor.Decode(&game); err != nil {
|
||||
@ -284,7 +219,6 @@ func GetGameItemsByIDs(ids []primitive.ObjectID) ([]*model.GameItem, error) {
|
||||
return items, err
|
||||
}
|
||||
|
||||
// SearchGameItems page start from 1, return (items, totalPage, error)
|
||||
func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, int, error) {
|
||||
var items []*model.GameInfo
|
||||
name = removeNoneAlphaNumeric.ReplaceAllString(name, " ")
|
||||
@ -292,21 +226,6 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
|
||||
name = strings.TrimSpace(name)
|
||||
name = strings.Replace(name, " ", ".*", -1)
|
||||
name = fmt.Sprintf("%s.*", name)
|
||||
|
||||
key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
var data struct {
|
||||
Items []*model.GameInfo
|
||||
TotalPage int
|
||||
}
|
||||
err := json.Unmarshal([]byte(val), &data)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return data.Items, data.TotalPage, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@ -325,9 +244,7 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer func(cursor *mongo.Cursor, ctx context.Context) {
|
||||
_ = cursor.Close(ctx)
|
||||
}(cursor, ctx)
|
||||
defer cursor.Close(ctx)
|
||||
for cursor.Next(ctx) {
|
||||
var game model.GameInfo
|
||||
if err = cursor.Decode(&game); err != nil {
|
||||
@ -342,19 +259,40 @@ func SearchGameInfos(name string, page int, pageSize int) ([]*model.GameInfo, in
|
||||
if err := cursor.Err(); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return items, int(totalPage), nil
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(struct {
|
||||
func SearchGameInfosCache(name string, page int, pageSize int) ([]*model.GameInfo, int, error) {
|
||||
type res struct {
|
||||
Items []*model.GameInfo
|
||||
TotalPage int
|
||||
}{
|
||||
Items: items,
|
||||
TotalPage: int(totalPage),
|
||||
})
|
||||
if err == nil {
|
||||
_ = cache.SetWithExpire(key, string(jsonBytes), time.Minute*5)
|
||||
}
|
||||
|
||||
return items, int(totalPage), nil
|
||||
name = strings.ToLower(name)
|
||||
if config.Config.RedisAvaliable {
|
||||
key := fmt.Sprintf("searchGameDetails:%s:%d:%d", name, page, pageSize)
|
||||
val, exist := cache.Get(key)
|
||||
if exist {
|
||||
var data res
|
||||
err := json.Unmarshal([]byte(val), &data)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return data.Items, data.TotalPage, nil
|
||||
} else {
|
||||
data, totalPage, err := SearchGameInfos(name, page, pageSize)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
dataBytes, err := json.Marshal(res{Items: data, TotalPage: totalPage})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
_ = cache.AddWithExpire(key, string(dataBytes), 5*time.Minute)
|
||||
return data, totalPage, nil
|
||||
}
|
||||
} else {
|
||||
return SearchGameInfos(name, page, pageSize)
|
||||
}
|
||||
}
|
||||
|
||||
func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {
|
||||
@ -375,45 +313,6 @@ func GetGameInfoByPlatformID(platform string, id int) (*model.GameInfo, error) {
|
||||
return &game, nil
|
||||
}
|
||||
|
||||
func GetGameInfosByPlatformIDs(platform string, ids []int) ([]*model.GameInfo, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
var filter interface{}
|
||||
switch platform {
|
||||
case "steam":
|
||||
filter = bson.M{"steam_id": bson.M{"$in": ids}}
|
||||
case "igdb":
|
||||
filter = bson.M{"igdb_id": bson.M{"$in": ids}}
|
||||
}
|
||||
var games []*model.GameInfo
|
||||
cursor, err := GameInfoCollection.Find(ctx, filter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = cursor.All(ctx, &games); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return games, nil
|
||||
}
|
||||
|
||||
func HasGameItemOrganized(id primitive.ObjectID) (bool, []*model.GameInfo) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
filter := bson.M{"games": id}
|
||||
var res []*model.GameInfo
|
||||
cursor, err := GameInfoCollection.Find(ctx, filter)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
if err = cursor.All(ctx, &res); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
if len(res) == 0 {
|
||||
return false, nil
|
||||
}
|
||||
return true, res
|
||||
}
|
||||
|
||||
func GetUnorganizedGameItems(num int) ([]*model.GameItem, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@ -440,9 +339,7 @@ func GetUnorganizedGameItems(num int) ([]*model.GameItem, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func(cursor *mongo.Cursor, ctx context.Context) {
|
||||
_ = cursor.Close(ctx)
|
||||
}(cursor, ctx)
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
for cursor.Next(ctx) {
|
||||
var game model.GameItem
|
||||
@ -471,40 +368,28 @@ func GetGameInfoByID(id primitive.ObjectID) (*model.GameInfo, error) {
|
||||
}
|
||||
|
||||
func DeduplicateGameItems() ([]primitive.ObjectID, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
type queryRes struct {
|
||||
ID struct {
|
||||
RawName string `bson:"raw_name"`
|
||||
Download string `bson:"download"`
|
||||
} `bson:"_id"`
|
||||
Count int `bson:"count"`
|
||||
ID string `bson:"_id"`
|
||||
Total int `bson:"total"`
|
||||
IDs []primitive.ObjectID `bson:"ids"`
|
||||
}
|
||||
|
||||
var res []primitive.ObjectID
|
||||
|
||||
pipeline := mongo.Pipeline{
|
||||
bson.D{
|
||||
{Key: "$group",
|
||||
Value: bson.D{
|
||||
{Key: "_id",
|
||||
Value: bson.D{
|
||||
{Key: "raw_name", Value: "$raw_name"},
|
||||
{Key: "download", Value: "$download"},
|
||||
},
|
||||
},
|
||||
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
|
||||
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
|
||||
},
|
||||
},
|
||||
},
|
||||
bson.D{{Key: "$match", Value: bson.D{{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}}}}},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
var qres []queryRes
|
||||
|
||||
pipeline := mongo.Pipeline{
|
||||
bson.D{{Key: "$group", Value: bson.D{
|
||||
{Key: "_id", Value: "$download"},
|
||||
{Key: "total", Value: bson.D{{Key: "$sum", Value: 1}}},
|
||||
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
|
||||
}}},
|
||||
bson.D{{Key: "$match", Value: bson.D{
|
||||
{Key: "total", Value: bson.D{{Key: "$gt", Value: 1}}},
|
||||
}}},
|
||||
}
|
||||
cursor, err := GameItemCollection.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -512,14 +397,34 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
|
||||
if err = cursor.All(ctx, &qres); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range qres {
|
||||
res = append(res, item.IDs[1:]...)
|
||||
}
|
||||
err = DeleteGameItemsByIDs(res)
|
||||
idsToDelete := item.IDs[:len(item.IDs)-1]
|
||||
res = append(res, idsToDelete...)
|
||||
_, err = GameItemCollection.DeleteMany(ctx, bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToDelete}}}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cursor, err := GameInfoCollection.Find(ctx, bson.M{"games": bson.M{"$in": idsToDelete}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var infos []*model.GameInfo
|
||||
if err := cursor.All(ctx, &infos); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, info := range infos {
|
||||
newGames := make([]primitive.ObjectID, 0, len(info.GameIDs))
|
||||
for _, id := range info.GameIDs {
|
||||
if !slices.Contains(idsToDelete, id) {
|
||||
newGames = append(newGames, id)
|
||||
}
|
||||
}
|
||||
info.GameIDs = newGames
|
||||
if err := SaveGameInfo(info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
_, _ = CleanOrphanGamesInGameInfos()
|
||||
return res, nil
|
||||
}
|
||||
@ -662,60 +567,7 @@ func GetSameNameGameInfos() (map[string][]primitive.ObjectID, error) {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func DeleteGameInfosByIDs(ids []primitive.ObjectID) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := GameInfoCollection.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MergeGameInfosWithSameIGDBID() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
pipeline := mongo.Pipeline{
|
||||
bson.D{{Key: "$group", Value: bson.D{
|
||||
{Key: "_id", Value: "$igdb_id"},
|
||||
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
|
||||
{Key: "docs", Value: bson.D{{Key: "$push", Value: "$$ROOT"}}},
|
||||
}}},
|
||||
bson.D{{Key: "$match", Value: bson.D{{Key: "count", Value: bson.D{{Key: "$gt", Value: 1}}}}}},
|
||||
}
|
||||
|
||||
cursor, err := GameInfoCollection.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
type queryRes struct {
|
||||
IGDBID int `bson:"_id"`
|
||||
Infos []model.GameInfo `bson:"docs"`
|
||||
}
|
||||
var res []queryRes
|
||||
if err = cursor.All(ctx, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range res {
|
||||
gameIDs := make([]primitive.ObjectID, 0)
|
||||
deleteInfoIDs := make([]primitive.ObjectID, 0)
|
||||
for _, info := range item.Infos[1:] {
|
||||
gameIDs = append(gameIDs, info.GameIDs...)
|
||||
deleteInfoIDs = append(deleteInfoIDs, info.ID)
|
||||
}
|
||||
item.Infos[0].GameIDs = utils.Unique(append(item.Infos[0].GameIDs, gameIDs...))
|
||||
err = DeleteGameInfosByIDs(deleteInfoIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func MergeGameInfosWithSameName() error {
|
||||
func MergeSameNameGameInfos() error {
|
||||
games, err := GetSameNameGameInfos()
|
||||
if err != nil {
|
||||
return err
|
||||
@ -838,28 +690,6 @@ func DeleteGameItemByID(id primitive.ObjectID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteGameItemsByIDs(ids []primitive.ObjectID) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, err := GameItemCollection.DeleteMany(ctx, bson.M{"_id": bson.M{"$in": ids}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
update := bson.D{{Key: "$pull", Value: bson.D{
|
||||
{Key: "games", Value: bson.D{
|
||||
{Key: "$in", Value: ids},
|
||||
}},
|
||||
}}}
|
||||
_, err = GameInfoCollection.UpdateMany(ctx, bson.M{}, update)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetAllAuthors() ([]string, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
@ -918,30 +748,3 @@ func GetGameInfoByGameItemID(id primitive.ObjectID) (*model.GameInfo, error) {
|
||||
}
|
||||
return res[0], nil
|
||||
}
|
||||
|
||||
func GetOutdatedGameInfos(maxNum int) ([]*model.GameInfo, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
opts := options.Find().SetLimit(int64(maxNum))
|
||||
filter := bson.M{
|
||||
"info_updated_at": bson.M{"$lt": time.Now().Add(-24 * time.Hour * 30)},
|
||||
}
|
||||
cursor, err := GameInfoCollection.Find(ctx, filter, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res []*model.GameInfo
|
||||
if err = cursor.All(ctx, &res); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func MergeGameInfo(oldInfo *model.GameInfo, newInfo *model.GameInfo) {
|
||||
newInfo.ID = oldInfo.ID
|
||||
newInfo.UpdatedAt = time.Now()
|
||||
newInfo.GameIDs = oldInfo.GameIDs
|
||||
newInfo.IGDBID = oldInfo.IGDBID
|
||||
newInfo.SteamID = oldInfo.SteamID
|
||||
newInfo.CreatedAt = oldInfo.CreatedAt
|
||||
}
|
||||
|
@ -9,7 +9,7 @@ import (
|
||||
)
|
||||
|
||||
func ImportGameInfo(filePath string) error {
|
||||
var gameInfo []*model.GameInfo
|
||||
gameInfo := []*model.GameInfo{}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -27,7 +27,7 @@ func ImportGameInfo(filePath string) error {
|
||||
}
|
||||
|
||||
func ImportGameItem(filePath string) error {
|
||||
var gameItem []*model.GameItem
|
||||
gameItem := []*model.GameItem{}
|
||||
data, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -1,9 +1,10 @@
|
||||
services:
|
||||
pcgamedb:
|
||||
image: nite07/pcgamedb
|
||||
build: .
|
||||
container_name: pcgamedb
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "8080:8080"
|
||||
- 127.0.0.1:8080:8080
|
||||
environment:
|
||||
- LOG_LEVEL=info
|
||||
- SERVER_PORT=8080
|
||||
@ -17,6 +18,7 @@ services:
|
||||
- REDIS_DB=0
|
||||
# Read more about environment variables: config/config.go
|
||||
pcgamedb-mongodb:
|
||||
container_name: pcgamedb-mongodb
|
||||
image: mongo:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
@ -26,6 +28,7 @@ services:
|
||||
- ./mongodb:/data/db
|
||||
pcgamedb-redis:
|
||||
image: redis:latest
|
||||
container_name: pcgamedb-redis
|
||||
volumes:
|
||||
- ./redis:/data
|
||||
command: redis-server --appendonly yes
|
||||
|
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
15
go.mod
15
go.mod
@ -1,6 +1,6 @@
|
||||
module pcgamedb
|
||||
|
||||
go 1.23.3
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.0
|
||||
@ -9,8 +9,8 @@ require (
|
||||
github.com/bogdanfinn/tls-client v1.7.8
|
||||
github.com/btcsuite/btcutil v1.0.2
|
||||
github.com/gin-contrib/cors v1.7.2
|
||||
github.com/gin-contrib/multitemplate v1.0.1
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
github.com/jlaffaye/ftp v0.2.0
|
||||
github.com/redis/go-redis/v9 v9.7.0
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/spf13/cobra v1.8.1
|
||||
@ -25,9 +25,6 @@ require (
|
||||
)
|
||||
|
||||
require (
|
||||
git.nite07.com/nite/ccs v0.0.0-20241218080306-1c4bbc6520c3 // indirect
|
||||
github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 // indirect
|
||||
github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/anacrolix/generics v0.0.3-0.20240902042256-7fb2702ef0ca // indirect
|
||||
github.com/anacrolix/missinggo v1.3.0 // indirect
|
||||
@ -52,10 +49,10 @@ require (
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.23.0 // indirect
|
||||
github.com/go-resty/resty/v2 v2.16.2 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/huandu/xstrings v1.5.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
@ -73,8 +70,7 @@ require (
|
||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/quic-go/quic-go v0.48.2 // indirect
|
||||
github.com/refraction-networking/utls v1.6.7 // indirect
|
||||
github.com/quic-go/quic-go v0.48.1 // indirect
|
||||
github.com/spaolacci/murmur3 v1.1.0 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 // indirect
|
||||
@ -93,6 +89,5 @@ require (
|
||||
golang.org/x/tools v0.27.0 // indirect
|
||||
google.golang.org/protobuf v1.35.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
h12.io/socks v1.0.3 // indirect
|
||||
lukechampine.com/blake3 v1.3.0 // indirect
|
||||
)
|
||||
|
287
go.sum
287
go.sum
@ -1,37 +1,8 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo=
|
||||
crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk=
|
||||
crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4=
|
||||
dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU=
|
||||
dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU=
|
||||
dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4=
|
||||
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
|
||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241202053845-550f69c19ba2 h1:hsnRzTS+6Dd+ZFPe486Dg59X0azNOHviDabChc8dKKk=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241202053845-550f69c19ba2/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203153655-eff00a6afd7e h1:SJVt97JqleQiRNP6HpH5OWSneXhBs8bhaQ0Em20KDl4=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203153655-eff00a6afd7e/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203154730-1ecdcb99a31b h1:5g2GTuFxIfba8Tm8liHW6FJQxbjiIu3NmYdr5u7OaiM=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203154730-1ecdcb99a31b/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203155017-60608c26cf61 h1:BjsC5e1NuReTuThi8BT4a4e6SFBQJ3C3R6XchhDF01s=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203155017-60608c26cf61/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff h1:f5OEMRc/zhMxSfHdTJt3dW73NELywBlRxShenb/uaUM=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203155427-10e314ae7eff/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580 h1:B2ewPM44DgyrkycIrUfyTRLM7mXggA0JE8pNbvxjFKw=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241203155655-662e1dc6e580/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c h1:UcIxgKmcQGZqjTJWsQf9MVDviUQFlU+ZK6HJjpW+nAU=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241204132531-f6469471bb6c/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760 h1:ZCMgQt2ILohQ3MSk6RVhnRY4fbQTJVLREbWZjLtVs+Y=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241204135023-d34ae7399760/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241218080306-1c4bbc6520c3 h1:AgG6cZYX7CSzBnJ2jZ9T0ATbutHWbKfrJSPbWRHUW14=
|
||||
git.nite07.com/nite/ccs v0.0.0-20241218080306-1c4bbc6520c3/go.mod h1:+kZxYKbZJ3igYXdgCStq+SocI2Wy0fE0RaGqW6YD71w=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26 h1:6fexoGmvzoXMSk14BZ0AirapVm5c3KUsEjE0jLlVKi8=
|
||||
github.com/Danny-Dasilva/CycleTLS/cycletls v1.0.26/go.mod h1:QFi/EVO7qqru3Ftxz1LR+96jIc91Tifv0DnskF/gWQ8=
|
||||
github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1 h1:/lqhaiz7xdPr6kuaW1tQ/8DdpWdxkdyd9W/6EHz4oRw=
|
||||
github.com/Danny-Dasilva/fhttp v0.0.0-20240217042913-eeeb0b347ce1/go.mod h1:Hvab/V/YKCDXsEpKYKHjAXH5IFOmoq9FsfxjztEqvDc=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4=
|
||||
@ -71,12 +42,10 @@ github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pm
|
||||
github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8=
|
||||
github.com/anacrolix/torrent v1.57.1 h1:CS8rYfC2Oe15NPBhwCNs/3WBY6HiBCPDFpY+s9aFHbA=
|
||||
github.com/anacrolix/torrent v1.57.1/go.mod h1:NNBg4lP2/us9Hp5+cLNcZRILM69cNoKIkqMGqr9AuR0=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
@ -88,7 +57,6 @@ github.com/bogdanfinn/tls-client v1.7.8 h1:oAr+Ox0syJjgcY5ibKLb+r1ofabB1WDysNTY5
|
||||
github.com/bogdanfinn/tls-client v1.7.8/go.mod h1:pQwF0eqfL0gf0mu8hikvu6deZ3ijSPruJDzEKEnnXjU=
|
||||
github.com/bogdanfinn/utls v1.6.1 h1:dKDYAcXEyFFJ3GaWaN89DEyjyRraD1qb4osdEK89ass=
|
||||
github.com/bogdanfinn/utls v1.6.1/go.mod h1:VXIbRZaiY/wHZc6Hu+DZ4O2CgTzjhjCg/Ou3V4r/39Y=
|
||||
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
|
||||
github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
|
||||
github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo=
|
||||
github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8=
|
||||
@ -107,8 +75,6 @@ github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVa
|
||||
github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc=
|
||||
github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY=
|
||||
github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
|
||||
github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
|
||||
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
|
||||
github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k=
|
||||
github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk=
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
@ -117,18 +83,13 @@ github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
|
||||
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
|
||||
github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
|
||||
github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -142,40 +103,29 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
|
||||
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
|
||||
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
|
||||
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
|
||||
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
|
||||
github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI=
|
||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||
github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw=
|
||||
github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E=
|
||||
github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
|
||||
github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
|
||||
github.com/gin-contrib/multitemplate v1.0.1 h1:Asi8boB7NctSoQzbWDosLObon0cYMP5OM+ihQMjlW5M=
|
||||
github.com/gin-contrib/multitemplate v1.0.1/go.mod h1:uU+PnuKoiEHWqB9Zvco+Kqv9KNrsHi6IZOUUgTctMPA=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
|
||||
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
|
||||
github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
|
||||
github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
|
||||
github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
|
||||
github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
|
||||
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
|
||||
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
|
||||
@ -192,11 +142,7 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
|
||||
github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||
github.com/go-resty/resty/v2 v2.16.2 h1:CpRqTjIzq/rweXUt9+GxzzQdlkqMdt8Lm/fuK/CAbAg=
|
||||
github.com/go-resty/resty/v2 v2.16.2/go.mod h1:0fHAoK7JoBy/Ch36N8VFeMsK7xQOHhvWaC3iOktwmIU=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
|
||||
@ -204,10 +150,7 @@ github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
@ -216,45 +159,30 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
|
||||
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/h12w/go-socks5 v0.0.0-20200522160539-76189e178364/go.mod h1:eDJQioIyy4Yn3MVivT7rv/39gAJTrA7lgmYr8EW950c=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo=
|
||||
@ -262,11 +190,11 @@ github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63
|
||||
github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
|
||||
github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
|
||||
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
|
||||
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg=
|
||||
github.com/jlaffaye/ftp v0.2.0/go.mod h1:is2Ds5qkhceAPy2xD6RLI6hmp/qysSoymZ+Z2uTnspI=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ=
|
||||
@ -274,13 +202,11 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
|
||||
github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
@ -293,20 +219,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
|
||||
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
@ -327,47 +249,12 @@ github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsC
|
||||
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
|
||||
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
|
||||
github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
|
||||
github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
|
||||
github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU=
|
||||
github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
|
||||
github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk=
|
||||
github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0=
|
||||
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
|
||||
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
|
||||
github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
|
||||
github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc=
|
||||
github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk=
|
||||
github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
|
||||
github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
|
||||
github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
|
||||
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
|
||||
github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
|
||||
github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
|
||||
github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM=
|
||||
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
|
||||
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
|
||||
github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
|
||||
github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
|
||||
github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw=
|
||||
github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
|
||||
github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
|
||||
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
|
||||
github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
@ -375,7 +262,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
@ -384,67 +270,31 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
|
||||
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
|
||||
github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
|
||||
github.com/quic-go/qtls-go1-20 v0.3.1/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
|
||||
github.com/quic-go/quic-go v0.37.4/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU=
|
||||
github.com/quic-go/quic-go v0.48.1 h1:y/8xmfWI9qmGTc+lBr4jKRUWLGSlSigv847ULJ4hYXA=
|
||||
github.com/quic-go/quic-go v0.48.1/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
|
||||
github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE=
|
||||
github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
|
||||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/refraction-networking/utls v1.5.4/go.mod h1:SPuDbBmgLGp8s+HLNc83FuavwZCFoMmExj+ltUHiHUw=
|
||||
github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM=
|
||||
github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
|
||||
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
|
||||
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
|
||||
github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
|
||||
github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw=
|
||||
github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI=
|
||||
github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU=
|
||||
github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag=
|
||||
github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg=
|
||||
github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw=
|
||||
github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y=
|
||||
github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
|
||||
github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q=
|
||||
github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ=
|
||||
github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I=
|
||||
github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0=
|
||||
github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ=
|
||||
github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk=
|
||||
github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
|
||||
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs=
|
||||
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
|
||||
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
|
||||
@ -459,8 +309,6 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
@ -475,7 +323,6 @@ github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A=
|
||||
github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5 h1:YqAladjX7xpA6BM04leXMWAEjS0mTZ5kUU9KRBriQJc=
|
||||
github.com/tam7t/hpkp v0.0.0-20160821193359-2b70b4024ed5/go.mod h1:2JjD2zLQYH5HO74y5+aE3remJQvl6q4Sn6aWA2wD1Ng=
|
||||
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
|
||||
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
|
||||
@ -483,8 +330,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
|
||||
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
|
||||
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@ -497,13 +342,9 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM=
|
||||
go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4=
|
||||
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
|
||||
go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
||||
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@ -513,210 +354,109 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
|
||||
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
|
||||
golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg=
|
||||
golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
|
||||
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
|
||||
golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
|
||||
golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
|
||||
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
|
||||
golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
|
||||
golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo=
|
||||
golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak=
|
||||
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
|
||||
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
|
||||
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
|
||||
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
|
||||
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
|
||||
golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
|
||||
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
|
||||
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
|
||||
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
|
||||
golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
|
||||
golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o=
|
||||
golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y=
|
||||
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
|
||||
google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
|
||||
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
|
||||
@ -725,10 +465,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
|
||||
google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
@ -738,7 +474,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
@ -746,19 +481,11 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
|
||||
h12.io/socks v1.0.3 h1:Ka3qaQewws4j4/eDQnOdpr4wXsC//dXtWvftlIcCQUo=
|
||||
h12.io/socks v1.0.3/go.mod h1:AIhxy1jOId/XCz9BO+EIgNL2rQiPTBNnOfnVnQ+3Eck=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE=
|
||||
lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck=
|
||||
sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
|
||||
|
@ -19,25 +19,9 @@ type GameInfo struct {
|
||||
Languages []string `json:"languages" bson:"languages"`
|
||||
Screenshots []string `json:"screenshots" bson:"screenshots"`
|
||||
GameIDs []primitive.ObjectID `json:"game_ids" bson:"games"`
|
||||
Games []*GameItem `json:"games" bson:"-"`
|
||||
Games []*GameItem `json:"game_downloads" bson:"-"`
|
||||
CreatedAt time.Time `json:"created_at" bson:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
|
||||
FirstReleaseDate time.Time `json:"first_release_date" bson:"first_release_date"`
|
||||
GameEngines []string `json:"game_engines" bson:"game_engines"`
|
||||
GameModes []string `json:"game_modes" bson:"game_modes"`
|
||||
Genres []string `json:"genres" bson:"genres"`
|
||||
Themes []string `json:"themes" bson:"themes"`
|
||||
Platforms []string `json:"platforms" bson:"platforms"`
|
||||
PlayerPerspectives []string `json:"player_perspectives" bson:"player_perspectives"`
|
||||
SimilarGames []int `json:"similar_games" bson:"similar_games"`
|
||||
Videos []string `json:"videos" bson:"videos"`
|
||||
Websites []string `json:"websites" bson:"websites"`
|
||||
Collections []GameCollection `json:"collections" bson:"collections"`
|
||||
}
|
||||
|
||||
type GameCollection struct {
|
||||
Games []int `json:"games"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type GameItem struct {
|
||||
@ -49,7 +33,6 @@ type GameItem struct {
|
||||
Url string `json:"url" bson:"url"`
|
||||
Password string `json:"password,omitempty" bson:"password"`
|
||||
Author string `json:"author,omitempty" bson:"author"`
|
||||
Platform string `json:"platform,omitempty" bson:"platform"`
|
||||
UpdateFlag string `json:"-" bson:"update_flag,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at" bson:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
|
||||
|
231
model/igdb.go
231
model/igdb.go
@ -1,183 +1,66 @@
|
||||
package model
|
||||
|
||||
type IGDBGameDetail struct {
|
||||
ID int `json:"id"`
|
||||
AgeRatings []int `json:"age_ratings"`
|
||||
AggregatedRating float64 `json:"aggregated_rating"`
|
||||
AggregatedRatingCount int `json:"aggregated_rating_count"`
|
||||
Artworks []int `json:"artworks"`
|
||||
Category int `json:"category"`
|
||||
Cover struct {
|
||||
ID int `json:"id"`
|
||||
AlphaChannel bool `json:"alpha_channel"`
|
||||
Animated bool `json:"animated"`
|
||||
Game int `json:"game"`
|
||||
Height int `json:"height"`
|
||||
ImageID string `json:"image_id"`
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"cover"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Dlcs []int `json:"dlcs"`
|
||||
ExternalGames []int `json:"external_games"`
|
||||
FirstReleaseDate int `json:"first_release_date"`
|
||||
GameEngines []struct {
|
||||
ID int `json:"id"`
|
||||
Companies []int `json:"companies"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Logo int `json:"logo"`
|
||||
Name string `json:"name"`
|
||||
Platforms []int `json:"platforms"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"game_engines"`
|
||||
GameModes []struct {
|
||||
ID int `json:"id"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"game_modes"`
|
||||
Genres []struct {
|
||||
ID int `json:"id"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"genres"`
|
||||
Hypes int `json:"hypes"`
|
||||
InvolvedCompanies []struct {
|
||||
ID int `json:"id"`
|
||||
Company int `json:"company"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Developer bool `json:"developer"`
|
||||
Game int `json:"game"`
|
||||
Porting bool `json:"porting"`
|
||||
Publisher bool `json:"publisher"`
|
||||
Supporting bool `json:"supporting"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"involved_companies"`
|
||||
Name string `json:"name"`
|
||||
Platforms []struct {
|
||||
ID int `json:"id"`
|
||||
Abbreviation string `json:"abbreviation"`
|
||||
AlternativeName string `json:"alternative_name"`
|
||||
Category int `json:"category"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Name string `json:"name"`
|
||||
PlatformLogo int `json:"platform_logo"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Versions []int `json:"versions"`
|
||||
Websites []int `json:"websites"`
|
||||
Checksum string `json:"checksum"`
|
||||
Generation int `json:"generation,omitempty"`
|
||||
PlatformFamily int `json:"platform_family,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
} `json:"platforms"`
|
||||
PlayerPerspectives []struct {
|
||||
ID int `json:"id"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"player_perspectives"`
|
||||
Rating float64 `json:"rating"`
|
||||
RatingCount int `json:"rating_count"`
|
||||
ReleaseDates []struct {
|
||||
ID int `json:"id"`
|
||||
Category int `json:"category"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Date int `json:"date"`
|
||||
Game int `json:"game"`
|
||||
Human string `json:"human"`
|
||||
M int `json:"m"`
|
||||
Platform int `json:"platform"`
|
||||
Region int `json:"region"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
Y int `json:"y"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"release_dates"`
|
||||
Screenshots []struct {
|
||||
ID int `json:"id"`
|
||||
Game int `json:"game"`
|
||||
Height int `json:"height"`
|
||||
ImageID string `json:"image_id"`
|
||||
URL string `json:"url"`
|
||||
Width int `json:"width"`
|
||||
Checksum string `json:"checksum"`
|
||||
AlphaChannel bool `json:"alpha_channel,omitempty"`
|
||||
Animated bool `json:"animated,omitempty"`
|
||||
} `json:"screenshots"`
|
||||
SimilarGames []int `json:"similar_games"`
|
||||
Slug string `json:"slug"`
|
||||
Summary string `json:"summary"`
|
||||
Tags []int `json:"tags"`
|
||||
Themes []struct {
|
||||
ID int `json:"id"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"themes"`
|
||||
TotalRating float64 `json:"total_rating"`
|
||||
TotalRatingCount int `json:"total_rating_count"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Videos []struct {
|
||||
ID int `json:"id"`
|
||||
Game int `json:"game"`
|
||||
Name string `json:"name"`
|
||||
VideoID string `json:"video_id"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"videos"`
|
||||
Websites []struct {
|
||||
ID int `json:"id"`
|
||||
Category int `json:"category"`
|
||||
Game int `json:"game"`
|
||||
Trusted bool `json:"trusted"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"websites"`
|
||||
Checksum string `json:"checksum"`
|
||||
LanguageSupports []struct {
|
||||
ID int `json:"id"`
|
||||
Game int `json:"game"`
|
||||
Language int `json:"language"`
|
||||
LanguageSupportType int `json:"language_support_type"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
Checksum string `json:"checksum"`
|
||||
} `json:"language_supports"`
|
||||
Collections []struct {
|
||||
ID int `json:"id"`
|
||||
CreatedAt int `json:"created_at"`
|
||||
Games []int `json:"games"`
|
||||
Name string `json:"name"`
|
||||
Slug string `json:"slug"`
|
||||
UpdatedAt int `json:"updated_at"`
|
||||
URL string `json:"url"`
|
||||
Checksum string `json:"checksum"`
|
||||
Type int `json:"type"`
|
||||
} `json:"collections"`
|
||||
VersionParent int `json:"version_parent,omitempty"`
|
||||
VersionTitle string `json:"version_title,omitempty"`
|
||||
ID int `json:"id,omitempty"`
|
||||
ParentGame int `json:"parent_game,omitempty"`
|
||||
AgeRatings []int `json:"age_ratings,omitempty"`
|
||||
AlternativeNames []struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
} `json:"alternative_names,omitempty"`
|
||||
Category int `json:"category,omitempty"`
|
||||
Cover struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
} `json:"cover,omitempty"`
|
||||
CreatedAt int `json:"created_at,omitempty"`
|
||||
ExternalGames []int `json:"external_games,omitempty"`
|
||||
FirstReleaseDate int `json:"first_release_date,omitempty"`
|
||||
Franchises []int `json:"franchises,omitempty"`
|
||||
GameModes []int `json:"game_modes,omitempty"`
|
||||
Genres []int `json:"genres,omitempty"`
|
||||
InvolvedCompanies []struct {
|
||||
Company int `json:"company,omitempty"`
|
||||
Developer bool `json:"developer,omitempty"`
|
||||
Publisher bool `json:"publisher,omitempty"`
|
||||
} `json:"involved_companies,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
Platforms []int `json:"platforms,omitempty"`
|
||||
PlayerPerspectives []int `json:"player_perspectives,omitempty"`
|
||||
Rating float64 `json:"rating,omitempty"`
|
||||
RatingCount int `json:"rating_count,omitempty"`
|
||||
ReleaseDates []int `json:"release_dates,omitempty"`
|
||||
Screenshots []struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
} `json:"screenshots,omitempty"`
|
||||
SimilarGames []int `json:"similar_games,omitempty"`
|
||||
Slug string `json:"slug,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Tags []int `json:"tags,omitempty"`
|
||||
Themes []int `json:"themes,omitempty"`
|
||||
TotalRating float64 `json:"total_rating,omitempty"`
|
||||
TotalRatingCount int `json:"total_rating_count,omitempty"`
|
||||
UpdatedAt int `json:"updated_at,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
VersionParent int `json:"version_parent,omitempty"`
|
||||
VersionTitle string `json:"version_title,omitempty"`
|
||||
Checksum string `json:"checksum,omitempty"`
|
||||
Websites []int `json:"websites,omitempty"`
|
||||
GameLocalizations []int `json:"game_localizations,omitempty"`
|
||||
AggregatedRating float64 `json:"aggregated_rating,omitempty"`
|
||||
AggregatedRatingCount int `json:"aggregated_rating_count,omitempty"`
|
||||
Artworks []int `json:"artworks,omitempty"`
|
||||
Bundles []int `json:"bundles,omitempty"`
|
||||
Collection int `json:"collection,omitempty"`
|
||||
GameEngines []int `json:"game_engines,omitempty"`
|
||||
Keywords []int `json:"keywords,omitempty"`
|
||||
MultiplayerModes []int `json:"multiplayer_modes,omitempty"`
|
||||
StandaloneExpansions []int `json:"standalone_expansions,omitempty"`
|
||||
Storyline string `json:"storyline,omitempty"`
|
||||
Videos []int `json:"videos,omitempty"`
|
||||
LanguageSupports []struct {
|
||||
Language int `json:"language,omitempty"`
|
||||
LanguageSupportType int `json:"language_support_type,omitempty"`
|
||||
} `json:"language_supports,omitempty"`
|
||||
Collections []int `json:"collections,omitempty"`
|
||||
}
|
||||
|
||||
type IGDBGameDetails []*IGDBGameDetail
|
||||
|
@ -17,7 +17,7 @@ type GetGameItemByRawNameRequest struct {
|
||||
type GetGameItemByRawNameResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
GameItem []*model.GameItem `json:"games,omitempty"`
|
||||
GameItem []*model.GameItem `json:"game_downloads,omitempty"`
|
||||
}
|
||||
|
||||
// GetGameItemByRawName retrieves game download details by raw name.
|
||||
@ -26,7 +26,7 @@ type GetGameItemByRawNameResponse struct {
|
||||
// @Tags game
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param name path string true "Game Raw Name"
|
||||
// @Param name path string true "Game Download Raw Name"
|
||||
// @Success 200 {object} GetGameItemByRawNameResponse
|
||||
// @Failure 400 {object} GetGameItemByRawNameResponse
|
||||
// @Failure 500 {object} GetGameItemByRawNameResponse
|
||||
|
@ -19,10 +19,10 @@ type GetGameItemsByAuthorResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
TotalPage int `json:"total_page"`
|
||||
GameItems []*model.GameItem `json:"games,omitempty"`
|
||||
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
|
||||
}
|
||||
|
||||
// GetGameItemsByAuthorHandler returns games by author
|
||||
// GetGameItemsByAuthorHandler returns all game downloads by author
|
||||
// @Summary Get game downloads by author
|
||||
// @Description Get game downloads by author
|
||||
// @Tags game
|
||||
|
@ -1,113 +0,0 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type GetPopularGamesResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Games []*model.GameInfo `json:"games"`
|
||||
}
|
||||
|
||||
// GetPopularGameInfosHandler Get popular games
|
||||
// @Summary Get popular games
|
||||
// @Description Get popular games based on a specified type
|
||||
// @Tags popular
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param type path string true "Type(igdb-most-visited, igdb-most-wanted-to-play, igdb-most-playing, igdb-most-played, steam-top, steam-week-top, steam-month-top, steam-best-of-the-year, steam-most-played)"
|
||||
// @Success 200 {object} GetPopularGamesResponse
|
||||
// @Failure 400 {object} GetPopularGamesResponse
|
||||
// @Failure 500 {object} GetPopularGamesResponse
|
||||
// @Router /popular/{type} [get]
|
||||
func GetPopularGameInfosHandler(c *gin.Context) {
|
||||
rankingType, exist := c.Params.Get("type")
|
||||
if !exist {
|
||||
c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
|
||||
Status: "error",
|
||||
Message: "Missing ranking type",
|
||||
})
|
||||
}
|
||||
popularityType := 0
|
||||
var steam250Func func() ([]*model.GameInfo, error) = nil
|
||||
switch rankingType {
|
||||
case "igdb-most-visited":
|
||||
popularityType = 1
|
||||
case "igdb-most-wanted-to-play":
|
||||
popularityType = 2
|
||||
case "igdb-most-playing":
|
||||
popularityType = 3
|
||||
case "igdb-most-played":
|
||||
popularityType = 4
|
||||
case "steam-top":
|
||||
steam250Func = crawler.GetSteam250Top250
|
||||
case "steam-week-top":
|
||||
steam250Func = crawler.GetSteam250WeekTop50
|
||||
case "steam-best-of-the-year":
|
||||
steam250Func = crawler.GetSteam250BestOfTheYear
|
||||
case "steam-most-played":
|
||||
steam250Func = crawler.GetSteam250MostPlayed
|
||||
case "steam-month-top":
|
||||
steam250Func = crawler.GetSteam250MonthTop50
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, GetPopularGamesResponse{
|
||||
Status: "error",
|
||||
Message: "Invalid ranking type",
|
||||
})
|
||||
}
|
||||
var infos []*model.GameInfo
|
||||
var err error
|
||||
if steam250Func != nil {
|
||||
infos, err = steam250Func()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, GetPopularGamesResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
if len(infos) > 10 {
|
||||
infos = infos[:10]
|
||||
}
|
||||
} else {
|
||||
offset := 0
|
||||
for len(infos) < 10 {
|
||||
ids, err := crawler.GetIGDBPopularGameIDs(popularityType, offset, 20)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, GetPopularGamesResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
offset += 20
|
||||
pids := make([]int, 20)
|
||||
for _, id := range ids {
|
||||
pid, err := crawler.GetIGDBAppParent(id)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
pids = append(pids, pid)
|
||||
}
|
||||
newInfos, err := db.GetGameInfosByPlatformIDs("igdb", pids)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, GetPopularGamesResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
infos = append(infos, newInfos...)
|
||||
}
|
||||
if len(infos) > 10 {
|
||||
infos = infos[:10]
|
||||
}
|
||||
}
|
||||
c.JSON(http.StatusOK, GetPopularGamesResponse{
|
||||
Status: "ok",
|
||||
Games: infos,
|
||||
})
|
||||
}
|
66
server/handler/get_ranking.go
Normal file
66
server/handler/get_ranking.go
Normal file
@ -0,0 +1,66 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/model"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type GetRankingResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Games []*model.GameInfo `json:"games"`
|
||||
}
|
||||
|
||||
// GetRanking retrieves game rankings.
|
||||
// @Summary Retrieve rankings
|
||||
// @Description Retrieves rankings based on a specified type
|
||||
// @Tags ranking
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param type path string true "Ranking Type(top, week-top, best-of-the-year, most-played)"
|
||||
// @Success 200 {object} GetRankingResponse
|
||||
// @Failure 400 {object} GetRankingResponse
|
||||
// @Failure 500 {object} GetRankingResponse
|
||||
// @Router /ranking/{type} [get]
|
||||
func GetRankingHandler(c *gin.Context) {
|
||||
rankingType, exist := c.Params.Get("type")
|
||||
if !exist {
|
||||
c.JSON(http.StatusBadRequest, GetRankingResponse{
|
||||
Status: "error",
|
||||
Message: "Missing ranking type",
|
||||
})
|
||||
}
|
||||
var f func() ([]*model.GameInfo, error)
|
||||
switch rankingType {
|
||||
case "top":
|
||||
f = crawler.GetSteam250Top250Cache
|
||||
case "week-top":
|
||||
f = crawler.GetSteam250WeekTop50Cache
|
||||
case "best-of-the-year":
|
||||
f = crawler.GetSteam250BestOfTheYearCache
|
||||
case "most-played":
|
||||
f = crawler.GetSteam250MostPlayedCache
|
||||
default:
|
||||
c.JSON(http.StatusBadRequest, GetRankingResponse{
|
||||
Status: "error",
|
||||
Message: "Invalid ranking type",
|
||||
})
|
||||
return
|
||||
}
|
||||
rank, err := f()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, GetRankingResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, GetRankingResponse{
|
||||
Status: "ok",
|
||||
Games: rank,
|
||||
})
|
||||
}
|
@ -17,16 +17,16 @@ type GetUnorganizedGameItemsResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Size int `json:"size,omitempty"`
|
||||
GameItems []*model.GameItem `json:"games,omitempty"`
|
||||
GameItems []*model.GameItem `json:"game_downloads,omitempty"`
|
||||
}
|
||||
|
||||
// GetUnorganizedGameItems retrieves a list of unorganized games.
|
||||
// @Summary List unorganized games
|
||||
// @Description Retrieves games that have not been organized
|
||||
// GetUnorganizedGameItems retrieves a list of unorganized game downloads.
|
||||
// @Summary List unorganized game downloads
|
||||
// @Description Retrieves game downloads that have not been organized
|
||||
// @Tags game
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param num query int false "Number of games to retrieve"
|
||||
// @Param num query int false "Number of game downloads to retrieve"
|
||||
// @Success 200 {object} GetUnorganizedGameItemsResponse
|
||||
// @Failure 400 {object} GetUnorganizedGameItemsResponse
|
||||
// @Failure 500 {object} GetUnorganizedGameItemsResponse
|
||||
|
@ -16,14 +16,17 @@ import (
|
||||
type HealthCheckResponse struct {
|
||||
Version string `json:"version"`
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Date string `json:"date"`
|
||||
Uptime string `json:"uptime"`
|
||||
Alloc string `json:"alloc"`
|
||||
AutoCrawl bool `json:"auto_crawl"`
|
||||
AutoCrawlCron string `json:"auto_crawl_cron"`
|
||||
GameItem int64 `json:"game_num"`
|
||||
GameInfo int64 `json:"game_info_num"`
|
||||
Unorganized int64 `json:"unorganized_game_num"`
|
||||
AutoCrawlCron string `json:"auto_crawl_cron,omitempty"`
|
||||
GameItem int64 `json:"game_download,omitempty"`
|
||||
GameInfo int64 `json:"game_info,omitempty"`
|
||||
Unorganized int64 `json:"unorganized,omitempty"`
|
||||
RedisAvaliable bool `json:"redis_avaliable"`
|
||||
OnlineFixAvaliable bool `json:"online_fix_avaliable"`
|
||||
MegaAvaliable bool `json:"mega_avaliable"`
|
||||
}
|
||||
|
||||
@ -57,6 +60,8 @@ func HealthCheckHandler(c *gin.Context) {
|
||||
GameItem: downloadCount,
|
||||
GameInfo: infoCount,
|
||||
Unorganized: unorganizedCount,
|
||||
RedisAvaliable: config.Config.RedisAvaliable,
|
||||
OnlineFixAvaliable: config.Config.OnlineFixAvaliable,
|
||||
MegaAvaliable: config.Config.MegaAvaliable,
|
||||
})
|
||||
}
|
||||
|
@ -53,7 +53,7 @@ func SearchGamesHandler(c *gin.Context) {
|
||||
if req.PageSize > 10 {
|
||||
req.PageSize = 10
|
||||
}
|
||||
items, totalPage, err := db.SearchGameInfos(req.Keyword, req.Page, req.PageSize)
|
||||
items, totalPage, err := db.SearchGameInfosCache(req.Keyword, req.Page, req.PageSize)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, SearchGamesResponse{
|
||||
Status: "error",
|
||||
|
100
server/handler/update_game_info.go
Normal file
100
server/handler/update_game_info.go
Normal file
@ -0,0 +1,100 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/model"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
)
|
||||
|
||||
type UpdateGameInfoRequest struct {
|
||||
GameID string `json:"game_id" binding:"required"`
|
||||
Platform string `json:"platform" binding:"required"`
|
||||
PlatformID int `json:"platform_id" binding:"required"`
|
||||
}
|
||||
|
||||
type UpdateGameInfoResponse struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
GameInfo *model.GameInfo `json:"game_info,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateGameInfoHandler updates game information.
|
||||
// @Summary Update game info
|
||||
// @Description Updates details of a game
|
||||
// @Tags game
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param Authorization header string true "Authorization: Bearer <api_key>"
|
||||
// @Param body body handler.UpdateGameInfoRequest true "Update Game Info Request"
|
||||
// @Success 200 {object} handler.UpdateGameInfoResponse
|
||||
// @Failure 400 {object} handler.UpdateGameInfoResponse
|
||||
// @Failure 401 {object} handler.UpdateGameInfoResponse
|
||||
// @Failure 500 {object} handler.UpdateGameInfoResponse
|
||||
// @Router /game/update [post]
|
||||
func UpdateGameInfoHandler(c *gin.Context) {
|
||||
var req UpdateGameInfoRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
req.Platform = strings.ToLower(req.Platform)
|
||||
platformMap := map[string]bool{
|
||||
"steam": true,
|
||||
"igdb": true,
|
||||
}
|
||||
if _, ok := platformMap[req.Platform]; !ok {
|
||||
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
|
||||
Status: "error",
|
||||
Message: "Invalid platform",
|
||||
})
|
||||
return
|
||||
}
|
||||
objID, err := primitive.ObjectIDFromHex(req.GameID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, UpdateGameInfoResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
info, err := db.GetGameInfoByID(objID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
newInfo, err := crawler.GenerateGameInfo(req.Platform, req.PlatformID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
newInfo.ID = objID
|
||||
newInfo.GameIDs = info.GameIDs
|
||||
err = db.SaveGameInfo(newInfo)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, UpdateGameInfoResponse{
|
||||
Status: "error",
|
||||
Message: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, UpdateGameInfoResponse{
|
||||
Status: "ok",
|
||||
Message: "Game info updated successfully",
|
||||
GameInfo: newInfo,
|
||||
})
|
||||
}
|
@ -1,10 +1,12 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"pcgamedb/log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"pcgamedb/log"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
@ -12,61 +14,34 @@ import (
|
||||
func Logger() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
startTime := time.Now()
|
||||
path := c.Request.URL.Path
|
||||
raw := c.Request.URL.RawQuery
|
||||
|
||||
c.Next()
|
||||
|
||||
if shouldSkipLog(path) {
|
||||
endTime := time.Now()
|
||||
latencyTime := endTime.Sub(startTime).Milliseconds()
|
||||
reqMethod := c.Request.Method
|
||||
reqURI := c.Request.RequestURI
|
||||
statusCode := c.Writer.Status()
|
||||
clientIP := c.ClientIP()
|
||||
|
||||
if strings.HasPrefix(reqURI, "/swagger/") ||
|
||||
strings.EqualFold(reqURI, "/favicon.ico") {
|
||||
return
|
||||
}
|
||||
|
||||
if raw != "" {
|
||||
path = path + "?" + raw
|
||||
}
|
||||
|
||||
fields := []zap.Field{
|
||||
zap.Int("status", c.Writer.Status()),
|
||||
zap.String("method", c.Request.Method),
|
||||
zap.String("path", path),
|
||||
zap.String("ip", getRealIP(c)),
|
||||
zap.Duration("latency", time.Since(startTime)),
|
||||
}
|
||||
log.Logger.Info(
|
||||
"request",
|
||||
zap.Int("code", statusCode),
|
||||
zap.String("method", reqMethod),
|
||||
zap.String("uri", reqURI),
|
||||
zap.String("ip", clientIP),
|
||||
zap.String("latency", strconv.Itoa(int(latencyTime))+"ms"),
|
||||
)
|
||||
|
||||
if len(c.Errors) > 0 {
|
||||
fields = append(fields, zap.Strings("errors", c.Errors.Errors()))
|
||||
for _, e := range c.Errors.Errors() {
|
||||
log.Logger.Error(e)
|
||||
}
|
||||
}
|
||||
log.Logger.Info("Request", fields...)
|
||||
}
|
||||
}
|
||||
|
||||
func getRealIP(c *gin.Context) string {
|
||||
if ip := c.GetHeader("X-Real-IP"); ip != "" {
|
||||
return ip
|
||||
}
|
||||
if ip := c.GetHeader("X-Forwarded-For"); ip != "" {
|
||||
if index := strings.Index(ip, ","); index != -1 {
|
||||
return strings.TrimSpace(ip[:index])
|
||||
}
|
||||
return ip
|
||||
}
|
||||
if ip := c.GetHeader("X-Originating-IP"); ip != "" {
|
||||
return ip
|
||||
}
|
||||
return c.ClientIP()
|
||||
}
|
||||
|
||||
func shouldSkipLog(path string) bool {
|
||||
skipPaths := []string{
|
||||
"/swagger/",
|
||||
"/favicon.ico",
|
||||
"/health",
|
||||
}
|
||||
|
||||
for _, p := range skipPaths {
|
||||
if strings.HasPrefix(path, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
167
server/route.go
167
server/route.go
@ -1,177 +1,25 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"pcgamedb/crawler"
|
||||
"pcgamedb/db"
|
||||
"pcgamedb/log"
|
||||
"pcgamedb/server/handler"
|
||||
"pcgamedb/server/middleware"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-contrib/cors"
|
||||
"github.com/gin-contrib/multitemplate"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"pcgamedb/docs"
|
||||
docs "pcgamedb/docs"
|
||||
|
||||
swaggerfiles "github.com/swaggo/files"
|
||||
ginSwagger "github.com/swaggo/gin-swagger"
|
||||
)
|
||||
|
||||
//go:embed templates templates/layouts static
|
||||
var frontendFS embed.FS
|
||||
|
||||
func initRoute(app *gin.Engine) {
|
||||
app.Use(cors.New(cors.Config{
|
||||
AllowAllOrigins: true,
|
||||
}))
|
||||
|
||||
initFrontend(app)
|
||||
initApi(app)
|
||||
}
|
||||
|
||||
func initFrontend(app *gin.Engine) {
|
||||
app.Use(middleware.Logger())
|
||||
|
||||
// Load static files
|
||||
staticFs, err := fs.Sub(frontendFS, "static")
|
||||
if err != nil {
|
||||
log.Logger.Fatal("Error loading static files", zap.Error(err))
|
||||
return
|
||||
}
|
||||
app.StaticFS("/static", http.FS(staticFs))
|
||||
|
||||
// Load templates
|
||||
// directly using templates app.LoadHTMLFiles() to load all templates leads to error
|
||||
// because use templates with same name in different html file will case overridden
|
||||
// so we need to load all templates manually and use multitemplate to render them
|
||||
r := multitemplate.NewRenderer()
|
||||
|
||||
layoutFiles, err := frontendFS.ReadDir("templates/layouts")
|
||||
if err != nil {
|
||||
log.Logger.Fatal("Error loading layout templates", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
rootFiles, err := frontendFS.ReadDir("templates")
|
||||
if err != nil {
|
||||
log.Logger.Fatal("Error loading root templates", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
for _, rootFile := range rootFiles {
|
||||
if rootFile.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := filepath.Base(rootFile.Name())
|
||||
templateFiles := []string{"templates/" + name}
|
||||
for _, layout := range layoutFiles {
|
||||
if !layout.IsDir() {
|
||||
templateFiles = append(templateFiles, "templates/layouts/"+layout.Name())
|
||||
}
|
||||
}
|
||||
r.AddFromFS(name, frontendFS, templateFiles...)
|
||||
}
|
||||
|
||||
app.HTMLRender = r
|
||||
|
||||
// Load routes
|
||||
app.GET("/", func(ctx *gin.Context) {
|
||||
monthTop, err := crawler.GetSteam250MonthTop50()
|
||||
if err != nil {
|
||||
ctx.HTML(500, "500.html", err)
|
||||
return
|
||||
}
|
||||
mostPlayed, err := crawler.GetSteam250MostPlayed()
|
||||
if err != nil {
|
||||
ctx.HTML(500, "500.html", err)
|
||||
return
|
||||
}
|
||||
bestOfTheYear, err := crawler.GetSteam250BestOfTheYear()
|
||||
if err != nil {
|
||||
ctx.HTML(500, "500.html", err)
|
||||
return
|
||||
}
|
||||
ctx.HTML(200, "index.html", gin.H{
|
||||
"MonthTop": monthTop,
|
||||
"MostPlayed": mostPlayed,
|
||||
"BestOfTheYear": bestOfTheYear,
|
||||
})
|
||||
})
|
||||
|
||||
app.GET("/game/:id", func(ctx *gin.Context) {
|
||||
idStr := ctx.Param("id")
|
||||
id, err := primitive.ObjectIDFromHex(idStr)
|
||||
if err != nil {
|
||||
ctx.HTML(400, "400.html", nil)
|
||||
return
|
||||
}
|
||||
info, err := db.GetGameInfoByID(id)
|
||||
if err != nil {
|
||||
ctx.HTML(500, "500.html", err)
|
||||
return
|
||||
}
|
||||
games, err := db.GetGameItemsByIDs(info.GameIDs)
|
||||
if err != nil {
|
||||
ctx.HTML(500, "500.html", err)
|
||||
return
|
||||
}
|
||||
info.Games = games
|
||||
ctx.HTML(200, "game.html", info)
|
||||
})
|
||||
|
||||
app.GET("/search", func(ctx *gin.Context) {
|
||||
key := ctx.Query("key")
|
||||
page := ctx.Query("page")
|
||||
key = strings.TrimSpace(key)
|
||||
if len(key) < 2 {
|
||||
ctx.HTML(400, "400.html", errors.New("Search key should be at least 2 characters long"))
|
||||
return
|
||||
}
|
||||
if page == "" {
|
||||
page = "1"
|
||||
}
|
||||
pageInt, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
ctx.HTML(400, "400.html", err)
|
||||
return
|
||||
}
|
||||
games, totalPage, err := db.SearchGameInfos(key, pageInt, 10)
|
||||
if err != nil {
|
||||
ctx.HTML(500, "500.html", err)
|
||||
return
|
||||
}
|
||||
res := gin.H{
|
||||
"Games": games,
|
||||
"TotalPage": totalPage,
|
||||
"CurrentPage": pageInt,
|
||||
"Key": key,
|
||||
}
|
||||
if pageInt > 1 {
|
||||
res["PrevPage"] = pageInt - 1
|
||||
}
|
||||
if pageInt < totalPage {
|
||||
res["NextPage"] = pageInt + 1
|
||||
}
|
||||
ctx.HTML(200, "search.html", res)
|
||||
})
|
||||
}
|
||||
|
||||
func initApi(app *gin.Engine) {
|
||||
apiGroup := app.Group("/api")
|
||||
|
||||
GameInfoGroup := apiGroup.Group("/game")
|
||||
GameInfoGroup.Use(middleware.Logger())
|
||||
GameInfoGroup := app.Group("/game")
|
||||
GameItemGroup := GameInfoGroup.Group("/raw")
|
||||
GameItemGroup.Use(middleware.Logger())
|
||||
|
||||
GameItemGroup.GET("/unorganized", handler.GetUnorganizedGameItemsHandler)
|
||||
GameItemGroup.POST("/organize", middleware.Auth(), handler.OrganizeGameItemHandler)
|
||||
@ -183,13 +31,14 @@ func initApi(app *gin.Engine) {
|
||||
GameInfoGroup.GET("/name/:name", handler.GetGameInfosByNameHandler)
|
||||
GameInfoGroup.GET("/platform/:platform_type/:platform_id", handler.GetGameInfoByPlatformIDHandler)
|
||||
GameInfoGroup.GET("/id/:id", handler.GetGameInfoByIDHandler)
|
||||
GameInfoGroup.PUT("/update", middleware.Auth(), handler.UpdateGameInfoHandler)
|
||||
GameInfoGroup.DELETE("/id/:id", middleware.Auth(), handler.DeleteGameInfoHandler)
|
||||
|
||||
apiGroup.GET("/popular/:type", middleware.Logger(), handler.GetPopularGameInfosHandler)
|
||||
apiGroup.GET("/healthcheck", handler.HealthCheckHandler)
|
||||
apiGroup.GET("/author", middleware.Logger(), handler.GetAllAuthorsHandler)
|
||||
apiGroup.POST("/clean", middleware.Logger(), middleware.Auth(), handler.CleanGameHandler)
|
||||
app.GET("/ranking/:type", handler.GetRankingHandler)
|
||||
app.GET("/healthcheck", handler.HealthCheckHandler)
|
||||
app.GET("/author", handler.GetAllAuthorsHandler)
|
||||
app.POST("/clean", middleware.Auth(), handler.CleanGameHandler)
|
||||
|
||||
docs.SwaggerInfo.BasePath = "/api"
|
||||
apiGroup.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler))
|
||||
app.GET("/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler))
|
||||
}
|
||||
|
@ -26,11 +26,10 @@ func Run() {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
gin.DefaultWriter = io.Discard
|
||||
app := gin.New()
|
||||
app.Use(middleware.Logger())
|
||||
app.Use(middleware.Recovery())
|
||||
initRoute(app)
|
||||
log.Logger.Info("Server running", zap.String("port", config.Config.Server.Port))
|
||||
|
||||
// Start auto-crawl task
|
||||
if config.Config.Server.AutoCrawl {
|
||||
go func() {
|
||||
c := cron.New()
|
||||
@ -41,17 +40,6 @@ func Run() {
|
||||
c.Start()
|
||||
}()
|
||||
}
|
||||
|
||||
// Start auto-update task
|
||||
go func() {
|
||||
c := cron.New()
|
||||
_, err := c.AddFunc("0 */3 * * *", func() { task.UpdateOutdatedGameInfos(log.TaskLogger) })
|
||||
if err != nil {
|
||||
log.Logger.Error("Error adding cron job", zap.Error(err))
|
||||
}
|
||||
c.Start()
|
||||
}()
|
||||
|
||||
err := app.Run(":" + config.Config.Server.Port)
|
||||
if err != nil {
|
||||
log.Logger.Panic("Failed to run server", zap.Error(err))
|
||||
|
Binary file not shown.
Before Width: | Height: | Size: 11 KiB |
@ -1,7 +0,0 @@
|
||||
{{template "base" .}}
|
||||
{{define "title"}}Bad Request{{end}}
|
||||
{{define "styles"}} {{end}}
|
||||
{{define "content"}}
|
||||
Bad Request
|
||||
{{.}}
|
||||
{{end}}
|
@ -1,4 +0,0 @@
|
||||
{{template "base" .}}
|
||||
{{define "title"}}Page Not Found{{end}}
|
||||
{{define "styles"}} {{end}}
|
||||
{{define "content"}} Page Not Found {{end}}
|
@ -1,4 +0,0 @@
|
||||
{{template "base" .}}
|
||||
{{define "title"}}Server Error{{end}}
|
||||
{{define "styles"}}{{end}}
|
||||
{{define "content"}} {{ . }} {{end}}
|
@ -1,219 +0,0 @@
|
||||
{{template "base" .}}
|
||||
|
||||
{{define "title"}}{{.Name}} - Details{{end}}
|
||||
|
||||
{{define "styles"}}
|
||||
<style>
|
||||
.game-cover {
|
||||
max-height: 400px;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.screenshot-gallery {
|
||||
height: 600px;
|
||||
}
|
||||
|
||||
.swiper-slide img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
}
|
||||
|
||||
.info-label {
|
||||
font-weight: bold;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.download-card {
|
||||
transition: transform 0.2s;
|
||||
}
|
||||
|
||||
.download-card:hover {
|
||||
transform: translateY(-3px);
|
||||
box-shadow: 0 4px 15px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.tag {
|
||||
display: inline-block;
|
||||
padding: 0.25rem 0.5rem;
|
||||
margin: 0.25rem;
|
||||
background-color: #f8f9fa;
|
||||
border-radius: 0.25rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
</style>
|
||||
{{end}}
|
||||
|
||||
{{define "content"}}
|
||||
<!-- Game Details -->
|
||||
<div class="container py-4">
|
||||
<!-- Basic Info -->
|
||||
<div class="row mb-4">
|
||||
<div class="col-md-4">
|
||||
{{if .Cover}}
|
||||
<img src="{{.Cover}}" class="img-fluid rounded game-cover" alt="{{.Name}}" />
|
||||
{{else}}
|
||||
<div class="game-cover bg-secondary d-flex align-items-center justify-content-center rounded">
|
||||
<span class="text-white">No Image</span>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
<div class="col-md-8">
|
||||
<h1 class="mb-3">{{.Name}}</h1>
|
||||
|
||||
{{if .Aliases}}
|
||||
<div>
|
||||
<span class="info-label">Aliases:</span>
|
||||
{{range .Aliases}}
|
||||
<span class="tag">{{.}}</span>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<div>
|
||||
<span class="info-label">Developers:</span>
|
||||
{{range .Developers}}
|
||||
<span class="tag">{{.}}</span>
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<span class="info-label">Publishers:</span>
|
||||
{{range .Publishers}}
|
||||
<span class="tag">{{.}}</span>
|
||||
{{end}}
|
||||
</div>
|
||||
|
||||
{{if .Languages}}
|
||||
<div>
|
||||
<span class="info-label">Languages:</span>
|
||||
{{range .Languages}}
|
||||
<span class="tag">{{.}}</span>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}} {{if .Description}}
|
||||
<div>
|
||||
<p>{{.Description}}</p>
|
||||
</div>
|
||||
{{end}} {{if .SteamID}}
|
||||
<div>
|
||||
<a href="https://store.steampowered.com/app/{{.SteamID}}" target="_blank" class="btn btn-primary">
|
||||
Steam
|
||||
</a>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Screenshots -->
|
||||
{{if .Screenshots}}
|
||||
<div class="mb-4">
|
||||
<div class="swiper screenshot-gallery">
|
||||
<div class="swiper-wrapper">
|
||||
{{range .Screenshots}}
|
||||
<div class="swiper-slide">
|
||||
<img src="{{.}}" alt="screenshot" />
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
<div class="swiper-pagination"></div>
|
||||
<div class="swiper-button-next"></div>
|
||||
<div class="swiper-button-prev"></div>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
<!-- Download Links -->
|
||||
{{if .Games}}
|
||||
<div class="mb-4">
|
||||
<h3 class="mb-3">Download</h3>
|
||||
<div class="row g-4">
|
||||
{{range .Games}}
|
||||
<div class="col-md-6">
|
||||
<div class="card download-card">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">{{.RawName}}</h5>
|
||||
{{if .Size}}
|
||||
<div class="card-text">
|
||||
<small class="text-muted">Size: {{.Size}}</small>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if .Author}}
|
||||
<div class="card-text">
|
||||
<small class="text-muted">Source: {{.Author}}</small>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if .Platform}}
|
||||
<div class="card-text">
|
||||
<small class="text-muted">Platform: {{.Platform}}</small>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if .Password}}
|
||||
<div class="card-text">
|
||||
<small class="text-muted">Unzip password: <code>{{.Password}}</code></small>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if .UpdatedAt}}
|
||||
<div class="card-text">
|
||||
<small class="text-muted">Updated: {{.UpdatedAt}}</small>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="mt-2 d-flex justify-content-between align-items-center">
|
||||
<div class="input-group" style="max-width: 300px;">
|
||||
<input type="text" class="form-control form-control-sm" value="{{.Download}}" readonly>
|
||||
<button class="btn btn-outline-secondary btn-sm" type="button"
|
||||
onclick="copyToClipboard(this.previousElementSibling)">
|
||||
Copy
|
||||
</button>
|
||||
</div>
|
||||
<a href="{{.Url}}" class="btn btn-outline-primary" target="_blank">Source Page</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
|
||||
{{define "scripts"}}
|
||||
<script src="https://cdn.jsdelivr.net/npm/swiper@8/swiper-bundle.min.js"></script>
|
||||
<script>
|
||||
const swiper = new Swiper(".screenshot-gallery", {
|
||||
slidesPerView: 1,
|
||||
spaceBetween: 30,
|
||||
loop: true,
|
||||
pagination: {
|
||||
el: ".swiper-pagination",
|
||||
clickable: true,
|
||||
},
|
||||
navigation: {
|
||||
nextEl: ".swiper-button-next",
|
||||
prevEl: ".swiper-button-prev",
|
||||
},
|
||||
});
|
||||
|
||||
function copyToClipboard(input) {
|
||||
input.select();
|
||||
input.setSelectionRange(0, 99999);
|
||||
|
||||
navigator.clipboard.writeText(input.value).then(() => {
|
||||
const button = input.nextElementSibling;
|
||||
const originalText = button.textContent;
|
||||
button.textContent = 'Copied';
|
||||
button.classList.remove('btn-outline-secondary');
|
||||
button.classList.add('btn-success');
|
||||
|
||||
setTimeout(() => {
|
||||
button.textContent = originalText;
|
||||
button.classList.remove('btn-success');
|
||||
button.classList.add('btn-outline-secondary');
|
||||
}, 1500);
|
||||
}).catch(err => {
|
||||
console.error('Failed to copy:', err);
|
||||
});
|
||||
}
|
||||
</script>
|
||||
{{end}}
|
@ -1,137 +0,0 @@
|
||||
{{template "base" .}}
|
||||
|
||||
{{define "title"}}GameDB{{end}}
|
||||
|
||||
{{define "styles"}}
|
||||
<style>
|
||||
.game-card {
|
||||
height: 100%;
|
||||
transition: transform 0.2s;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.game-card:hover {
|
||||
transform: translateY(-5px);
|
||||
box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
.game-cover {
|
||||
height: 200px;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.game-description {
|
||||
display: -webkit-box;
|
||||
line-clamp: 3;
|
||||
-webkit-line-clamp: 3;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
@media (min-width: 992px) {
|
||||
.col-lg-5-item {
|
||||
flex: 0 0 20%;
|
||||
max-width: 20%;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
{{end}}
|
||||
|
||||
{{define "content"}}
|
||||
<!-- Games Grid -->
|
||||
<div class="container py-4">
|
||||
<h2 class="mb-4">Month Top</h2>
|
||||
<div class="row g-4">
|
||||
{{range .MonthTop}}
|
||||
<div class="col-12 col-sm-6 col-md-4 col-lg-5-item">
|
||||
<a href="/game/{{.ID.Hex}}" class="card game-card">
|
||||
{{if .Cover}}
|
||||
<img src="{{.Cover}}" class="card-img-top game-cover" alt="{{.Name}}" />
|
||||
{{else}}
|
||||
<div class="card-img-top game-cover bg-secondary d-flex align-items-center justify-content-center">
|
||||
<span class="text-white">No Image</span>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">{{.Name}}</h5>
|
||||
<p class="card-text game-description">{{.Description}}</p>
|
||||
{{if .Publishers}}
|
||||
<div class="publishers mb-2">
|
||||
<small class="text-muted">Publishers:</small>
|
||||
{{range $index, $publisher := .Publishers}} {{if $index}}, {{end}}
|
||||
<small class="text-muted">{{$publisher}}</small>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container py-4">
|
||||
<h2 class="mb-4">Most Played</h2>
|
||||
<div class="row g-4">
|
||||
{{range .MostPlayed}}
|
||||
<div class="col-12 col-sm-6 col-md-4 col-lg-5-item">
|
||||
<a href="/game/{{.ID.Hex}}" class="card game-card">
|
||||
{{if .Cover}}
|
||||
<img src="{{.Cover}}" class="card-img-top game-cover" alt="{{.Name}}" />
|
||||
{{else}}
|
||||
<div class="card-img-top game-cover bg-secondary d-flex align-items-center justify-content-center">
|
||||
<span class="text-white">No Image</span>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">{{.Name}}</h5>
|
||||
<p class="card-text game-description">{{.Description}}</p>
|
||||
{{if .Publishers}}
|
||||
<div class="publishers mb-2">
|
||||
<small class="text-muted">Publishers:</small>
|
||||
{{range $index, $publisher := .Publishers}} {{if $index}}, {{end}}
|
||||
<small class="text-muted">{{$publisher}}</small>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container py-4">
|
||||
<h2 class="mb-4">Best of the Year</h2>
|
||||
<div class="row g-4">
|
||||
{{range .BestOfTheYear}}
|
||||
<div class="col-12 col-sm-6 col-md-4 col-lg-5-item">
|
||||
<a href="/game/{{.ID.Hex}}" class="card game-card">
|
||||
{{if .Cover}}
|
||||
<img src="{{.Cover}}" class="card-img-top game-cover" alt="{{.Name}}" />
|
||||
{{else}}
|
||||
<div class="card-img-top game-cover bg-secondary d-flex align-items-center justify-content-center">
|
||||
<span class="text-white">No Image</span>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">{{.Name}}</h5>
|
||||
<p class="card-text game-description">{{.Description}}</p>
|
||||
{{if .Publishers}}
|
||||
<div class="publishers mb-2">
|
||||
<small class="text-muted">Publishers:</small>
|
||||
{{range $index, $publisher := .Publishers}} {{if $index}}, {{end}}
|
||||
<small class="text-muted">{{$publisher}}</small>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</div>
|
||||
{{end}}
|
||||
{{define "scripts"}} {{end}}
|
@ -1,23 +0,0 @@
|
||||
{{define "base"}}
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN" data-bs-theme="dark">
|
||||
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>{{template "title" .}}</title>
|
||||
<link rel="icon" type="image/x-icon" href="/static/favicon.png">
|
||||
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet" />
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/swiper@8/swiper-bundle.min.css" />
|
||||
{{block "styles" .}}{{end}}
|
||||
</head>
|
||||
|
||||
<body>
|
||||
{{template "header" .}} {{block "content" .}}{{end}} {{template "footer" .}}
|
||||
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
|
||||
{{block "scripts" .}}{{end}}
|
||||
</body>
|
||||
|
||||
</html>
|
||||
{{end}}
|
@ -1,12 +0,0 @@
|
||||
{{define "footer"}}
|
||||
<!-- Footer -->
|
||||
<footer class="text-light py-4 mt-5">
|
||||
<div class="container">
|
||||
<div class="col text-center">
|
||||
<div><a href="https://git.nite07.com/nite/pcgamedb" target="_blank">Open Source</a> | <a
|
||||
href="/api/swagger/index.html">API Doc</a></div>
|
||||
<div>Made by <a href="https://www.nite07.com" target="_blank">Nite</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
{{end}}
|
@ -1,25 +0,0 @@
|
||||
{{define "header"}}
|
||||
<!-- Header -->
|
||||
<nav class="navbar navbar-expand-lg">
|
||||
<div class="container">
|
||||
<span>
|
||||
<a class="navbar-brand" href="/">GameDB</a>
|
||||
<a href="https://t.me/nitegame" target="_blank">Telegram</a>
|
||||
</span>
|
||||
<div class="navbar-collapse" id="navbarNav">
|
||||
<ul class="navbar-nav">
|
||||
<!-- <li class="nav-item"><a class="nav-link" href="https://www.nite07.com" target="_blank">Blog</a></li> -->
|
||||
</ul>
|
||||
|
||||
<div class="ms-auto">
|
||||
<form action="/search" method="GET" class="d-flex">
|
||||
<div class="input-group">
|
||||
<input type="text" class="form-control" name="key" placeholder="Input full english name of game" />
|
||||
<button class="btn btn-primary" type="submit">Search</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
{{end}}
|
@ -1,102 +0,0 @@
|
||||
{{template "base" .}}
|
||||
|
||||
{{define "title"}}{{.Key}} - Search{{end}}
|
||||
|
||||
{{define "styles"}}
|
||||
<style>
|
||||
.game-card {
|
||||
height: 100%;
|
||||
transition: transform 0.2s;
|
||||
text-decoration: none;
|
||||
color: inherit;
|
||||
}
|
||||
|
||||
.game-card:hover {
|
||||
transform: translateY(-5px);
|
||||
box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
|
||||
}
|
||||
|
||||
.game-cover {
|
||||
height: 200px;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.game-description {
|
||||
display: -webkit-box;
|
||||
line-clamp: 3;
|
||||
-webkit-line-clamp: 3;
|
||||
-webkit-box-orient: vertical;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
@media (min-width: 992px) {
|
||||
.col-lg-5-item {
|
||||
flex: 0 0 20%;
|
||||
max-width: 20%;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
{{end}}
|
||||
|
||||
{{define "content"}}
|
||||
<!-- page -->
|
||||
<div class="container py-4">
|
||||
<h2 class="mb-4">Search</h2>
|
||||
<small>{{.CurrentPage}}/{{.TotalPage}} pages found for "{{.Key}}"</small>
|
||||
</div>
|
||||
<!-- Games Grid -->
|
||||
<div class="container py-4">
|
||||
{{if .Games}}
|
||||
<div class="row g-4">
|
||||
{{range .Games}}
|
||||
<div class="col-12 col-sm-6 col-md-4 col-lg-5-item">
|
||||
<a href="/game/{{.ID.Hex}}" class="card game-card">
|
||||
{{if .Cover}}
|
||||
<img src="{{.Cover}}" class="card-img-top game-cover" alt="{{.Name}}" />
|
||||
{{else}}
|
||||
<div class="card-img-top game-cover bg-secondary d-flex align-items-center justify-content-center">
|
||||
<span class="text-white">No Image</span>
|
||||
</div>
|
||||
{{end}}
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">{{.Name}}</h5>
|
||||
<p class="card-text game-description">{{.Description}}</p>
|
||||
{{if .Publishers}}
|
||||
<div class="publishers mb-2">
|
||||
<small class="text-muted">Publishers:</small>
|
||||
{{range $index, $publisher := .Publishers}} {{if $index}}, {{end}}
|
||||
<small class="text-muted">{{$publisher}}</small>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
{{else}}
|
||||
<div class="text-center py-5">
|
||||
<h4 class="text-muted">No results found for "{{.Key}}"</h4>
|
||||
</div>
|
||||
{{end}}
|
||||
</div>
|
||||
<!-- Pagination -->
|
||||
<div class="container py-4">
|
||||
<nav aria-label="Page navigation">
|
||||
<ul class="pagination justify-content-center">
|
||||
{{if .PrevPage}}
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="/search?key={{.Key}}&page={{.PrevPage}}">Previous</a>
|
||||
</li>
|
||||
{{end}}
|
||||
{{if .NextPage}}
|
||||
<li class="page-item">
|
||||
<a class="page-link" href="/search?key={{.Key}}&page={{.NextPage}}">Next</a>
|
||||
</li>
|
||||
{{end}}
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
{{end}}
|
||||
{{define "scripts"}} {{end}}
|
@ -28,9 +28,9 @@ func Clean(logger *zap.Logger) {
|
||||
for _, id := range ids {
|
||||
logger.Info("Cleaned game info with empty game ids", zap.Any("game_id", id))
|
||||
}
|
||||
err = db.MergeGameInfosWithSameIGDBID()
|
||||
err = db.MergeSameNameGameInfos()
|
||||
if err != nil {
|
||||
logger.Error("Failed to merge game infos with same igdb id", zap.Error(err))
|
||||
logger.Error("Failed to merge same name game infos", zap.Error(err))
|
||||
}
|
||||
logger.Info("Cleaning task completed")
|
||||
}
|
||||
|
@ -1,6 +1,7 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"pcgamedb/config"
|
||||
@ -9,7 +10,6 @@ import (
|
||||
"pcgamedb/model"
|
||||
"pcgamedb/utils"
|
||||
|
||||
"go.mongodb.org/mongo-driver/bson/primitive"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
@ -46,19 +46,8 @@ func Crawl(logger *zap.Logger) {
|
||||
Clean(logger)
|
||||
|
||||
// trigger webhooks
|
||||
var ids []primitive.ObjectID
|
||||
infos := []*model.GameInfo{}
|
||||
for _, game := range games {
|
||||
ids = append(ids, game.ID)
|
||||
}
|
||||
|
||||
var infos []*model.GameInfo
|
||||
if len(ids) != 0 {
|
||||
items, err := db.GetGameItemsByIDs(ids)
|
||||
if err != nil {
|
||||
logger.Error("Failed to get game items", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for _, game := range items {
|
||||
info, err := db.GetGameInfoByGameItemID(game.ID)
|
||||
if err != nil {
|
||||
logger.Error("Failed to get game info", zap.Error(err))
|
||||
@ -67,8 +56,6 @@ func Crawl(logger *zap.Logger) {
|
||||
info.Games = append(info.Games, game)
|
||||
infos = append(infos, info)
|
||||
}
|
||||
}
|
||||
|
||||
for _, u := range config.Config.Webhooks.CrawlTask {
|
||||
_, err := url.Parse(u)
|
||||
if err != nil {
|
||||
@ -76,7 +63,14 @@ func Crawl(logger *zap.Logger) {
|
||||
continue
|
||||
}
|
||||
logger.Info("webhook triggered", zap.String("task", "crawl"), zap.String("url", u))
|
||||
_, err = utils.Request().SetHeader("Content-Type", "application/json").SetBody(infos).Post(u)
|
||||
_, err = utils.Fetch(utils.FetchConfig{
|
||||
Url: u,
|
||||
Method: http.MethodPost,
|
||||
Headers: map[string]string{
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
Data: infos,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("Failed to trigger webhook", zap.String("task", "crawl"), zap.String("url", u), zap.Error(err))
|
||||
}
|
||||
|
@ -1,27 +0,0 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"pcgamedb/crawler"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
func UpdateOutdatedGameInfos(logger *zap.Logger) {
|
||||
channel, err := crawler.UpdateGameInfo(10)
|
||||
count := 0
|
||||
if err != nil {
|
||||
logger.Error("Failed to update game info", zap.Error(err))
|
||||
return
|
||||
}
|
||||
for info := range channel {
|
||||
logger.Info("Updated game info",
|
||||
zap.String("id", info.ID.String()),
|
||||
zap.String("name", info.Name),
|
||||
)
|
||||
count++
|
||||
if count == 10 {
|
||||
break
|
||||
}
|
||||
}
|
||||
logger.Info("Updated game info count", zap.Int("count", count))
|
||||
}
|
147
utils/cf-clearance-scraper.go
Normal file
147
utils/cf-clearance-scraper.go
Normal file
@ -0,0 +1,147 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// https://github.com/ZFC-Digital/cf-clearance-scraper
|
||||
|
||||
type ccsRequest struct {
|
||||
Url string `json:"url"`
|
||||
Mode string `json:"mode"`
|
||||
SiteKey string `json:"siteKey"`
|
||||
}
|
||||
|
||||
type WAFSession struct {
|
||||
Cookies []struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
Expires float64 `json:"expires"`
|
||||
Size int `json:"size"`
|
||||
HTTPOnly bool `json:"httpOnly"`
|
||||
Secure bool `json:"secure"`
|
||||
Session bool `json:"session"`
|
||||
SameSite string `json:"sameSite"`
|
||||
Priority string `json:"priority"`
|
||||
SameParty bool `json:"sameParty"`
|
||||
SourceScheme string `json:"sourceScheme"`
|
||||
PartitionKey string `json:"partitionKey"`
|
||||
} `json:"cookies"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
|
||||
func CCSWAFSession(ccsUrl string, requestUrl string) (*WAFSession, error) {
|
||||
data := ccsRequest{
|
||||
Url: requestUrl,
|
||||
Mode: "waf-session",
|
||||
}
|
||||
resp, err := Fetch(FetchConfig{
|
||||
Url: ccsUrl,
|
||||
Method: "POST",
|
||||
Data: data,
|
||||
Timeout: 60 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var response WAFSession
|
||||
err = json.Unmarshal(resp.Data, &response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if response.Code != 200 {
|
||||
return nil, errors.New("Failed to get WAF session")
|
||||
}
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
func CCSSource(ccsUrl string, requestUrl string) (string, error) {
|
||||
data := ccsRequest{
|
||||
Url: requestUrl,
|
||||
Mode: "source",
|
||||
}
|
||||
resp, err := Fetch(FetchConfig{
|
||||
Url: ccsUrl,
|
||||
Method: "POST",
|
||||
Data: data,
|
||||
Timeout: 60 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
type response struct {
|
||||
Source string `json:"source"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
var ccsResp response
|
||||
err = json.Unmarshal(resp.Data, &ccsResp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ccsResp.Code != 200 {
|
||||
return "", errors.New("Failed to get source")
|
||||
}
|
||||
return ccsResp.Source, nil
|
||||
}
|
||||
|
||||
func CCSTurnstileToken(ccsUrl string, requestUrl string, siteKey string) (string, error) {
|
||||
data := ccsRequest{
|
||||
Url: requestUrl,
|
||||
Mode: "turnstile-min",
|
||||
SiteKey: siteKey,
|
||||
}
|
||||
resp, err := Fetch(FetchConfig{
|
||||
Url: ccsUrl,
|
||||
Method: "POST",
|
||||
Data: data,
|
||||
Timeout: 60 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var ccsResp struct {
|
||||
Token string `json:"token"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
err = json.Unmarshal(resp.Data, &ccsResp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ccsResp.Code != 200 {
|
||||
return "", errors.New("Failed to get source")
|
||||
}
|
||||
return ccsResp.Token, nil
|
||||
}
|
||||
|
||||
func CCSTurnstileMaxToken(ccsUrl string, requestUrl string) (string, error) {
|
||||
data := ccsRequest{
|
||||
Url: requestUrl,
|
||||
Mode: "turnstile-max",
|
||||
}
|
||||
resp, err := Fetch(FetchConfig{
|
||||
Url: ccsUrl,
|
||||
Method: "POST",
|
||||
Data: data,
|
||||
Timeout: 60 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var ccsResp struct {
|
||||
Token string `json:"token"`
|
||||
Code int `json:"code"`
|
||||
}
|
||||
err = json.Unmarshal(resp.Data, &ccsResp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if ccsResp.Code != 200 {
|
||||
return "", errors.New("Failed to get source")
|
||||
}
|
||||
return ccsResp.Token, nil
|
||||
}
|
@ -1,16 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
|
||||
"golang.org/x/text/encoding/htmlindex"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
func Windows1251ToUTF8(b []byte) []byte {
|
||||
decoder, _ := htmlindex.Get("windows-1251")
|
||||
reader := transform.NewReader(bytes.NewReader(b), decoder.NewDecoder().Transformer)
|
||||
body, _ := io.ReadAll(reader)
|
||||
return body
|
||||
}
|
222
utils/fetch.go
Normal file
222
utils/fetch.go
Normal file
@ -0,0 +1,222 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html/charset"
|
||||
)
|
||||
|
||||
const userAgent string = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
|
||||
|
||||
type FetchConfig struct {
|
||||
Method string
|
||||
Url string
|
||||
Data interface{}
|
||||
RetryTimes int
|
||||
Headers map[string]string
|
||||
Cookies map[string]string
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
type FetchResponse struct {
|
||||
StatusCode int
|
||||
Data []byte
|
||||
Header http.Header
|
||||
Cookie []*http.Cookie
|
||||
}
|
||||
|
||||
func Fetch(cfg FetchConfig) (*FetchResponse, error) {
|
||||
var req *http.Request
|
||||
var resp *http.Response
|
||||
var backoff time.Duration = 1
|
||||
var reqBody io.Reader = nil
|
||||
var err error
|
||||
|
||||
if cfg.RetryTimes == 0 {
|
||||
cfg.RetryTimes = 3
|
||||
}
|
||||
if cfg.Method == "" {
|
||||
cfg.Method = "GET"
|
||||
}
|
||||
if cfg.Timeout == 0 {
|
||||
cfg.Timeout = 10 * time.Second
|
||||
}
|
||||
|
||||
if cfg.Data != nil && (cfg.Method == "POST" || cfg.Method == "PUT") {
|
||||
if cfg.Headers == nil {
|
||||
cfg.Headers = map[string]string{}
|
||||
}
|
||||
newHeaders := make(map[string]string)
|
||||
for k, v := range cfg.Headers {
|
||||
newHeaders[strings.ToLower(k)] = v
|
||||
}
|
||||
cfg.Headers = newHeaders
|
||||
if _, exist := cfg.Headers["content-type"]; !exist {
|
||||
cfg.Headers["content-type"] = "application/json"
|
||||
}
|
||||
v := cfg.Headers["content-type"]
|
||||
if v == "application/x-www-form-urlencoded" {
|
||||
switch data := cfg.Data.(type) {
|
||||
case map[string]string:
|
||||
params := url.Values{}
|
||||
for k, v := range data {
|
||||
params.Set(k, v)
|
||||
}
|
||||
reqBody = strings.NewReader(params.Encode())
|
||||
case string:
|
||||
reqBody = strings.NewReader(data)
|
||||
case url.Values:
|
||||
reqBody = strings.NewReader(data.Encode())
|
||||
default:
|
||||
return nil, errors.New("unsupported data type")
|
||||
}
|
||||
} else if v == "application/json" {
|
||||
switch data := cfg.Data.(type) {
|
||||
case []byte:
|
||||
reqBody = bytes.NewReader(data)
|
||||
case string:
|
||||
reqBody = strings.NewReader(data)
|
||||
case interface{}:
|
||||
jsonData, err := json.Marshal(cfg.Data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqBody = bytes.NewReader(jsonData)
|
||||
default:
|
||||
return nil, errors.New("unsupported data type")
|
||||
}
|
||||
} else {
|
||||
reqBody = strings.NewReader(cfg.Data.(string))
|
||||
}
|
||||
}
|
||||
|
||||
var bodyBuffer *bytes.Buffer
|
||||
if reqBody != nil {
|
||||
bodyBuffer = new(bytes.Buffer)
|
||||
_, err = io.Copy(bodyBuffer, reqBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
for retryTime := 0; retryTime <= cfg.RetryTimes; retryTime++ {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
var currentReqBody io.Reader
|
||||
if bodyBuffer != nil {
|
||||
currentReqBody = bytes.NewReader(bodyBuffer.Bytes())
|
||||
}
|
||||
|
||||
req, err = http.NewRequestWithContext(ctx, cfg.Method, cfg.Url, currentReqBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v, exist := cfg.Headers["user-agent"]; exist {
|
||||
if v != "" {
|
||||
req.Header.Set("user-agent", v)
|
||||
}
|
||||
} else {
|
||||
req.Header.Set("user-agent", userAgent)
|
||||
}
|
||||
for k, v := range cfg.Headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
for k, v := range cfg.Cookies {
|
||||
req.AddCookie(&http.Cookie{Name: k, Value: v})
|
||||
}
|
||||
resp, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
if isRetryableError(err) {
|
||||
err = errors.New("request error: " + err.Error())
|
||||
time.Sleep(backoff * time.Second)
|
||||
backoff *= 2
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if resp == nil {
|
||||
return nil, errors.New("response is nil")
|
||||
}
|
||||
|
||||
if isRetryableStatusCode(resp.StatusCode) {
|
||||
err = errors.New("response status code: " + resp.Status)
|
||||
time.Sleep(backoff * time.Second)
|
||||
backoff *= 2
|
||||
continue
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("content-type")
|
||||
var reader io.Reader
|
||||
if strings.Contains(contentType, "charset=") {
|
||||
reader, err = charset.NewReader(resp.Body, contentType)
|
||||
} else {
|
||||
reader = resp.Body
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataBytes, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
res := &FetchResponse{
|
||||
StatusCode: resp.StatusCode,
|
||||
Header: resp.Header,
|
||||
Cookie: resp.Cookies(),
|
||||
Data: dataBytes,
|
||||
}
|
||||
|
||||
return res, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func isRetryableStatusCode(statusCode int) bool {
|
||||
switch statusCode {
|
||||
case http.StatusInternalServerError,
|
||||
http.StatusBadGateway,
|
||||
http.StatusServiceUnavailable,
|
||||
http.StatusGatewayTimeout,
|
||||
http.StatusTooManyRequests:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isRetryableError(err error) bool {
|
||||
if err != nil {
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) && netErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func FetchWithWAFSession(cfg FetchConfig, session *WAFSession) (*FetchResponse, error) {
|
||||
if cfg.Cookies == nil {
|
||||
cfg.Cookies = map[string]string{}
|
||||
}
|
||||
for _, cookie := range session.Cookies {
|
||||
cfg.Cookies[cookie.Name] = cookie.Value
|
||||
}
|
||||
if cfg.Headers == nil {
|
||||
cfg.Headers = map[string]string{}
|
||||
}
|
||||
for k, v := range session.Headers {
|
||||
cfg.Headers[k] = v
|
||||
}
|
||||
return Fetch(cfg)
|
||||
}
|
@ -8,13 +8,18 @@ import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
func SolveKeepLinks(URL string) (string, error) {
|
||||
id := URL[strings.LastIndex(URL, "/")+1:]
|
||||
resp, err := Request().SetHeader("Cookie", fmt.Sprintf("flag[%s]", id)+"=1").Get(URL)
|
||||
func SolveKeepLinks(url string) (string, error) {
|
||||
id := url[strings.LastIndex(url, "/")+1:]
|
||||
resp, err := Fetch(FetchConfig{
|
||||
Url: url,
|
||||
Cookies: map[string]string{
|
||||
fmt.Sprintf("flag[%s]", id): "1",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
|
||||
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Data))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
@ -15,7 +15,7 @@ func ConvertTorrentToMagnet(torrent []byte) (string, string, error) {
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
var size = uint64(info.Length)
|
||||
var size uint64 = uint64(info.Length)
|
||||
if size == 0 {
|
||||
for _, file := range info.Files {
|
||||
size += uint64(file.Length)
|
||||
|
@ -39,7 +39,7 @@ func MegaDownload(url string, path string) (string, []string, error) {
|
||||
pathRegex := regexp.MustCompile(`(?i)Download finished: (.*)`)
|
||||
pathRegexRes := pathRegex.FindAllStringSubmatch(out.String(), -1)
|
||||
if len(pathRegexRes) == 0 {
|
||||
return "", nil, errors.New("mega download failed")
|
||||
return "", nil, errors.New("Mega download failed")
|
||||
}
|
||||
pathRegexRes[0][1] = strings.TrimSpace(pathRegexRes[0][1])
|
||||
res, err := walkDir(pathRegexRes[0][1])
|
||||
@ -54,7 +54,7 @@ func walkDir(path string) ([]string, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res []string
|
||||
res := []string{}
|
||||
for _, file := range files {
|
||||
if file.IsDir() {
|
||||
subFiles, err := walkDir(filepath.Join(path, file.Name()))
|
||||
|
@ -31,17 +31,22 @@ func padStart(s string, minLength int, padRune rune) string {
|
||||
return padding + s
|
||||
}
|
||||
|
||||
func DecryptPrivateBin(URL string, password string) (string, error) {
|
||||
if !strings.Contains(URL, "#") {
|
||||
return "", errors.New("missing Decrypt Key")
|
||||
func DecryptPrivateBin(url string, password string) (string, error) {
|
||||
if !strings.Contains(url, "#") {
|
||||
return "", errors.New("Missing Decrypt Key")
|
||||
}
|
||||
key := strings.Split(URL, "#")[1]
|
||||
resp, err := Request().SetHeader("Accept", "application/json, text/javascript, */*; q=0.01").Get(URL)
|
||||
key := strings.Split(url, "#")[1]
|
||||
resp, err := Fetch(FetchConfig{
|
||||
Url: url,
|
||||
Headers: map[string]string{
|
||||
"Accept": "application/json, text/javascript, */*; q=0.01",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
data := PrivateBinData{}
|
||||
err = json.Unmarshal(resp.Body(), &data)
|
||||
err = json.Unmarshal(resp.Data, &data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
@ -1,18 +0,0 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
var client *resty.Client
|
||||
|
||||
func init() {
|
||||
client = resty.New()
|
||||
client.SetRetryCount(3).SetRetryWaitTime(1 * time.Second)
|
||||
}
|
||||
|
||||
func Request() *resty.Request {
|
||||
return client.R().SetHeader("Accept-Charset", "utf-8").SetHeader("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0")
|
||||
}
|
@ -1,11 +1,21 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
import "strings"
|
||||
|
||||
func levenshteinDistance(str1, str2 string) int {
|
||||
func min(a, b, c int) int {
|
||||
if a < b {
|
||||
if a < c {
|
||||
return a
|
||||
}
|
||||
return c
|
||||
}
|
||||
if b < c {
|
||||
return b
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func LevenshteinDistance(str1, str2 string) int {
|
||||
str1 = strings.ToLower(str1)
|
||||
str2 = strings.ToLower(str2)
|
||||
s1, s2 := []rune(str1), []rune(str2)
|
||||
@ -42,29 +52,21 @@ func levenshteinDistance(str1, str2 string) int {
|
||||
return d[lenS1][lenS2]
|
||||
}
|
||||
|
||||
var noneAlphaNumericRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
|
||||
|
||||
func removeSpacesAndNoneAlphaNumeric(str string) string {
|
||||
str = strings.ToLower(str)
|
||||
str = noneAlphaNumericRegex.ReplaceAllString(str, "")
|
||||
return str
|
||||
}
|
||||
|
||||
func Similarity(str1, str2 string) float64 {
|
||||
str1 = removeSpacesAndNoneAlphaNumeric(str1)
|
||||
str2 = removeSpacesAndNoneAlphaNumeric(str2)
|
||||
distance := levenshteinDistance(str1, str2)
|
||||
str1 = strings.ReplaceAll(str1, " ", "")
|
||||
str2 = strings.ReplaceAll(str2, " ", "")
|
||||
distance := LevenshteinDistance(str1, str2)
|
||||
maxLength := len(str1)
|
||||
if len(str2) > maxLength {
|
||||
maxLength = len(str2)
|
||||
}
|
||||
|
||||
adjustedLength := float64(maxLength + (len(str1)+len(str2))/2)
|
||||
djustedMaxLength := maxLength + (len(str1) + len(str2))
|
||||
|
||||
if maxLength == 0 {
|
||||
return 1.0
|
||||
}
|
||||
|
||||
similarity := 1.0 - float64(distance)/float64(adjustedLength)
|
||||
similarity := 1.0 - float64(distance)/float64(djustedMaxLength)
|
||||
return similarity
|
||||
}
|
||||
|
@ -17,18 +17,10 @@ func SizeToBytes(size string) (uint64, error) {
|
||||
"TB": 1024 * 1024 * 1024 * 1024,
|
||||
}
|
||||
|
||||
unitsSlice := []string{
|
||||
"TB",
|
||||
"GB",
|
||||
"MB",
|
||||
"KB",
|
||||
"B",
|
||||
}
|
||||
|
||||
var unit string
|
||||
var value float64
|
||||
|
||||
for _, u := range unitsSlice {
|
||||
for u := range units {
|
||||
if strings.HasSuffix(size, u) {
|
||||
unit = u
|
||||
numStr := strings.TrimSuffix(size, u)
|
||||
@ -65,6 +57,6 @@ func BytesToSize(size uint64) string {
|
||||
case size >= KB:
|
||||
return fmt.Sprintf("%.1f KB", float64(size)/float64(KB))
|
||||
default:
|
||||
return fmt.Sprintf("%d B", size)
|
||||
return fmt.Sprintf("%d Bytes", size)
|
||||
}
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user