From cd9b7412b8e233d04f24d289226c3391343ff3b3 Mon Sep 17 00:00:00 2001 From: nite Date: Sat, 28 Dec 2024 00:10:06 +0800 Subject: [PATCH] u --- cmd/server.go | 2 +- crawler/1337x.go | 6 +- crawler/chovka.go | 6 +- crawler/fitgirl.go | 6 +- crawler/freegog.go | 7 +- crawler/game.go | 27 +---- crawler/goggames.go | 90 +++++---------- crawler/igdb.go | 112 ++++-------------- crawler/nxbrew.go | 13 +++ crawler/onlinefix.go | 4 +- crawler/rutracker.go | 19 +-- crawler/steam.go | 20 +--- crawler/steam250.go | 25 +--- crawler/steamrip.go | 41 ++----- crawler/xatab.go | 7 +- db/game.go | 12 +- model/game.go | 24 ++-- server/templates/game.html | 180 +++++++++++++++++------------ server/templates/layouts/base.html | 2 +- utils/mgnet.go | 81 +++++++++++++ utils/ouo.go | 2 +- 21 files changed, 324 insertions(+), 362 deletions(-) create mode 100644 crawler/nxbrew.go create mode 100644 utils/mgnet.go diff --git a/cmd/server.go b/cmd/server.go index d275cea..f66f047 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -23,7 +23,7 @@ var serverCmdCfg serverCommandConfig func init() { serverCmd.Flags().StringVarP(&serverCmdCfg.Port, "port", "p", "8080", "server port") - serverCmd.Flags().BoolVarP(&serverCmdCfg.AutoCrawl, "auto-crawl", "c", true, "enable auto crawl") + serverCmd.Flags().BoolVarP(&serverCmdCfg.AutoCrawl, "auto-crawl", "c", false, "enable auto crawl") RootCmd.AddCommand(serverCmd) } diff --git a/crawler/1337x.go b/crawler/1337x.go index c6f0fda..a652ee7 100644 --- a/crawler/1337x.go +++ b/crawler/1337x.go @@ -98,7 +98,7 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameItem, error) { } func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL)) + c.logger.Info("Crawling game", zap.String("URL", URL)) resp, err := utils.Request().Get(URL) if err != nil { c.logger.Error("Failed to fetch URL", zap.String("URL", URL), zap.Error(err)) @@ -136,7 +136,9 @@ func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { item.RawName = strings.Replace(item.RawName, "Download ", "", 1) item.RawName = strings.TrimSpace(strings.Replace(item.RawName, "Torrent | 1337x", " ", 1)) item.Name = c.formatter(item.RawName) - item.DownloadLinks = []string{magnetRegexRes[0]} + item.Downloads = map[string]string{ + "magnet": magnetRegexRes[0], + } item.Author = strings.Replace(c.source, "-torrents", "", -1) item.Platform = c.platform diff --git a/crawler/chovka.go b/crawler/chovka.go index e93ea7f..4a66f3c 100644 --- a/crawler/chovka.go +++ b/crawler/chovka.go @@ -31,7 +31,7 @@ func (c *ChovkaCrawler) Name() string { } func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL)) + c.logger.Info("Crawling game", zap.String("URL", URL)) resp, err := utils.Request().Get(URL) if err != nil { c.logger.Error("Failed to fetch URL", zap.String("URL", URL), zap.Error(err)) @@ -75,7 +75,9 @@ func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { } item.Size = size - item.DownloadLinks = []string{magnet} + item.Downloads = map[string]string{ + "magnet": magnet, + } c.logger.Info("Successfully crawled URL", zap.String("URL", URL)) return item, nil } diff --git a/crawler/fitgirl.go b/crawler/fitgirl.go index 2698020..c2cc090 100644 --- a/crawler/fitgirl.go +++ b/crawler/fitgirl.go @@ -32,7 +32,7 @@ func (c *FitGirlCrawler) Name() string { } func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL)) + c.logger.Info("Crawling game", zap.String("URL", URL)) resp, err := utils.Request().Get(URL) if err != nil { c.logger.Error("Failed to fetch URL", zap.String("URL", URL), zap.Error(err)) @@ -82,7 +82,9 @@ func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { item.Url = URL item.Size = size item.Author = "FitGirl" - item.DownloadLinks = []string{magnet} + item.Downloads = map[string]string{ + "magnet": magnet, + } item.Platform = "windows" c.logger.Info("Successfully crawled URL", zap.String("URL", URL)) diff --git a/crawler/freegog.go b/crawler/freegog.go index c0162e9..07fdce3 100644 --- a/crawler/freegog.go +++ b/crawler/freegog.go @@ -34,7 +34,6 @@ func NewFreeGOGCrawler(cfClearanceUrl string, logger *zap.Logger) *FreeGOGCrawle } func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) { - c.logger.Info("Fetching session for FreeGOGCrawler") cacheKey := "freegog_waf_session" var session ccs.Session if val, exist := cache.Get(cacheKey); exist { @@ -134,7 +133,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) { } func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL)) + c.logger.Info("Crawling game", zap.String("URL", URL)) session, err := c.getSession() if err != nil { c.logger.Error("Failed to get session", zap.Error(err)) @@ -186,7 +185,9 @@ func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { c.logger.Error("Failed to decode magnet link", zap.String("URL", URL), zap.Error(err)) return nil, fmt.Errorf("failed to decode magnet link on page %s: %w", URL, err) } - item.DownloadLinks = []string{string(magnet)} + item.Downloads = map[string]string{ + "magnet": string(magnet), + } } else { c.logger.Error("Failed to find magnet link", zap.String("URL", URL)) return nil, fmt.Errorf("failed to find magnet link on page %s", URL) diff --git a/crawler/game.go b/crawler/game.go index a429a64..5dfc411 100644 --- a/crawler/game.go +++ b/crawler/game.go @@ -2,6 +2,7 @@ package crawler import ( "errors" + "fmt" "regexp" "strings" "time" @@ -10,8 +11,6 @@ import ( "game-crawler/model" "game-crawler/utils" - "go.uber.org/zap" - "go.mongodb.org/mongo-driver/bson/primitive" "go.mongodb.org/mongo-driver/mongo" ) @@ -45,8 +44,6 @@ func OrganizeGameItem(game *model.GameItem) error { steamID, err := GetSteamIDByIGDBID(item.IGDBID) if err == nil { item.SteamID = steamID - } else { - return err } } @@ -123,12 +120,9 @@ func FormatName(name string) string { // SupplementPlatformIDToGameInfo supplements missing platform IDs (SteamID or IGDBID) for all game info entries. func SupplementPlatformIDToGameInfo() error { - logger := zap.L() - logger.Info("Starting to supplement missing platform IDs") infos, err := db.GetAllGameInfos() if err != nil { - logger.Error("Failed to fetch game infos", zap.Error(err)) - return err + return fmt.Errorf("failed to fetch game infos: %w", err) } for _, info := range infos { @@ -141,8 +135,6 @@ func SupplementPlatformIDToGameInfo() error { if err == nil { info.SteamID = steamID changed = true - } else { - logger.Warn("Failed to get SteamID from IGDB", zap.Int("IGDBID", info.IGDBID), zap.Error(err)) } } @@ -153,18 +145,13 @@ func SupplementPlatformIDToGameInfo() error { if err == nil { info.IGDBID = igdbID changed = true - } else { - logger.Warn("Failed to get IGDBID from SteamID", zap.Int("SteamID", info.SteamID), zap.Error(err)) } } if changed { - logger.Info("Supplemented platform IDs", zap.String("Name", info.Name), zap.Int("IGDBID", info.IGDBID), zap.Int("SteamID", info.SteamID)) if err := db.SaveGameInfo(info); err != nil { - logger.Error("Failed to save updated game info", zap.String("Name", info.Name), zap.Error(err)) + return fmt.Errorf("failed to save game info: %w", err) } - } else { - logger.Info("No changes needed", zap.String("Name", info.Name), zap.Int("IGDBID", info.IGDBID), zap.Int("SteamID", info.SteamID)) } } return nil @@ -172,12 +159,9 @@ func SupplementPlatformIDToGameInfo() error { // UpdateGameInfo updates outdated game info entries and returns a channel to monitor updates. func UpdateGameInfo(num int) (chan *model.GameInfo, error) { - logger := zap.L() - logger.Info("Starting to update outdated game info", zap.Int("Num", num)) infos, err := db.GetOutdatedGameInfos(num) if err != nil { - logger.Error("Failed to fetch outdated game infos", zap.Error(err)) - return nil, err + return nil, fmt.Errorf("failed to fetch outdated game infos: %w", err) } updateChan := make(chan *model.GameInfo) @@ -188,18 +172,15 @@ func UpdateGameInfo(num int) (chan *model.GameInfo, error) { if info.IGDBID != 0 { newInfo, err := GenerateIGDBGameInfo(info.IGDBID) if err != nil { - logger.Warn("Failed to generate IGDB game info", zap.Int("IGDBID", info.IGDBID), zap.Error(err)) continue } db.MergeGameInfo(info, newInfo) if err := db.SaveGameInfo(newInfo); err != nil { - logger.Error("Failed to save updated game info", zap.String("Name", newInfo.Name), zap.Error(err)) continue } updateChan <- newInfo - logger.Info("Updated game info", zap.String("Name", newInfo.Name), zap.Int("IGDBID", newInfo.IGDBID)) } } }() diff --git a/crawler/goggames.go b/crawler/goggames.go index 19f5161..f4c9d41 100644 --- a/crawler/goggames.go +++ b/crawler/goggames.go @@ -45,45 +45,39 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { token, err := ccs.TurnstileToken(c.cfClearanceUrl, apiUrl, "0x4AAAAAAAfOlgvCKbOdW1zc") if err != nil { - c.logger.Error("Failed to get Turnstile token", zap.Error(err), zap.String("apiUrl", apiUrl)) - return nil, fmt.Errorf("failed to get Turnstile token for URL %s: %w", apiUrl, err) + c.logger.Error("Failed to get Turnstile token", zap.Error(err), zap.String("URL", URL)) + return nil, fmt.Errorf("failed to get Turnstile token for URL %s: %w", URL, err) } resp, err := utils.Request().SetHeader("cf-turnstile-response", token).Get(apiUrl) if err != nil { - c.logger.Error("Failed to fetch data from API", zap.Error(err), zap.String("apiUrl", apiUrl)) - return nil, fmt.Errorf("failed to fetch API data for URL %s: %w", apiUrl, err) + c.logger.Error("Failed to fetch data from API", zap.Error(err), zap.String("URL", URL)) + return nil, fmt.Errorf("failed to fetch API data for URL %s: %w", URL, err) } data := gameResult{} err = json.Unmarshal(resp.Body(), &data) if err != nil { - c.logger.Error("Failed to unmarshal API response", zap.Error(err), zap.String("apiUrl", apiUrl)) - return nil, fmt.Errorf("failed to parse API response for URL %s: %w", apiUrl, err) + c.logger.Error("Failed to unmarshal API response", zap.Error(err), zap.String("URL", URL)) + return nil, fmt.Errorf("failed to parse API response for URL %s: %w", URL, err) } name := data.Title - // Find download links - fileHosters := []string{ - "gofile", - "fileditch", - "qiwi", - "filesfm", - "pixeldrain", - "1fichier", - } - links := make([]string, 0) - for _, h := range fileHosters { - if value, exist := data.Links.Game[h]; exist { - for _, link := range value.Links { - links = append(links, link.Link) - } + links := make(map[string]string, 0) + for _, v := range data.Links.Game { + for _, link := range v.Links { + links[fmt.Sprintf("%s(%s)", link.Label, v.Name)] = link.Link } - if value, exist := data.Links.Patch[h]; exist { - for _, link := range value.Links { - links = append(links, link.Link) - } + } + for _, v := range data.Links.Patch { + for _, link := range v.Links { + links[fmt.Sprintf("%s(%s)", link.Label, v.Name)] = link.Link + } + } + for _, v := range data.Links.Goodie { + for _, link := range v.Links { + links[fmt.Sprintf("%s(%s)", link.Label, v.Name)] = link.Link } } @@ -112,7 +106,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { item.Name = name item.RawName = name - item.DownloadLinks = links + item.Downloads = links item.Url = URL item.Size = utils.BytesToSize(size) item.Author = "GOGGames" @@ -138,6 +132,10 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) { urls := make([]string, 0) var updateFlags []string // link+date for _, item := range data.Data { + if item.Infohash == "" { + // skip unreleased games + continue + } urls = append(urls, fmt.Sprintf(constant.GOGGamesPageURL, item.Slug)) updateFlags = append(updateFlags, base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s%s", item.GogURL, item.LastUpdate)))) } @@ -279,39 +277,13 @@ type gameResult struct { Md5Filename string `json:"md5_filename"` Infohash string `json:"infohash"` Links struct { - Goodie struct { - OneFichier struct { - ID string `json:"id"` - Name string `json:"name"` - Links []struct { - Label string `json:"label"` - Link string `json:"link"` - } `json:"links"` - } `json:"1fichier"` - Vikingfile struct { - ID string `json:"id"` - Name string `json:"name"` - Links []struct { - Label string `json:"label"` - Link string `json:"link"` - } `json:"links"` - } `json:"vikingfile"` - Pixeldrain struct { - ID string `json:"id"` - Name string `json:"name"` - Links []struct { - Label string `json:"label"` - Link string `json:"link"` - } `json:"links"` - } `json:"pixeldrain"` - Gofile struct { - ID string `json:"id"` - Name string `json:"name"` - Links []struct { - Label string `json:"label"` - Link string `json:"link"` - } `json:"links"` - } `json:"gofile"` + Goodie map[string]struct { + ID string `json:"id"` + Name string `json:"name"` + Links []struct { + Label string `json:"label"` + Link string `json:"link"` + } `json:"links"` } `json:"goodie"` Game map[string]struct { ID string `json:"id"` diff --git a/crawler/igdb.go b/crawler/igdb.go index a6c0344..26e4d16 100644 --- a/crawler/igdb.go +++ b/crawler/igdb.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "net/url" - "runtime/debug" "strconv" "strings" "time" @@ -20,7 +19,6 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/go-resty/resty/v2" - "go.uber.org/zap" ) type twitchToken struct { @@ -34,13 +32,9 @@ func (t *twitchToken) getToken() (string, error) { } token, expires, err := loginTwitch() if err != nil { - zap.L().Error("failed to login to Twitch", zap.Error(err)) return "", fmt.Errorf("failed to login twitch: %w", err) } - err = cache.SetWithExpire("twitch_token", token, expires) - if err != nil { - zap.L().Error("failed to set Twitch token in cache", zap.Error(err)) - } + _ = cache.SetWithExpire("twitch_token", token, expires) return token, nil } @@ -54,8 +48,7 @@ func loginTwitch() (string, time.Duration, error) { resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String()) if err != nil { - zap.L().Error("failed to make Twitch login request", zap.String("url", baseURL.String()), zap.Error(err)) - return "", 0, err + return "", 0, fmt.Errorf("failed to make request: %s: %w", baseURL.String(), err) } data := struct { @@ -65,8 +58,7 @@ func loginTwitch() (string, time.Duration, error) { }{} err = json.Unmarshal(resp.Body(), &data) if err != nil { - zap.L().Error("failed to parse Twitch login response", zap.String("response", string(resp.Body())), zap.Error(err)) - return "", 0, err + return "", 0, fmt.Errorf("failed to parse response: %w", err) } return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil } @@ -74,8 +66,7 @@ func loginTwitch() (string, time.Duration, error) { func igdbRequest(URL string, dataBody any) (*resty.Response, error) { t, err := token.getToken() if err != nil { - zap.L().Error("failed to get Twitch token", zap.Error(err)) - return nil, err + return nil, fmt.Errorf("failed to get Twitch token: %w", err) } resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{ @@ -86,8 +77,7 @@ func igdbRequest(URL string, dataBody any) (*resty.Response, error) { }).Post(URL) if err != nil { - zap.L().Error("failed to make IGDB request", zap.String("url", URL), zap.Any("dataBody", dataBody), zap.Error(err)) - return nil, err + return nil, fmt.Errorf("failed to make request: %s: %w", URL, err) } return resp, nil } @@ -95,22 +85,19 @@ func igdbRequest(URL string, dataBody any) (*resty.Response, error) { func getIGDBID(name string) (int, error) { resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name)) if err != nil { - zap.L().Error("failed to search IGDB ID", zap.String("name", name), zap.Error(err)) - return 0, err + return 0, fmt.Errorf("failed to search IGDB ID: %s: %w", name, err) } if string(resp.Body()) == "[]" { resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name)) if err != nil { - zap.L().Error("failed to fallback search IGDB ID", zap.String("name", name), zap.Error(err)) - return 0, err + return 0, fmt.Errorf("failed to fallback search: %s: %w", name, err) } } var data model.IGDBSearches if err = json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("failed to unmarshal IGDB search response", zap.String("response", string(resp.Body())), zap.Error(err)) - return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack()) + return 0, fmt.Errorf("failed to parse IGDB search response: %w", err) } if len(data) == 1 { @@ -132,8 +119,7 @@ func getIGDBID(name string) (int, error) { detail, err := GetIGDBAppDetail(item.Game) if err != nil { - zap.L().Error("failed to get IGDB app detail", zap.Int("gameID", item.Game), zap.Error(err)) - return 0, err + return 0, fmt.Errorf("failed to get IGDB app detail: %d: %w", item.Game, err) } for _, altName := range detail.AlternativeNames { if sim := utils.Similarity(altName.Name, name); sim >= 0.8 { @@ -149,7 +135,6 @@ func getIGDBID(name string) (int, error) { return GetIGDBAppParent(data[maxSimilarityIndex].Game) } - zap.L().Warn("no IGDB ID found", zap.String("name", name)) return 0, fmt.Errorf("IGDB ID not found: %s", name) } @@ -235,33 +220,27 @@ func GetIGDBAppParent(id int) (int, error) { if exist { id, err := strconv.Atoi(val) if err != nil { - zap.L().Error("failed to parse cached IGDB parent ID", zap.String("cacheKey", key), zap.Error(err)) - return 0, err + return 0, fmt.Errorf("failed to parse cached IGDB parent ID: %s: %w", key, err) } return id, nil } detail, err := GetIGDBAppDetail(id) if err != nil { - zap.L().Error("failed to fetch IGDB app detail for parent", zap.Int("gameID", id), zap.Error(err)) - return 0, err + return 0, fmt.Errorf("failed to fetch IGDB app detail for parent: %d: %w", id, err) } hasParent := false for detail.VersionParent != 0 { hasParent = true detail, err = GetIGDBAppDetail(detail.VersionParent) if err != nil { - zap.L().Error("failed to fetch IGDB version parent", zap.Int("parentID", detail.VersionParent), zap.Error(err)) - return 0, err + return 0, fmt.Errorf("failed to fetch IGDB version parent: %d: %w", detail.VersionParent, err) } } if hasParent { return detail.ID, nil } - err = cache.Set(key, id) - if err != nil { - zap.L().Error("failed to cache IGDB parent ID", zap.String("cacheKey", key), zap.Error(err)) - } + _ = cache.Set(key, id) return id, nil } @@ -270,7 +249,6 @@ func GetIGDBAppParent(id int) (int, error) { func GetIGDBID(name string) (int, error) { key := fmt.Sprintf("igdb_id:%s", name) if val, exist := cache.Get(key); exist { - zap.L().Info("cache hit for IGDB ID", zap.String("name", name), zap.String("cacheKey", key)) return strconv.Atoi(val) } @@ -279,10 +257,7 @@ func GetIGDBID(name string) (int, error) { for _, n := range normalizedNames { id, err := getIGDBID(n) if err == nil { - cacheErr := cache.Set(key, id) - if cacheErr != nil { - zap.L().Warn("failed to cache IGDB ID", zap.String("name", n), zap.Error(cacheErr)) - } + _ = cache.Set(key, id) return id, nil } } @@ -291,16 +266,12 @@ func GetIGDBID(name string) (int, error) { for _, n := range normalizedNames { id, err := getIGDBIDBySteamSearch(n) if err == nil { - cacheErr := cache.Set(key, id) - if cacheErr != nil { - zap.L().Warn("failed to cache IGDB ID after Steam search", zap.String("name", n), zap.Error(cacheErr)) - } + _ = cache.Set(key, id) return id, nil } } - zap.L().Warn("failed to retrieve IGDB ID", zap.String("name", name)) - return 0, fmt.Errorf("IGDB ID not found for '%s'", name) + return 0, fmt.Errorf("IGDB ID not found: %s", name) } func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { @@ -309,27 +280,23 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { if exist { var data model.IGDBGameDetail if err := json.Unmarshal([]byte(val), &data); err != nil { - zap.L().Error("failed to parse cached IGDB game detail", zap.String("cacheKey", key), zap.Error(err)) - return nil, err + return nil, fmt.Errorf("failed to parse cached IGDB game detail: %s: %w", key, err) } return &data, nil } resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id = %v; fields *,alternative_names.*,language_supports.*,screenshots.*,cover.*,involved_companies.*,game_engines.*,game_modes.*,genres.*,player_perspectives.*,release_dates.*,videos.*,websites.*,platforms.*,themes.*,collections.*;`, id)) if err != nil { - zap.L().Error("failed to fetch IGDB game detail", zap.Int("gameID", id), zap.Error(err)) - return nil, err + return nil, fmt.Errorf("failed to fetch IGDB game detail for ID %d: %w", id, err) } var data model.IGDBGameDetails if err = json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("failed to unmarshal IGDB game detail response", zap.String("response", string(resp.Body())), zap.Error(err)) - return nil, err + return nil, fmt.Errorf("failed to unmarshal IGDB game detail response: %w", err) } if len(data) == 0 { - zap.L().Warn("IGDB game not found", zap.Int("gameID", id)) - return nil, errors.New("IGDB App not found") + return nil, fmt.Errorf("IGDB game detail not found for ID %d", id) } if data[0].Name == "" { @@ -338,10 +305,7 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { jsonBytes, err := json.Marshal(data[0]) if err == nil { - err = cache.Set(key, string(jsonBytes)) - if err != nil { - zap.L().Error("failed to cache IGDB game detail", zap.String("cacheKey", key), zap.Error(err)) - } + _ = cache.Set(key, string(jsonBytes)) } return data[0], nil @@ -351,33 +315,26 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) { func GetIGDBCompany(id int) (string, error) { key := fmt.Sprintf("igdb_companies:%d", id) if val, exist := cache.Get(key); exist { - zap.L().Info("cache hit for IGDB company", zap.Int("companyID", id), zap.String("cacheKey", key)) return val, nil } query := fmt.Sprintf(`where id=%d; fields *;`, id) resp, err := igdbRequest(constant.IGDBCompaniesURL, query) if err != nil { - zap.L().Error("failed to fetch IGDB company", zap.Int("companyID", id), zap.Error(err)) return "", fmt.Errorf("failed to fetch IGDB company for ID %d: %w", id, err) } var data model.IGDBCompanies if err = json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("failed to unmarshal IGDB company response", zap.String("response", string(resp.Body())), zap.Error(err)) return "", fmt.Errorf("failed to unmarshal IGDB companies response: %w", err) } if len(data) == 0 { - zap.L().Warn("no company found in IGDB for ID", zap.Int("companyID", id)) return "", errors.New("company not found") } companyName := data[0].Name - cacheErr := cache.Set(key, companyName) - if cacheErr != nil { - zap.L().Warn("failed to cache IGDB company name", zap.Int("companyID", id), zap.Error(cacheErr)) - } + _ = cache.Set(key, companyName) return companyName, nil } @@ -385,7 +342,6 @@ func GetIGDBCompany(id int) (string, error) { func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { detail, err := GetIGDBAppDetail(id) if err != nil { - zap.L().Error("failed to fetch IGDB app detail", zap.Int("igdbID", id), zap.Error(err)) return nil, fmt.Errorf("failed to fetch IGDB app detail for ID %d: %w", id, err) } @@ -415,7 +371,6 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { for _, company := range detail.InvolvedCompanies { companyName, err := GetIGDBCompany(company.Company) if err != nil { - zap.L().Warn("failed to fetch company name", zap.Int("companyID", company.Company), zap.Error(err)) continue } if company.Developer { @@ -445,7 +400,6 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) { func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { id, err := GetIGDBID(game.Name) if err != nil { - zap.L().Error("failed to get IGDB ID for game", zap.String("gameName", game.Name), zap.Error(err)) return nil, fmt.Errorf("failed to get IGDB ID for game '%s': %w", game.Name, err) } @@ -457,7 +411,6 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { info, err = GenerateIGDBGameInfo(id) if err != nil { - zap.L().Error("failed to generate IGDB game info", zap.Int("igdbID", id), zap.Error(err)) return nil, fmt.Errorf("failed to generate IGDB game info for ID %d: %w", id, err) } @@ -469,14 +422,12 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) { func GetIGDBIDBySteamAppID(id int) (int, error) { key := fmt.Sprintf("igdb_id_by_steam_app_id:%d", id) if val, exist := cache.Get(key); exist { - zap.L().Info("cache hit for IGDB ID by Steam App ID", zap.Int("steamAppID", id), zap.String("cacheKey", key)) return strconv.Atoi(val) } query := fmt.Sprintf(`where url = "https://store.steampowered.com/app/%d" | url = "https://store.steampowered.com/app/%d/"; fields game;`, id, id) resp, err := igdbRequest(constant.IGDBWebsitesURL, query) if err != nil { - zap.L().Error("failed to fetch IGDB ID by Steam App ID", zap.Int("steamAppID", id), zap.Error(err)) return 0, fmt.Errorf("failed to fetch IGDB ID by Steam App ID %d: %w", id, err) } @@ -484,20 +435,15 @@ func GetIGDBIDBySteamAppID(id int) (int, error) { Game int `json:"game"` } if err = json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("failed to unmarshal IGDB response", zap.String("response", string(resp.Body())), zap.Error(err)) return 0, fmt.Errorf("failed to unmarshal IGDB response for Steam App ID %d: %w", id, err) } if len(data) == 0 || data[0].Game == 0 { - zap.L().Warn("no matching IGDB game found for Steam App ID", zap.Int("steamAppID", id)) return 0, errors.New("no matching IGDB game found") } igdbID := data[0].Game - cacheErr := cache.Set(key, strconv.Itoa(igdbID)) - if cacheErr != nil { - zap.L().Warn("failed to cache IGDB ID by Steam App ID", zap.Int("steamAppID", id), zap.Error(cacheErr)) - } + _ = cache.Set(key, strconv.Itoa(igdbID)) return GetIGDBAppParent(igdbID) } @@ -506,14 +452,12 @@ func GetIGDBIDBySteamAppID(id int) (int, error) { func GetIGDBIDBySteamBundleID(id int) (int, error) { key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%d", id) if val, exist := cache.Get(key); exist { - zap.L().Info("cache hit for IGDB ID by Steam Bundle ID", zap.Int("steamBundleID", id), zap.String("cacheKey", key)) return strconv.Atoi(val) } query := fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%d" | url = "https://store.steampowered.com/bundle/%d/"; fields game;`, id, id) resp, err := igdbRequest(constant.IGDBWebsitesURL, query) if err != nil { - zap.L().Error("failed to fetch IGDB ID by Steam Bundle ID", zap.Int("steamBundleID", id), zap.Error(err)) return 0, fmt.Errorf("failed to fetch IGDB ID by Steam Bundle ID %d: %w", id, err) } @@ -521,20 +465,15 @@ func GetIGDBIDBySteamBundleID(id int) (int, error) { Game int `json:"game"` } if err = json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("failed to unmarshal IGDB response", zap.String("response", string(resp.Body())), zap.Error(err)) return 0, fmt.Errorf("failed to unmarshal IGDB response for Steam Bundle ID %d: %w", id, err) } if len(data) == 0 || data[0].Game == 0 { - zap.L().Warn("no matching IGDB game found for Steam Bundle ID", zap.Int("steamBundleID", id)) return 0, errors.New("no matching IGDB game found") } igdbID := data[0].Game - cacheErr := cache.Set(key, strconv.Itoa(igdbID)) - if cacheErr != nil { - zap.L().Warn("failed to cache IGDB ID by Steam Bundle ID", zap.Int("steamBundleID", id), zap.Error(cacheErr)) - } + _ = cache.Set(key, strconv.Itoa(igdbID)) return GetIGDBAppParent(igdbID) } @@ -548,7 +487,6 @@ func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) { query := fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %d; offset %d; where popularity_type = %d;", limit, offset, popularityType) resp, err := igdbRequest(constant.IGDBPopularityURL, query) if err != nil { - zap.L().Error("failed to fetch popular IGDB game IDs", zap.Int("popularityType", popularityType), zap.Error(err)) return nil, fmt.Errorf("failed to fetch popular IGDB game IDs for type %d: %w", popularityType, err) } @@ -557,7 +495,6 @@ func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) { Value float64 `json:"value"` } if err = json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("failed to unmarshal IGDB popular games response", zap.String("response", string(resp.Body())), zap.Error(err)) return nil, fmt.Errorf("failed to unmarshal IGDB popular games response: %w", err) } @@ -565,7 +502,6 @@ func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) { for _, d := range data { parentID, err := GetIGDBAppParent(d.GameID) if err != nil { - zap.L().Warn("failed to fetch parent IGDB ID for game", zap.Int("gameID", d.GameID), zap.Error(err)) gameIDs = append(gameIDs, d.GameID) } else { gameIDs = append(gameIDs, parentID) diff --git a/crawler/nxbrew.go b/crawler/nxbrew.go new file mode 100644 index 0000000..719115d --- /dev/null +++ b/crawler/nxbrew.go @@ -0,0 +1,13 @@ +package crawler + +import "go.uber.org/zap" + +type NxbrewCrawler struct { + logger *zap.Logger +} + +func NewNxbrewCrawler(logger *zap.Logger) *NxbrewCrawler { + return &NxbrewCrawler{ + logger: logger, + } +} diff --git a/crawler/onlinefix.go b/crawler/onlinefix.go index bc5cfec..ad613f7 100644 --- a/crawler/onlinefix.go +++ b/crawler/onlinefix.go @@ -176,7 +176,9 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { return nil, fmt.Errorf("failed to convert torrent to magnet: %w", err) } - item.DownloadLinks = []string{magnet} + item.Downloads = map[string]string{ + "magnet": magnet, + } item.Size = size } else { c.logger.Warn("Unsupported download link format", zap.String("url", downloadURL)) diff --git a/crawler/rutracker.go b/crawler/rutracker.go index 24c3164..571cf67 100644 --- a/crawler/rutracker.go +++ b/crawler/rutracker.go @@ -1,6 +1,7 @@ package crawler import ( + "bytes" "encoding/base64" "encoding/json" "fmt" @@ -11,7 +12,6 @@ import ( "game-crawler/utils" "net/url" "strconv" - "strings" "time" "unicode" @@ -47,8 +47,6 @@ func NewRutrackerCrawler(source, platform, rid, username, password, cfClearanceU } func (r *RutrackerCrawler) getSession() (*ccs.Session, error) { - r.logger.Info("Fetching session for RutrackerCrawler") - if r.username == "" || r.password == "" { r.logger.Error("Username or password is empty") return nil, fmt.Errorf("username or password is empty") @@ -121,6 +119,7 @@ func (r *RutrackerCrawler) getSession() (*ccs.Session, error) { } func (r *RutrackerCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { + r.logger.Info("Crawling game", zap.String("URL", URL)) session, err := r.getSession() if err != nil { return nil, fmt.Errorf("failed to get session: %w", err) @@ -132,7 +131,7 @@ func (r *RutrackerCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { return nil, fmt.Errorf("failed to request URL: %w", err) } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.Body)) + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(utils.Windows1251ToUTF8([]byte(resp.Body)))) if err != nil { r.logger.Error("Failed to parse HTML", zap.Error(err)) return nil, fmt.Errorf("failed to parse HTML: %w", err) @@ -155,8 +154,9 @@ func (r *RutrackerCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { r.logger.Error("Failed to find magnet link") return nil, fmt.Errorf("failed to find magnet link") } - item.DownloadLinks = []string{magnet} - + item.Downloads = map[string]string{ + "magnet": magnet, + } sizeStr := doc.Find("#tor-size-humn").AttrOr("title", "") if sizeStr == "" { r.logger.Warn("Failed to find size") @@ -187,7 +187,7 @@ func (r *RutrackerCrawler) Crawl(page int) ([]*model.GameItem, error) { return nil, fmt.Errorf("failed to request URL: %w", err) } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.Body)) + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(utils.Windows1251ToUTF8([]byte(resp.Body)))) if err != nil { r.logger.Error("Failed to parse HTML", zap.Error(err)) return nil, fmt.Errorf("failed to parse HTML: %w", err) @@ -195,7 +195,7 @@ func (r *RutrackerCrawler) Crawl(page int) ([]*model.GameItem, error) { var urls []string var updateFlags []string doc.Find("[id^='trs-tr']").Each(func(i int, s *goquery.Selection) { - a := s.Find(".t-title") + a := s.Find(".t-title a") datetime := s.Find("td").Last().Text() url, exists := a.Attr("href") if !exists { @@ -219,6 +219,7 @@ func (r *RutrackerCrawler) Crawl(page int) ([]*model.GameItem, error) { r.logger.Error("Failed to crawl URL", zap.String("URL", URL), zap.Error(err)) continue } + item.UpdateFlag = updateFlags[i] err = db.SaveGameItem(item) if err != nil { r.logger.Error("Failed to save game item to database", zap.String("URL", URL), zap.Error(err)) @@ -264,7 +265,7 @@ func (r *RutrackerCrawler) GetTotalPageNum() (int, error) { return 0, fmt.Errorf("failed to request URL: %w", err) } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.Body)) + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(utils.Windows1251ToUTF8([]byte(resp.Body)))) if err != nil { r.logger.Error("Failed to parse HTML", zap.Error(err)) return 0, fmt.Errorf("failed to parse HTML: %w", err) diff --git a/crawler/steam.go b/crawler/steam.go index c54316a..59485dd 100644 --- a/crawler/steam.go +++ b/crawler/steam.go @@ -13,18 +13,14 @@ import ( "game-crawler/constant" "game-crawler/model" "game-crawler/utils" - - "go.uber.org/zap" ) // GetSteamAppDetail fetches the details of a Steam app by its ID. func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) { key := fmt.Sprintf("steam_game:%d", id) if val, exist := cache.Get(key); exist { - zap.L().Info("Cache hit for Steam app detail", zap.Int("steamID", id)) var detail model.SteamAppDetail if err := json.Unmarshal([]byte(val), &detail); err != nil { - zap.L().Warn("Failed to unmarshal cached Steam app detail", zap.Int("steamID", id), zap.Error(err)) return nil, fmt.Errorf("failed to unmarshal cached Steam app detail for ID %d: %w", id, err) } return &detail, nil @@ -39,18 +35,15 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) { "User-Agent": "", }).Get(baseURL.String()) if err != nil { - zap.L().Error("Failed to fetch Steam app detail", zap.Int("steamID", id), zap.String("url", baseURL.String()), zap.Error(err)) return nil, fmt.Errorf("failed to fetch Steam app detail for ID %d: %w", id, err) } var detail map[string]*model.SteamAppDetail if err := json.Unmarshal(resp.Body(), &detail); err != nil { - zap.L().Error("Failed to unmarshal Steam app detail response", zap.Int("steamID", id), zap.String("response", string(resp.Body())), zap.Error(err)) return nil, fmt.Errorf("failed to unmarshal Steam app detail for ID %d: %w", id, err) } if appDetail, ok := detail[strconv.Itoa(id)]; !ok || appDetail == nil { - zap.L().Warn("Steam app detail not found", zap.Int("steamID", id)) return nil, fmt.Errorf("steam app not found: %d", id) } else { // Cache the result @@ -66,7 +59,6 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) { func GenerateSteamGameInfo(id int) (*model.GameInfo, error) { detail, err := GetSteamAppDetail(id) if err != nil { - zap.L().Error("Failed to fetch Steam app detail for game info generation", zap.Int("steamID", id), zap.Error(err)) return nil, fmt.Errorf("failed to fetch Steam app detail for ID %d: %w", id, err) } @@ -84,7 +76,6 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) { item.Screenshots = append(item.Screenshots, screenshot.PathFull) } - zap.L().Info("Generated Steam game info", zap.Int("steamID", id), zap.String("name", item.Name)) return item, nil } @@ -92,10 +83,8 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) { func GetSteamIDByIGDBID(IGDBID int) (int, error) { key := fmt.Sprintf("steam_game:%d", IGDBID) if val, exist := cache.Get(key); exist { - zap.L().Info("Cache hit for Steam ID by IGDB ID", zap.Int("IGDBID", IGDBID)) id, err := strconv.Atoi(val) if err != nil { - zap.L().Warn("Failed to parse cached Steam ID", zap.Int("IGDBID", IGDBID), zap.Error(err)) return 0, fmt.Errorf("failed to parse cached Steam ID for IGDB ID %d: %w", IGDBID, err) } return id, nil @@ -104,7 +93,6 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) { query := fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID) resp, err := igdbRequest(constant.IGDBWebsitesURL, query) if err != nil { - zap.L().Error("Failed to fetch IGDB websites for Steam ID", zap.Int("IGDBID", IGDBID), zap.Error(err)) return 0, fmt.Errorf("failed to fetch IGDB websites for IGDB ID %d: %w", IGDBID, err) } @@ -113,12 +101,10 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) { Url string `json:"url"` } if err := json.Unmarshal(resp.Body(), &data); err != nil { - zap.L().Error("Failed to unmarshal IGDB websites response", zap.Int("IGDBID", IGDBID), zap.String("response", string(resp.Body())), zap.Error(err)) return 0, fmt.Errorf("failed to unmarshal IGDB websites response for IGDB ID %d: %w", IGDBID, err) } if len(data) == 0 { - zap.L().Warn("No Steam ID found for IGDB ID", zap.Int("IGDBID", IGDBID)) return 0, errors.New("steam ID not found") } @@ -127,23 +113,19 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) { regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`) idMatch := regex.FindStringSubmatch(v.Url) if len(idMatch) < 2 { - zap.L().Warn("Failed to parse Steam ID from URL", zap.String("url", v.Url)) return 0, errors.New("failed to parse Steam ID from URL") } steamID, err := strconv.Atoi(idMatch[1]) if err != nil { - zap.L().Error("Failed to convert Steam ID to integer", zap.String("url", v.Url), zap.Error(err)) return 0, fmt.Errorf("failed to convert Steam ID from URL %s: %w", v.Url, err) } // Cache the result _ = cache.Set(key, strconv.Itoa(steamID)) - zap.L().Info("Found Steam ID for IGDB ID", zap.Int("IGDBID", IGDBID), zap.Int("steamID", steamID)) return steamID, nil } } - zap.L().Warn("No valid Steam ID found in IGDB websites data", zap.Int("IGDBID", IGDBID)) - return 0, errors.New("steam ID not found") + return 0, fmt.Errorf("no valid Steam ID found for IGDB ID %d", IGDBID) } diff --git a/crawler/steam250.go b/crawler/steam250.go index 64061e8..401bf97 100644 --- a/crawler/steam250.go +++ b/crawler/steam250.go @@ -16,32 +16,26 @@ import ( "game-crawler/utils" "github.com/PuerkitoBio/goquery" - "go.uber.org/zap" ) // GetSteam250 fetches Steam250 game rankings from the given URL. func GetSteam250(URL string) ([]*model.GameInfo, error) { key := "steam250:" + url.QueryEscape(URL) if val, ok := cache.Get(key); ok { - zap.L().Info("Cache hit for Steam250 rankings", zap.String("url", URL)) var infos []*model.GameInfo if err := json.Unmarshal([]byte(val), &infos); err != nil { - zap.L().Warn("Failed to unmarshal cached Steam250 data", zap.String("url", URL), zap.Error(err)) return nil, fmt.Errorf("failed to unmarshal cached Steam250 data for URL %s: %w", URL, err) } return infos, nil } - zap.L().Info("Fetching Steam250 rankings from URL", zap.String("url", URL)) resp, err := utils.Request().Get(URL) if err != nil { - zap.L().Error("Failed to fetch Steam250 rankings", zap.String("url", URL), zap.Error(err)) return nil, fmt.Errorf("failed to fetch Steam250 rankings from URL %s: %w", URL, err) } doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) if err != nil { - zap.L().Error("Failed to parse Steam250 HTML document", zap.String("url", URL), zap.Error(err)) return nil, fmt.Errorf("failed to parse Steam250 HTML document for URL %s: %w", URL, err) } @@ -53,7 +47,6 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) { // Extract game name item.Name = s.Find(".title>a").First().Text() if item.Name == "" { - zap.L().Warn("Game name not found in Steam250 rankings", zap.String("url", URL), zap.Int("index", i)) return } @@ -61,13 +54,11 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) { idStr := s.Find(".store").AttrOr("href", "") idSlice := regexp.MustCompile(`app/(\d+)/`).FindStringSubmatch(idStr) if len(idSlice) < 2 { - zap.L().Warn("Failed to extract Steam ID from URL", zap.String("url", idStr), zap.Int("index", i)) return } steamID, err := strconv.Atoi(idSlice[1]) if err != nil { - zap.L().Warn("Failed to convert Steam ID to integer", zap.String("id", idSlice[1]), zap.Error(err)) return } @@ -77,15 +68,11 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) { }) if len(steamIDs) == 0 { - zap.L().Warn("No valid Steam IDs found in Steam250 rankings", zap.String("url", URL)) return nil, fmt.Errorf("no valid Steam IDs found in Steam250 rankings for URL %s", URL) } - // Fetch game info from the database - zap.L().Info("Fetching game info from database", zap.Ints("steamIDs", steamIDs)) infos, err := db.GetGameInfosByPlatformIDs("steam", steamIDs) if err != nil { - zap.L().Error("Failed to fetch game info from database", zap.Ints("steamIDs", steamIDs), zap.Error(err)) return nil, fmt.Errorf("failed to fetch game info for Steam IDs %v: %w", steamIDs, err) } @@ -97,12 +84,7 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) { // Cache the result jsonBytes, err := json.Marshal(infos) if err == nil { - cacheErr := cache.SetWithExpire(key, string(jsonBytes), 12*time.Hour) - if cacheErr != nil { - zap.L().Warn("Failed to cache Steam250 rankings", zap.String("url", URL), zap.Error(cacheErr)) - } - } else { - zap.L().Warn("Failed to marshal Steam250 rankings for caching", zap.String("url", URL), zap.Error(err)) + _ = cache.SetWithExpire(key, string(jsonBytes), 24*time.Hour) } return infos, nil @@ -110,31 +92,26 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) { // GetSteam250Top250 retrieves the top 250 games from Steam250. func GetSteam250Top250() ([]*model.GameInfo, error) { - zap.L().Info("Fetching Steam250 Top 250 games") return GetSteam250(constant.Steam250Top250URL) } // GetSteam250BestOfTheYear retrieves the best games of the current year from Steam250. func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) { year := time.Now().UTC().Year() - zap.L().Info("Fetching Steam250 Best of the Year games", zap.Int("year", year)) return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, year)) } // GetSteam250WeekTop50 retrieves the top 50 games of the week from Steam250. func GetSteam250WeekTop50() ([]*model.GameInfo, error) { - zap.L().Info("Fetching Steam250 Week Top 50 games") return GetSteam250(constant.Steam250WeekTop50URL) } // GetSteam250MonthTop50 retrieves the top 50 games of the month from Steam250. func GetSteam250MonthTop50() ([]*model.GameInfo, error) { - zap.L().Info("Fetching Steam250 Month Top 50 games") return GetSteam250(constant.Steam250MonthTop50URL) } // GetSteam250MostPlayed retrieves the most played games from Steam250. func GetSteam250MostPlayed() ([]*model.GameInfo, error) { - zap.L().Info("Fetching Steam250 Most Played games") return GetSteam250(constant.Steam250MostPlayedURL) } diff --git a/crawler/steamrip.go b/crawler/steamrip.go index 3df70b5..7eb944c 100644 --- a/crawler/steamrip.go +++ b/crawler/steamrip.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "errors" "fmt" + "net/url" "regexp" "strings" @@ -36,7 +37,7 @@ func (c *SteamRIPCrawler) Name() string { // CrawlByUrl crawls a single game page from SteamRIP by URL. func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - c.logger.Info("Crawling game details", zap.String("URL", URL)) + c.logger.Info("Crawling game", zap.String("URL", URL)) // Fetch the page content resp, err := utils.Request().Get(URL) @@ -78,42 +79,22 @@ func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { item.Size = "unknown" } - // Extract download links - item.DownloadLinks = c.extractDownloadLinks(string(resp.Body())) - if len(item.DownloadLinks) == 0 { + downloadLinks := map[string]string{} + doc.Find(".shortc-button").Each(func(i int, s *goquery.Selection) { + downloadLink, _ := s.Attr("href") + u, _ := url.Parse(downloadLink) + downloadLinks[u.Host] = downloadLink + }) + item.Downloads = downloadLinks + + if len(item.Downloads) == 0 { c.logger.Warn("No download links found", zap.String("URL", URL)) return nil, errors.New("failed to find download link") } - c.logger.Info("Crawled game details successfully", zap.String("Name", item.Name), zap.String("URL", URL)) return item, nil } -// extractDownloadLinks extracts download links from the game page HTML. -func (c *SteamRIPCrawler) extractDownloadLinks(pageContent string) []string { - var links []string - - // Match MegaDB links - megadbRegex := regexp.MustCompile(`(?i)(?:https?:)?(//megadb\.net/[^"]+)`) - if matches := megadbRegex.FindStringSubmatch(pageContent); len(matches) > 1 { - links = append(links, fmt.Sprintf("https:%s", matches[1])) - } - - // Match Gofile links - gofileRegex := regexp.MustCompile(`(?i)(?:https?:)?(//gofile\.io/d/[^"]+)`) - if matches := gofileRegex.FindStringSubmatch(pageContent); len(matches) > 1 { - links = append(links, fmt.Sprintf("https:%s", matches[1])) - } - - // Match Filecrypt links - filecryptRegex := regexp.MustCompile(`(?i)(?:https?:)?(//filecrypt\.co/Container/[^"]+)`) - if matches := filecryptRegex.FindStringSubmatch(pageContent); len(matches) > 1 { - links = append(links, fmt.Sprintf("https:%s", matches[1])) - } - - return links -} - // Crawl crawls a limited number of games from the SteamRIP game list. func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) { c.logger.Info("Starting SteamRIP crawl", zap.Int("limit", num)) diff --git a/crawler/xatab.go b/crawler/xatab.go index d1b67fa..7512d54 100644 --- a/crawler/xatab.go +++ b/crawler/xatab.go @@ -98,7 +98,7 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameItem, error) { // CrawlByUrl crawls a single game page from Xatab by URL. func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { - c.logger.Info("Crawling game details", zap.String("URL", URL)) + c.logger.Info("Crawling game", zap.String("URL", URL)) // Fetch the game page resp, err := utils.Request().Get(URL) @@ -151,9 +151,10 @@ func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) { } item.Size = size - item.DownloadLinks = []string{magnet} + item.Downloads = map[string]string{ + "magnet": magnet, + } - c.logger.Info("Crawled game details successfully", zap.String("Name", item.Name), zap.String("URL", URL)) return item, nil } diff --git a/db/game.go b/db/game.go index 11b1c35..2494b89 100644 --- a/db/game.go +++ b/db/game.go @@ -475,10 +475,7 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) { defer cancel() type queryRes struct { - ID struct { - RawName string `bson:"raw_name"` - Download string `bson:"download"` - } `bson:"_id"` + ID string `bson:"_id"` Count int `bson:"count"` IDs []primitive.ObjectID `bson:"ids"` } @@ -489,12 +486,7 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) { bson.D{ {Key: "$group", Value: bson.D{ - {Key: "_id", - Value: bson.D{ - {Key: "raw_name", Value: "$raw_name"}, - {Key: "download", Value: "$download"}, - }, - }, + {Key: "_id", Value: bson.D{{Key: "url", Value: "$url"}}}, {Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}}, {Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}}, }, diff --git a/model/game.go b/model/game.go index 895f64f..5303732 100644 --- a/model/game.go +++ b/model/game.go @@ -41,16 +41,16 @@ type GameCollection struct { } type GameItem struct { - ID primitive.ObjectID `json:"id" bson:"_id"` - Name string `json:"speculative_name" bson:"name"` - RawName string `json:"raw_name,omitempty" bson:"raw_name"` - DownloadLinks []string `json:"download_links,omitempty" bson:"download_links"` - Size string `json:"size,omitempty" bson:"size"` - Url string `json:"url" bson:"url"` - Password string `json:"password,omitempty" bson:"password"` - Author string `json:"author,omitempty" bson:"author"` - Platform string `json:"platform,omitempty" bson:"platform"` - UpdateFlag string `json:"update_flag,omitempty" bson:"update_flag"` - CreatedAt time.Time `json:"created_at" bson:"created_at"` - UpdatedAt time.Time `json:"updated_at" bson:"updated_at"` + ID primitive.ObjectID `json:"id" bson:"_id"` + Name string `json:"speculative_name" bson:"name"` + RawName string `json:"raw_name,omitempty" bson:"raw_name"` + Downloads map[string]string `json:"downloads,omitempty" bson:"downloads"` + Size string `json:"size,omitempty" bson:"size"` + Url string `json:"url" bson:"url"` + Password string `json:"password,omitempty" bson:"password"` + Author string `json:"author,omitempty" bson:"author"` + Platform string `json:"platform,omitempty" bson:"platform"` + UpdateFlag string `json:"update_flag,omitempty" bson:"update_flag"` + CreatedAt time.Time `json:"created_at" bson:"created_at"` + UpdatedAt time.Time `json:"updated_at" bson:"updated_at"` } diff --git a/server/templates/game.html b/server/templates/game.html index 42f1e22..c30e5a7 100644 --- a/server/templates/game.html +++ b/server/templates/game.html @@ -44,19 +44,12 @@ /* Masonry Container Styles */ .masonry-container { - column-count: 3; + column-count: 2; /* 3 Columns for Masonry */ column-gap: 1rem; /* Adjust Gap Between Columns */ } - @media (max-width: 992px) { - .masonry-container { - column-count: 2; - /* 2 Columns on Medium Screens */ - } - } - @media (max-width: 576px) { .masonry-container { column-count: 1; @@ -100,19 +93,23 @@ {{end}} + {{if .Developers}}
Developers: {{range .Developers}} {{.}} {{end}}
+ {{end}} + {{if .Publishers}}
Publishers: {{range .Publishers}} {{.}} {{end}}
+ {{end}} {{if .Languages}}
@@ -121,17 +118,48 @@ {{.}} {{end}}
- {{end}} {{if .Description}} + {{end}} + + {{if .Description}}

{{.Description}}

- {{end}} {{if .SteamID}} + {{end}} + + {{if .SteamID}}
Steam
{{end}} + + {{if .GameEngines}} +
+ Engines: + {{range .GameEngines}} + {{.}} + {{end}} +
+ {{end}} + + {{if .Genres}} +
+ Genres: + {{range .Genres}} + {{.}} + {{end}} +
+ {{end}} + + {{if .Themes}} +
+ Themes: + {{range .Themes}} + {{.}} + {{end}} +
+ {{end}} @@ -156,53 +184,66 @@ {{if .Games}}
-

Download

-
-
- -
- {{range .Games}} -
-
-
{{.RawName}}
- {{if .Size}} -
- Size: {{.Size}} -
- {{end}} - {{if .Author}} -
- Source: {{.Author}} -
- {{end}} - {{if .Platform}} -
- Platform: {{.Platform}} -
- {{end}} - {{if .Password}} -
- Unzip password: {{.Password}} -
- {{end}} - {{if .UpdatedAt}} -
- Updated: {{.UpdatedAt}} -
- {{end}} - {{range .DownloadLinks}} -
- - -
- {{end}} +
+ +
+ {{range .Games}} +
+
+
{{.RawName}}
+ {{if .Size}} +
+ Size: {{.Size}}
+ {{end}} + {{if .Author}} +
+ Source: {{.Author}} +
+ {{end}} + {{if .Platform}} +
+ Platform: {{.Platform}} +
+ {{end}} + {{if .Password}} +
+ Unzip password: {{.Password}} +
+ {{end}} + {{if .UpdatedAt}} +
+ Updated: {{.UpdatedAt}} +
+ {{end}} + + + + + + + + + + {{range $key, $value := .Downloads}} + + + + + {{end}} + +
LabelLink
{{$key}} +
+ + +
+
- {{end}}
+ {{end}}
@@ -227,25 +268,20 @@ }, }); - function copyToClipboard(input) { - input.select(); - input.setSelectionRange(0, 99999); + function copyToClipboard(button, text) { + const el = document.createElement("textarea"); + el.value = text; + document.body.appendChild(el); + el.select(); + document.execCommand("copy"); + document.body.removeChild(el); - navigator.clipboard.writeText(input.value).then(() => { - const button = input.nextElementSibling; - const originalText = button.textContent; - button.textContent = 'Copied'; - button.classList.remove('btn-outline-secondary'); - button.classList.add('btn-success'); - - setTimeout(() => { - button.textContent = originalText; - button.classList.remove('btn-success'); - button.classList.add('btn-outline-secondary'); - }, 1500); - }).catch(err => { - console.error('Failed to copy:', err); - }); + button.textContent = "Copied"; + button.disabled = true; + setTimeout(() => { + button.textContent = "Copy"; + button.disabled = false; + }, 2000); } {{end}} \ No newline at end of file diff --git a/server/templates/layouts/base.html b/server/templates/layouts/base.html index 315dc7b..3f0846b 100644 --- a/server/templates/layouts/base.html +++ b/server/templates/layouts/base.html @@ -1,6 +1,6 @@ {{define "base"}} - + diff --git a/utils/mgnet.go b/utils/mgnet.go new file mode 100644 index 0000000..5ef8dc5 --- /dev/null +++ b/utils/mgnet.go @@ -0,0 +1,81 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/PuerkitoBio/goquery" +) + +func GetLinkFromMgnet(URL string) (string, error) { + resp, err := Request().Get(URL) + if err != nil { + return "", fmt.Errorf("Error while requesting URL: %s: %s", err, URL) + } + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body())) + if err != nil { + return "", fmt.Errorf("Error while parsing HTML: %s: %s", err, URL) + } + ad_form_data := doc.Find("[name='ad_form_data']").AttrOr("value", "") + if ad_form_data == "" { + return "", fmt.Errorf("Failed to get ad_form_data: %s", URL) + } + token_fields := doc.Find("[name='_Token[fields]']").AttrOr("value", "") + if token_fields == "" { + return "", fmt.Errorf("Failed to get _Token[fields]: %s", URL) + } + token_unlocked := doc.Find("[name='_Token[unlocked]']").AttrOr("value", "") + if token_unlocked == "" { + return "", fmt.Errorf("Failed to get _Token[unlocked]: %s", URL) + } + cookies := resp.Cookies() + csrfToken := "" + for _, cookie := range cookies { + if cookie.Name == "csrfToken" { + csrfToken = cookie.Value + break + } + } + if csrfToken == "" { + return "", fmt.Errorf("Failed to get csrfToken: %s", URL) + } + + params := url.Values{} + params.Set("_method", "POST") + params.Set("_csrfToken", csrfToken) + params.Set("ad_form_data", ad_form_data) + params.Set("_Token[fields]", token_fields) + params.Set("_Token[unlocked]", token_unlocked) + cookies = append(cookies, &http.Cookie{ + Name: "ab", + Value: "2", + }) + + resp, err = Request().SetHeaders(map[string]string{ + "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", + "X-Requested-With": "XMLHttpRequest", + "Referer": URL, + }).SetCookies(cookies).SetBody(params.Encode()).Post("https://mgnet.site/links/go") + if err != nil { + return "", fmt.Errorf("Error while requesting URL: %s: %s", err, "https://mgnet.site/links/go") + } + + type requestResult struct { + Status string `json:"status"` + Message string `json:"message"` + URL string `json:"url"` + } + + res := requestResult{} + err = json.Unmarshal(resp.Body(), &res) + if err != nil { + return "", fmt.Errorf("Error while parsing JSON: %s", err) + } + if res.Status != "success" { + return "", fmt.Errorf("Failed to get link: %s: %s: %+v", res.Message, URL, res) + } + return res.URL, nil +} diff --git a/utils/ouo.go b/utils/ouo.go index aebc995..39c0d75 100644 --- a/utils/ouo.go +++ b/utils/ouo.go @@ -15,7 +15,7 @@ import ( "strings" ) -func OuoBypass(ouoURL string) (string, error) { +func GetLinkFromOUO(ouoURL string) (string, error) { tempURL := strings.Replace(ouoURL, "ouo.press", "ouo.io", 1) var res string u, err := url.Parse(tempURL)