nite/game-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

u

Browse Source
This commit is contained in:
Nite07 2024-12-28 00:10:06 +08:00
parent 29dd7fc058
commit cd9b7412b8
21 changed files with 324 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/server.go
View File

@ -23,7 +23,7 @@ var serverCmdCfg serverCommandConfig
func init() {
serverCmd.Flags().StringVarP(&serverCmdCfg.Port, "port", "p", "8080", "server port")
serverCmd.Flags().BoolVarP(&serverCmdCfg.AutoCrawl, "auto-crawl", "c", true, "enable auto crawl")
serverCmd.Flags().BoolVarP(&serverCmdCfg.AutoCrawl, "auto-crawl", "c", false, "enable auto crawl")
RootCmd.AddCommand(serverCmd)
}

6
crawler/1337x.go
View File

@ -98,7 +98,7 @@ func (c *s1337xCrawler) Crawl(page int) ([]*model.GameItem, error) {
}
func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL))
c.logger.Info("Crawling game", zap.String("URL", URL))
resp, err := utils.Request().Get(URL)
if err != nil {
c.logger.Error("Failed to fetch URL", zap.String("URL", URL), zap.Error(err))
@ -136,7 +136,9 @@ func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.RawName = strings.Replace(item.RawName, "Download ", "", 1)
item.RawName = strings.TrimSpace(strings.Replace(item.RawName, "Torrent | 1337x", " ", 1))
item.Name = c.formatter(item.RawName)
item.DownloadLinks = []string{magnetRegexRes[0]}
item.Downloads = map[string]string{
"magnet": magnetRegexRes[0],
}
item.Author = strings.Replace(c.source, "-torrents", "", -1)
item.Platform = c.platform

6
crawler/chovka.go
View File

@ -31,7 +31,7 @@ func (c *ChovkaCrawler) Name() string {
}
func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL))
c.logger.Info("Crawling game", zap.String("URL", URL))
resp, err := utils.Request().Get(URL)
if err != nil {
c.logger.Error("Failed to fetch URL", zap.String("URL", URL), zap.Error(err))
@ -75,7 +75,9 @@ func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
}
item.Size = size
item.DownloadLinks = []string{magnet}
item.Downloads = map[string]string{
"magnet": magnet,
}
c.logger.Info("Successfully crawled URL", zap.String("URL", URL))
return item, nil
}

6
crawler/fitgirl.go
View File

@ -32,7 +32,7 @@ func (c *FitGirlCrawler) Name() string {
}
func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL))
c.logger.Info("Crawling game", zap.String("URL", URL))
resp, err := utils.Request().Get(URL)
if err != nil {
c.logger.Error("Failed to fetch URL", zap.String("URL", URL), zap.Error(err))
@ -82,7 +82,9 @@ func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.Url = URL
item.Size = size
item.Author = "FitGirl"
item.DownloadLinks = []string{magnet}
item.Downloads = map[string]string{
"magnet": magnet,
}
item.Platform = "windows"
c.logger.Info("Successfully crawled URL", zap.String("URL", URL))

7
crawler/freegog.go
View File

@ -34,7 +34,6 @@ func NewFreeGOGCrawler(cfClearanceUrl string, logger *zap.Logger) *FreeGOGCrawle
}
func (c *FreeGOGCrawler) getSession() (*ccs.Session, error) {
c.logger.Info("Fetching session for FreeGOGCrawler")
cacheKey := "freegog_waf_session"
var session ccs.Session
if val, exist := cache.Get(cacheKey); exist {
@ -134,7 +133,7 @@ func (c *FreeGOGCrawler) Crawl(num int) ([]*model.GameItem, error) {
}
func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Info("Starting CrawlByUrl", zap.String("URL", URL))
c.logger.Info("Crawling game", zap.String("URL", URL))
session, err := c.getSession()
if err != nil {
c.logger.Error("Failed to get session", zap.Error(err))
@ -186,7 +185,9 @@ func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Error("Failed to decode magnet link", zap.String("URL", URL), zap.Error(err))
return nil, fmt.Errorf("failed to decode magnet link on page %s: %w", URL, err)
}
item.DownloadLinks = []string{string(magnet)}
item.Downloads = map[string]string{
"magnet": string(magnet),
}
} else {
c.logger.Error("Failed to find magnet link", zap.String("URL", URL))
return nil, fmt.Errorf("failed to find magnet link on page %s", URL)

27
crawler/game.go
View File

@ -2,6 +2,7 @@ package crawler
import (
"errors"
"fmt"
"regexp"
"strings"
"time"
@ -10,8 +11,6 @@ import (
"game-crawler/model"
"game-crawler/utils"
"go.uber.org/zap"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
)
@ -45,8 +44,6 @@ func OrganizeGameItem(game *model.GameItem) error {
steamID, err := GetSteamIDByIGDBID(item.IGDBID)
if err == nil {
item.SteamID = steamID
} else {
return err
}
}
@ -123,12 +120,9 @@ func FormatName(name string) string {
// SupplementPlatformIDToGameInfo supplements missing platform IDs (SteamID or IGDBID) for all game info entries.
func SupplementPlatformIDToGameInfo() error {
logger := zap.L()
logger.Info("Starting to supplement missing platform IDs")
infos, err := db.GetAllGameInfos()
if err != nil {
logger.Error("Failed to fetch game infos", zap.Error(err))
return err
return fmt.Errorf("failed to fetch game infos: %w", err)
}
for _, info := range infos {
@ -141,8 +135,6 @@ func SupplementPlatformIDToGameInfo() error {
if err == nil {
info.SteamID = steamID
changed = true
} else {
logger.Warn("Failed to get SteamID from IGDB", zap.Int("IGDBID", info.IGDBID), zap.Error(err))
}
}
@ -153,18 +145,13 @@ func SupplementPlatformIDToGameInfo() error {
if err == nil {
info.IGDBID = igdbID
changed = true
} else {
logger.Warn("Failed to get IGDBID from SteamID", zap.Int("SteamID", info.SteamID), zap.Error(err))
}
}
if changed {
logger.Info("Supplemented platform IDs", zap.String("Name", info.Name), zap.Int("IGDBID", info.IGDBID), zap.Int("SteamID", info.SteamID))
if err := db.SaveGameInfo(info); err != nil {
logger.Error("Failed to save updated game info", zap.String("Name", info.Name), zap.Error(err))
return fmt.Errorf("failed to save game info: %w", err)
}
} else {
logger.Info("No changes needed", zap.String("Name", info.Name), zap.Int("IGDBID", info.IGDBID), zap.Int("SteamID", info.SteamID))
}
}
return nil
@ -172,12 +159,9 @@ func SupplementPlatformIDToGameInfo() error {
// UpdateGameInfo updates outdated game info entries and returns a channel to monitor updates.
func UpdateGameInfo(num int) (chan *model.GameInfo, error) {
logger := zap.L()
logger.Info("Starting to update outdated game info", zap.Int("Num", num))
infos, err := db.GetOutdatedGameInfos(num)
if err != nil {
logger.Error("Failed to fetch outdated game infos", zap.Error(err))
return nil, err
return nil, fmt.Errorf("failed to fetch outdated game infos: %w", err)
}
updateChan := make(chan *model.GameInfo)
@ -188,18 +172,15 @@ func UpdateGameInfo(num int) (chan *model.GameInfo, error) {
if info.IGDBID != 0 {
newInfo, err := GenerateIGDBGameInfo(info.IGDBID)
if err != nil {
logger.Warn("Failed to generate IGDB game info", zap.Int("IGDBID", info.IGDBID), zap.Error(err))
continue
}
db.MergeGameInfo(info, newInfo)
if err := db.SaveGameInfo(newInfo); err != nil {
logger.Error("Failed to save updated game info", zap.String("Name", newInfo.Name), zap.Error(err))
continue
}
updateChan <- newInfo
logger.Info("Updated game info", zap.String("Name", newInfo.Name), zap.Int("IGDBID", newInfo.IGDBID))
}
}
}()

90
crawler/goggames.go
View File

@ -45,45 +45,39 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
token, err := ccs.TurnstileToken(c.cfClearanceUrl, apiUrl, "0x4AAAAAAAfOlgvCKbOdW1zc")
if err != nil {
c.logger.Error("Failed to get Turnstile token", zap.Error(err), zap.String("apiUrl", apiUrl))
return nil, fmt.Errorf("failed to get Turnstile token for URL %s: %w", apiUrl, err)
c.logger.Error("Failed to get Turnstile token", zap.Error(err), zap.String("URL", URL))
return nil, fmt.Errorf("failed to get Turnstile token for URL %s: %w", URL, err)
}
resp, err := utils.Request().SetHeader("cf-turnstile-response", token).Get(apiUrl)
if err != nil {
c.logger.Error("Failed to fetch data from API", zap.Error(err), zap.String("apiUrl", apiUrl))
return nil, fmt.Errorf("failed to fetch API data for URL %s: %w", apiUrl, err)
c.logger.Error("Failed to fetch data from API", zap.Error(err), zap.String("URL", URL))
return nil, fmt.Errorf("failed to fetch API data for URL %s: %w", URL, err)
}
data := gameResult{}
err = json.Unmarshal(resp.Body(), &data)
if err != nil {
c.logger.Error("Failed to unmarshal API response", zap.Error(err), zap.String("apiUrl", apiUrl))
return nil, fmt.Errorf("failed to parse API response for URL %s: %w", apiUrl, err)
c.logger.Error("Failed to unmarshal API response", zap.Error(err), zap.String("URL", URL))
return nil, fmt.Errorf("failed to parse API response for URL %s: %w", URL, err)
}
name := data.Title
// Find download links
fileHosters := []string{
"gofile",
"fileditch",
"qiwi",
"filesfm",
"pixeldrain",
"1fichier",
}
links := make([]string, 0)
for _, h := range fileHosters {
if value, exist := data.Links.Game[h]; exist {
for _, link := range value.Links {
links = append(links, link.Link)
}
links := make(map[string]string, 0)
for _, v := range data.Links.Game {
for _, link := range v.Links {
links[fmt.Sprintf("%s(%s)", link.Label, v.Name)] = link.Link
}
if value, exist := data.Links.Patch[h]; exist {
for _, link := range value.Links {
links = append(links, link.Link)
}
}
for _, v := range data.Links.Patch {
for _, link := range v.Links {
links[fmt.Sprintf("%s(%s)", link.Label, v.Name)] = link.Link
}
}
for _, v := range data.Links.Goodie {
for _, link := range v.Links {
links[fmt.Sprintf("%s(%s)", link.Label, v.Name)] = link.Link
}
}
@ -112,7 +106,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.Name = name
item.RawName = name
item.DownloadLinks = links
item.Downloads = links
item.Url = URL
item.Size = utils.BytesToSize(size)
item.Author = "GOGGames"
@ -138,6 +132,10 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) {
urls := make([]string, 0)
var updateFlags []string // link+date
for _, item := range data.Data {
if item.Infohash == "" {
// skip unreleased games
continue
}
urls = append(urls, fmt.Sprintf(constant.GOGGamesPageURL, item.Slug))
updateFlags = append(updateFlags, base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s%s", item.GogURL, item.LastUpdate))))
}
@ -279,39 +277,13 @@ type gameResult struct {
Md5Filename string `json:"md5_filename"`
Infohash string `json:"infohash"`
Links struct {
Goodie struct {
OneFichier struct {
ID string `json:"id"`
Name string `json:"name"`
Links []struct {
Label string `json:"label"`
Link string `json:"link"`
} `json:"links"`
} `json:"1fichier"`
Vikingfile struct {
ID string `json:"id"`
Name string `json:"name"`
Links []struct {
Label string `json:"label"`
Link string `json:"link"`
} `json:"links"`
} `json:"vikingfile"`
Pixeldrain struct {
ID string `json:"id"`
Name string `json:"name"`
Links []struct {
Label string `json:"label"`
Link string `json:"link"`
} `json:"links"`
} `json:"pixeldrain"`
Gofile struct {
ID string `json:"id"`
Name string `json:"name"`
Links []struct {
Label string `json:"label"`
Link string `json:"link"`
} `json:"links"`
} `json:"gofile"`
Goodie map[string]struct {
ID string `json:"id"`
Name string `json:"name"`
Links []struct {
Label string `json:"label"`
Link string `json:"link"`
} `json:"links"`
} `json:"goodie"`
Game map[string]struct {
ID string `json:"id"`

112
crawler/igdb.go
View File

@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"net/url"
"runtime/debug"
"strconv"
"strings"
"time"
@ -20,7 +19,6 @@ import (
"github.com/PuerkitoBio/goquery"
"github.com/go-resty/resty/v2"
"go.uber.org/zap"
)
type twitchToken struct {
@ -34,13 +32,9 @@ func (t *twitchToken) getToken() (string, error) {
}
token, expires, err := loginTwitch()
if err != nil {
zap.L().Error("failed to login to Twitch", zap.Error(err))
return "", fmt.Errorf("failed to login twitch: %w", err)
}
err = cache.SetWithExpire("twitch_token", token, expires)
if err != nil {
zap.L().Error("failed to set Twitch token in cache", zap.Error(err))
}
_ = cache.SetWithExpire("twitch_token", token, expires)
return token, nil
}
@ -54,8 +48,7 @@ func loginTwitch() (string, time.Duration, error) {
resp, err := utils.Request().SetHeader("User-Agent", "").Post(baseURL.String())
if err != nil {
zap.L().Error("failed to make Twitch login request", zap.String("url", baseURL.String()), zap.Error(err))
return "", 0, err
return "", 0, fmt.Errorf("failed to make request: %s: %w", baseURL.String(), err)
}
data := struct {
@ -65,8 +58,7 @@ func loginTwitch() (string, time.Duration, error) {
}{}
err = json.Unmarshal(resp.Body(), &data)
if err != nil {
zap.L().Error("failed to parse Twitch login response", zap.String("response", string(resp.Body())), zap.Error(err))
return "", 0, err
return "", 0, fmt.Errorf("failed to parse response: %w", err)
}
return data.AccessToken, time.Second * time.Duration(data.ExpiresIn), nil
}
@ -74,8 +66,7 @@ func loginTwitch() (string, time.Duration, error) {
func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
t, err := token.getToken()
if err != nil {
zap.L().Error("failed to get Twitch token", zap.Error(err))
return nil, err
return nil, fmt.Errorf("failed to get Twitch token: %w", err)
}
resp, err := utils.Request().SetBody(dataBody).SetHeaders(map[string]string{
@ -86,8 +77,7 @@ func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
}).Post(URL)
if err != nil {
zap.L().Error("failed to make IGDB request", zap.String("url", URL), zap.Any("dataBody", dataBody), zap.Error(err))
return nil, err
return nil, fmt.Errorf("failed to make request: %s: %w", URL, err)
}
return resp, nil
}
@ -95,22 +85,19 @@ func igdbRequest(URL string, dataBody any) (*resty.Response, error) {
func getIGDBID(name string) (int, error) {
resp, err := igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50; where game.platforms = [6] | game.platforms=[130] | game.platforms=[384] | game.platforms=[163];`, name))
if err != nil {
zap.L().Error("failed to search IGDB ID", zap.String("name", name), zap.Error(err))
return 0, err
return 0, fmt.Errorf("failed to search IGDB ID: %s: %w", name, err)
}
if string(resp.Body()) == "[]" {
resp, err = igdbRequest(constant.IGDBSearchURL, fmt.Sprintf(`search "%s"; fields *; limit 50;`, name))
if err != nil {
zap.L().Error("failed to fallback search IGDB ID", zap.String("name", name), zap.Error(err))
return 0, err
return 0, fmt.Errorf("failed to fallback search: %s: %w", name, err)
}
}
var data model.IGDBSearches
if err = json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("failed to unmarshal IGDB search response", zap.String("response", string(resp.Body())), zap.Error(err))
return 0, fmt.Errorf("failed to unmarshal: %w, %s", err, debug.Stack())
return 0, fmt.Errorf("failed to parse IGDB search response: %w", err)
}
if len(data) == 1 {
@ -132,8 +119,7 @@ func getIGDBID(name string) (int, error) {
detail, err := GetIGDBAppDetail(item.Game)
if err != nil {
zap.L().Error("failed to get IGDB app detail", zap.Int("gameID", item.Game), zap.Error(err))
return 0, err
return 0, fmt.Errorf("failed to get IGDB app detail: %d: %w", item.Game, err)
}
for _, altName := range detail.AlternativeNames {
if sim := utils.Similarity(altName.Name, name); sim >= 0.8 {
@ -149,7 +135,6 @@ func getIGDBID(name string) (int, error) {
return GetIGDBAppParent(data[maxSimilarityIndex].Game)
}
zap.L().Warn("no IGDB ID found", zap.String("name", name))
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
@ -235,33 +220,27 @@ func GetIGDBAppParent(id int) (int, error) {
if exist {
id, err := strconv.Atoi(val)
if err != nil {
zap.L().Error("failed to parse cached IGDB parent ID", zap.String("cacheKey", key), zap.Error(err))
return 0, err
return 0, fmt.Errorf("failed to parse cached IGDB parent ID: %s: %w", key, err)
}
return id, nil
}
detail, err := GetIGDBAppDetail(id)
if err != nil {
zap.L().Error("failed to fetch IGDB app detail for parent", zap.Int("gameID", id), zap.Error(err))
return 0, err
return 0, fmt.Errorf("failed to fetch IGDB app detail for parent: %d: %w", id, err)
}
hasParent := false
for detail.VersionParent != 0 {
hasParent = true
detail, err = GetIGDBAppDetail(detail.VersionParent)
if err != nil {
zap.L().Error("failed to fetch IGDB version parent", zap.Int("parentID", detail.VersionParent), zap.Error(err))
return 0, err
return 0, fmt.Errorf("failed to fetch IGDB version parent: %d: %w", detail.VersionParent, err)
}
}
if hasParent {
return detail.ID, nil
}
err = cache.Set(key, id)
if err != nil {
zap.L().Error("failed to cache IGDB parent ID", zap.String("cacheKey", key), zap.Error(err))
}
_ = cache.Set(key, id)
return id, nil
}
@ -270,7 +249,6 @@ func GetIGDBAppParent(id int) (int, error) {
func GetIGDBID(name string) (int, error) {
key := fmt.Sprintf("igdb_id:%s", name)
if val, exist := cache.Get(key); exist {
zap.L().Info("cache hit for IGDB ID", zap.String("name", name), zap.String("cacheKey", key))
return strconv.Atoi(val)
}
@ -279,10 +257,7 @@ func GetIGDBID(name string) (int, error) {
for _, n := range normalizedNames {
id, err := getIGDBID(n)
if err == nil {
cacheErr := cache.Set(key, id)
if cacheErr != nil {
zap.L().Warn("failed to cache IGDB ID", zap.String("name", n), zap.Error(cacheErr))
}
_ = cache.Set(key, id)
return id, nil
}
}
@ -291,16 +266,12 @@ func GetIGDBID(name string) (int, error) {
for _, n := range normalizedNames {
id, err := getIGDBIDBySteamSearch(n)
if err == nil {
cacheErr := cache.Set(key, id)
if cacheErr != nil {
zap.L().Warn("failed to cache IGDB ID after Steam search", zap.String("name", n), zap.Error(cacheErr))
}
_ = cache.Set(key, id)
return id, nil
}
}
zap.L().Warn("failed to retrieve IGDB ID", zap.String("name", name))
return 0, fmt.Errorf("IGDB ID not found for '%s'", name)
return 0, fmt.Errorf("IGDB ID not found: %s", name)
}
func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
@ -309,27 +280,23 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
if exist {
var data model.IGDBGameDetail
if err := json.Unmarshal([]byte(val), &data); err != nil {
zap.L().Error("failed to parse cached IGDB game detail", zap.String("cacheKey", key), zap.Error(err))
return nil, err
return nil, fmt.Errorf("failed to parse cached IGDB game detail: %s: %w", key, err)
}
return &data, nil
}
resp, err := igdbRequest(constant.IGDBGameURL, fmt.Sprintf(`where id = %v; fields *,alternative_names.*,language_supports.*,screenshots.*,cover.*,involved_companies.*,game_engines.*,game_modes.*,genres.*,player_perspectives.*,release_dates.*,videos.*,websites.*,platforms.*,themes.*,collections.*;`, id))
if err != nil {
zap.L().Error("failed to fetch IGDB game detail", zap.Int("gameID", id), zap.Error(err))
return nil, err
return nil, fmt.Errorf("failed to fetch IGDB game detail for ID %d: %w", id, err)
}
var data model.IGDBGameDetails
if err = json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("failed to unmarshal IGDB game detail response", zap.String("response", string(resp.Body())), zap.Error(err))
return nil, err
return nil, fmt.Errorf("failed to unmarshal IGDB game detail response: %w", err)
}
if len(data) == 0 {
zap.L().Warn("IGDB game not found", zap.Int("gameID", id))
return nil, errors.New("IGDB App not found")
return nil, fmt.Errorf("IGDB game detail not found for ID %d", id)
}
if data[0].Name == "" {
@ -338,10 +305,7 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
jsonBytes, err := json.Marshal(data[0])
if err == nil {
err = cache.Set(key, string(jsonBytes))
if err != nil {
zap.L().Error("failed to cache IGDB game detail", zap.String("cacheKey", key), zap.Error(err))
}
_ = cache.Set(key, string(jsonBytes))
}
return data[0], nil
@ -351,33 +315,26 @@ func GetIGDBAppDetail(id int) (*model.IGDBGameDetail, error) {
func GetIGDBCompany(id int) (string, error) {
key := fmt.Sprintf("igdb_companies:%d", id)
if val, exist := cache.Get(key); exist {
zap.L().Info("cache hit for IGDB company", zap.Int("companyID", id), zap.String("cacheKey", key))
return val, nil
}
query := fmt.Sprintf(`where id=%d; fields *;`, id)
resp, err := igdbRequest(constant.IGDBCompaniesURL, query)
if err != nil {
zap.L().Error("failed to fetch IGDB company", zap.Int("companyID", id), zap.Error(err))
return "", fmt.Errorf("failed to fetch IGDB company for ID %d: %w", id, err)
}
var data model.IGDBCompanies
if err = json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("failed to unmarshal IGDB company response", zap.String("response", string(resp.Body())), zap.Error(err))
return "", fmt.Errorf("failed to unmarshal IGDB companies response: %w", err)
}
if len(data) == 0 {
zap.L().Warn("no company found in IGDB for ID", zap.Int("companyID", id))
return "", errors.New("company not found")
}
companyName := data[0].Name
cacheErr := cache.Set(key, companyName)
if cacheErr != nil {
zap.L().Warn("failed to cache IGDB company name", zap.Int("companyID", id), zap.Error(cacheErr))
}
_ = cache.Set(key, companyName)
return companyName, nil
}
@ -385,7 +342,6 @@ func GetIGDBCompany(id int) (string, error) {
func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
detail, err := GetIGDBAppDetail(id)
if err != nil {
zap.L().Error("failed to fetch IGDB app detail", zap.Int("igdbID", id), zap.Error(err))
return nil, fmt.Errorf("failed to fetch IGDB app detail for ID %d: %w", id, err)
}
@ -415,7 +371,6 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
for _, company := range detail.InvolvedCompanies {
companyName, err := GetIGDBCompany(company.Company)
if err != nil {
zap.L().Warn("failed to fetch company name", zap.Int("companyID", company.Company), zap.Error(err))
continue
}
if company.Developer {
@ -445,7 +400,6 @@ func GenerateIGDBGameInfo(id int) (*model.GameInfo, error) {
func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
id, err := GetIGDBID(game.Name)
if err != nil {
zap.L().Error("failed to get IGDB ID for game", zap.String("gameName", game.Name), zap.Error(err))
return nil, fmt.Errorf("failed to get IGDB ID for game '%s': %w", game.Name, err)
}
@ -457,7 +411,6 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
info, err = GenerateIGDBGameInfo(id)
if err != nil {
zap.L().Error("failed to generate IGDB game info", zap.Int("igdbID", id), zap.Error(err))
return nil, fmt.Errorf("failed to generate IGDB game info for ID %d: %w", id, err)
}
@ -469,14 +422,12 @@ func OrganizeGameItemWithIGDB(game *model.GameItem) (*model.GameInfo, error) {
func GetIGDBIDBySteamAppID(id int) (int, error) {
key := fmt.Sprintf("igdb_id_by_steam_app_id:%d", id)
if val, exist := cache.Get(key); exist {
zap.L().Info("cache hit for IGDB ID by Steam App ID", zap.Int("steamAppID", id), zap.String("cacheKey", key))
return strconv.Atoi(val)
}
query := fmt.Sprintf(`where url = "https://store.steampowered.com/app/%d" | url = "https://store.steampowered.com/app/%d/"; fields game;`, id, id)
resp, err := igdbRequest(constant.IGDBWebsitesURL, query)
if err != nil {
zap.L().Error("failed to fetch IGDB ID by Steam App ID", zap.Int("steamAppID", id), zap.Error(err))
return 0, fmt.Errorf("failed to fetch IGDB ID by Steam App ID %d: %w", id, err)
}
@ -484,20 +435,15 @@ func GetIGDBIDBySteamAppID(id int) (int, error) {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("failed to unmarshal IGDB response", zap.String("response", string(resp.Body())), zap.Error(err))
return 0, fmt.Errorf("failed to unmarshal IGDB response for Steam App ID %d: %w", id, err)
}
if len(data) == 0 || data[0].Game == 0 {
zap.L().Warn("no matching IGDB game found for Steam App ID", zap.Int("steamAppID", id))
return 0, errors.New("no matching IGDB game found")
}
igdbID := data[0].Game
cacheErr := cache.Set(key, strconv.Itoa(igdbID))
if cacheErr != nil {
zap.L().Warn("failed to cache IGDB ID by Steam App ID", zap.Int("steamAppID", id), zap.Error(cacheErr))
}
_ = cache.Set(key, strconv.Itoa(igdbID))
return GetIGDBAppParent(igdbID)
}
@ -506,14 +452,12 @@ func GetIGDBIDBySteamAppID(id int) (int, error) {
func GetIGDBIDBySteamBundleID(id int) (int, error) {
key := fmt.Sprintf("igdb_id_by_steam_bundle_id:%d", id)
if val, exist := cache.Get(key); exist {
zap.L().Info("cache hit for IGDB ID by Steam Bundle ID", zap.Int("steamBundleID", id), zap.String("cacheKey", key))
return strconv.Atoi(val)
}
query := fmt.Sprintf(`where url = "https://store.steampowered.com/bundle/%d" | url = "https://store.steampowered.com/bundle/%d/"; fields game;`, id, id)
resp, err := igdbRequest(constant.IGDBWebsitesURL, query)
if err != nil {
zap.L().Error("failed to fetch IGDB ID by Steam Bundle ID", zap.Int("steamBundleID", id), zap.Error(err))
return 0, fmt.Errorf("failed to fetch IGDB ID by Steam Bundle ID %d: %w", id, err)
}
@ -521,20 +465,15 @@ func GetIGDBIDBySteamBundleID(id int) (int, error) {
Game int `json:"game"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("failed to unmarshal IGDB response", zap.String("response", string(resp.Body())), zap.Error(err))
return 0, fmt.Errorf("failed to unmarshal IGDB response for Steam Bundle ID %d: %w", id, err)
}
if len(data) == 0 || data[0].Game == 0 {
zap.L().Warn("no matching IGDB game found for Steam Bundle ID", zap.Int("steamBundleID", id))
return 0, errors.New("no matching IGDB game found")
}
igdbID := data[0].Game
cacheErr := cache.Set(key, strconv.Itoa(igdbID))
if cacheErr != nil {
zap.L().Warn("failed to cache IGDB ID by Steam Bundle ID", zap.Int("steamBundleID", id), zap.Error(cacheErr))
}
_ = cache.Set(key, strconv.Itoa(igdbID))
return GetIGDBAppParent(igdbID)
}
@ -548,7 +487,6 @@ func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) {
query := fmt.Sprintf("fields game_id,value,popularity_type; sort value desc; limit %d; offset %d; where popularity_type = %d;", limit, offset, popularityType)
resp, err := igdbRequest(constant.IGDBPopularityURL, query)
if err != nil {
zap.L().Error("failed to fetch popular IGDB game IDs", zap.Int("popularityType", popularityType), zap.Error(err))
return nil, fmt.Errorf("failed to fetch popular IGDB game IDs for type %d: %w", popularityType, err)
}
@ -557,7 +495,6 @@ func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) {
Value float64 `json:"value"`
}
if err = json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("failed to unmarshal IGDB popular games response", zap.String("response", string(resp.Body())), zap.Error(err))
return nil, fmt.Errorf("failed to unmarshal IGDB popular games response: %w", err)
}
@ -565,7 +502,6 @@ func GetIGDBPopularGameIDs(popularityType, offset, limit int) ([]int, error) {
for _, d := range data {
parentID, err := GetIGDBAppParent(d.GameID)
if err != nil {
zap.L().Warn("failed to fetch parent IGDB ID for game", zap.Int("gameID", d.GameID), zap.Error(err))
gameIDs = append(gameIDs, d.GameID)
} else {
gameIDs = append(gameIDs, parentID)

13
crawler/nxbrew.go Normal file
View File

@ -0,0 +1,13 @@
package crawler
import "go.uber.org/zap"
type NxbrewCrawler struct {
logger *zap.Logger
}
func NewNxbrewCrawler(logger *zap.Logger) *NxbrewCrawler {
return &NxbrewCrawler{
logger: logger,
}
}

4
crawler/onlinefix.go
View File

@ -176,7 +176,9 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, fmt.Errorf("failed to convert torrent to magnet: %w", err)
}
item.DownloadLinks = []string{magnet}
item.Downloads = map[string]string{
"magnet": magnet,
}
item.Size = size
} else {
c.logger.Warn("Unsupported download link format", zap.String("url", downloadURL))

19
crawler/rutracker.go
View File

@ -1,6 +1,7 @@
package crawler
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
@ -11,7 +12,6 @@ import (
"game-crawler/utils"
"net/url"
"strconv"
"strings"
"time"
"unicode"
@ -47,8 +47,6 @@ func NewRutrackerCrawler(source, platform, rid, username, password, cfClearanceU
}
func (r *RutrackerCrawler) getSession() (*ccs.Session, error) {
r.logger.Info("Fetching session for RutrackerCrawler")
if r.username == "" || r.password == "" {
r.logger.Error("Username or password is empty")
return nil, fmt.Errorf("username or password is empty")
@ -121,6 +119,7 @@ func (r *RutrackerCrawler) getSession() (*ccs.Session, error) {
}
func (r *RutrackerCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
r.logger.Info("Crawling game", zap.String("URL", URL))
session, err := r.getSession()
if err != nil {
return nil, fmt.Errorf("failed to get session: %w", err)
@ -132,7 +131,7 @@ func (r *RutrackerCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, fmt.Errorf("failed to request URL: %w", err)
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.Body))
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(utils.Windows1251ToUTF8([]byte(resp.Body))))
if err != nil {
r.logger.Error("Failed to parse HTML", zap.Error(err))
return nil, fmt.Errorf("failed to parse HTML: %w", err)
@ -155,8 +154,9 @@ func (r *RutrackerCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
r.logger.Error("Failed to find magnet link")
return nil, fmt.Errorf("failed to find magnet link")
}
item.DownloadLinks = []string{magnet}
item.Downloads = map[string]string{
"magnet": magnet,
}
sizeStr := doc.Find("#tor-size-humn").AttrOr("title", "")
if sizeStr == "" {
r.logger.Warn("Failed to find size")
@ -187,7 +187,7 @@ func (r *RutrackerCrawler) Crawl(page int) ([]*model.GameItem, error) {
return nil, fmt.Errorf("failed to request URL: %w", err)
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.Body))
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(utils.Windows1251ToUTF8([]byte(resp.Body))))
if err != nil {
r.logger.Error("Failed to parse HTML", zap.Error(err))
return nil, fmt.Errorf("failed to parse HTML: %w", err)
@ -195,7 +195,7 @@ func (r *RutrackerCrawler) Crawl(page int) ([]*model.GameItem, error) {
var urls []string
var updateFlags []string
doc.Find("[id^='trs-tr']").Each(func(i int, s *goquery.Selection) {
a := s.Find(".t-title")
a := s.Find(".t-title a")
datetime := s.Find("td").Last().Text()
url, exists := a.Attr("href")
if !exists {
@ -219,6 +219,7 @@ func (r *RutrackerCrawler) Crawl(page int) ([]*model.GameItem, error) {
r.logger.Error("Failed to crawl URL", zap.String("URL", URL), zap.Error(err))
continue
}
item.UpdateFlag = updateFlags[i]
err = db.SaveGameItem(item)
if err != nil {
r.logger.Error("Failed to save game item to database", zap.String("URL", URL), zap.Error(err))
@ -264,7 +265,7 @@ func (r *RutrackerCrawler) GetTotalPageNum() (int, error) {
return 0, fmt.Errorf("failed to request URL: %w", err)
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(resp.Body))
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(utils.Windows1251ToUTF8([]byte(resp.Body))))
if err != nil {
r.logger.Error("Failed to parse HTML", zap.Error(err))
return 0, fmt.Errorf("failed to parse HTML: %w", err)

20
crawler/steam.go
View File

@ -13,18 +13,14 @@ import (
"game-crawler/constant"
"game-crawler/model"
"game-crawler/utils"
"go.uber.org/zap"
)
// GetSteamAppDetail fetches the details of a Steam app by its ID.
func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
key := fmt.Sprintf("steam_game:%d", id)
if val, exist := cache.Get(key); exist {
zap.L().Info("Cache hit for Steam app detail", zap.Int("steamID", id))
var detail model.SteamAppDetail
if err := json.Unmarshal([]byte(val), &detail); err != nil {
zap.L().Warn("Failed to unmarshal cached Steam app detail", zap.Int("steamID", id), zap.Error(err))
return nil, fmt.Errorf("failed to unmarshal cached Steam app detail for ID %d: %w", id, err)
}
return &detail, nil
@ -39,18 +35,15 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
"User-Agent": "",
}).Get(baseURL.String())
if err != nil {
zap.L().Error("Failed to fetch Steam app detail", zap.Int("steamID", id), zap.String("url", baseURL.String()), zap.Error(err))
return nil, fmt.Errorf("failed to fetch Steam app detail for ID %d: %w", id, err)
}
var detail map[string]*model.SteamAppDetail
if err := json.Unmarshal(resp.Body(), &detail); err != nil {
zap.L().Error("Failed to unmarshal Steam app detail response", zap.Int("steamID", id), zap.String("response", string(resp.Body())), zap.Error(err))
return nil, fmt.Errorf("failed to unmarshal Steam app detail for ID %d: %w", id, err)
}
if appDetail, ok := detail[strconv.Itoa(id)]; !ok || appDetail == nil {
zap.L().Warn("Steam app detail not found", zap.Int("steamID", id))
return nil, fmt.Errorf("steam app not found: %d", id)
} else {
// Cache the result
@ -66,7 +59,6 @@ func GetSteamAppDetail(id int) (*model.SteamAppDetail, error) {
func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
detail, err := GetSteamAppDetail(id)
if err != nil {
zap.L().Error("Failed to fetch Steam app detail for game info generation", zap.Int("steamID", id), zap.Error(err))
return nil, fmt.Errorf("failed to fetch Steam app detail for ID %d: %w", id, err)
}
@ -84,7 +76,6 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
item.Screenshots = append(item.Screenshots, screenshot.PathFull)
}
zap.L().Info("Generated Steam game info", zap.Int("steamID", id), zap.String("name", item.Name))
return item, nil
}
@ -92,10 +83,8 @@ func GenerateSteamGameInfo(id int) (*model.GameInfo, error) {
func GetSteamIDByIGDBID(IGDBID int) (int, error) {
key := fmt.Sprintf("steam_game:%d", IGDBID)
if val, exist := cache.Get(key); exist {
zap.L().Info("Cache hit for Steam ID by IGDB ID", zap.Int("IGDBID", IGDBID))
id, err := strconv.Atoi(val)
if err != nil {
zap.L().Warn("Failed to parse cached Steam ID", zap.Int("IGDBID", IGDBID), zap.Error(err))
return 0, fmt.Errorf("failed to parse cached Steam ID for IGDB ID %d: %w", IGDBID, err)
}
return id, nil
@ -104,7 +93,6 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) {
query := fmt.Sprintf(`where game = %v; fields *; limit 500;`, IGDBID)
resp, err := igdbRequest(constant.IGDBWebsitesURL, query)
if err != nil {
zap.L().Error("Failed to fetch IGDB websites for Steam ID", zap.Int("IGDBID", IGDBID), zap.Error(err))
return 0, fmt.Errorf("failed to fetch IGDB websites for IGDB ID %d: %w", IGDBID, err)
}
@ -113,12 +101,10 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) {
Url string `json:"url"`
}
if err := json.Unmarshal(resp.Body(), &data); err != nil {
zap.L().Error("Failed to unmarshal IGDB websites response", zap.Int("IGDBID", IGDBID), zap.String("response", string(resp.Body())), zap.Error(err))
return 0, fmt.Errorf("failed to unmarshal IGDB websites response for IGDB ID %d: %w", IGDBID, err)
}
if len(data) == 0 {
zap.L().Warn("No Steam ID found for IGDB ID", zap.Int("IGDBID", IGDBID))
return 0, errors.New("steam ID not found")
}
@ -127,23 +113,19 @@ func GetSteamIDByIGDBID(IGDBID int) (int, error) {
regex := regexp.MustCompile(`https://store.steampowered.com/app/(\d+)/?`)
idMatch := regex.FindStringSubmatch(v.Url)
if len(idMatch) < 2 {
zap.L().Warn("Failed to parse Steam ID from URL", zap.String("url", v.Url))
return 0, errors.New("failed to parse Steam ID from URL")
}
steamID, err := strconv.Atoi(idMatch[1])
if err != nil {
zap.L().Error("Failed to convert Steam ID to integer", zap.String("url", v.Url), zap.Error(err))
return 0, fmt.Errorf("failed to convert Steam ID from URL %s: %w", v.Url, err)
}
// Cache the result
_ = cache.Set(key, strconv.Itoa(steamID))
zap.L().Info("Found Steam ID for IGDB ID", zap.Int("IGDBID", IGDBID), zap.Int("steamID", steamID))
return steamID, nil
}
}
zap.L().Warn("No valid Steam ID found in IGDB websites data", zap.Int("IGDBID", IGDBID))
return 0, errors.New("steam ID not found")
return 0, fmt.Errorf("no valid Steam ID found for IGDB ID %d", IGDBID)
}

25
crawler/steam250.go
View File

@ -16,32 +16,26 @@ import (
"game-crawler/utils"
"github.com/PuerkitoBio/goquery"
"go.uber.org/zap"
)
// GetSteam250 fetches Steam250 game rankings from the given URL.
func GetSteam250(URL string) ([]*model.GameInfo, error) {
key := "steam250:" + url.QueryEscape(URL)
if val, ok := cache.Get(key); ok {
zap.L().Info("Cache hit for Steam250 rankings", zap.String("url", URL))
var infos []*model.GameInfo
if err := json.Unmarshal([]byte(val), &infos); err != nil {
zap.L().Warn("Failed to unmarshal cached Steam250 data", zap.String("url", URL), zap.Error(err))
return nil, fmt.Errorf("failed to unmarshal cached Steam250 data for URL %s: %w", URL, err)
}
return infos, nil
}
zap.L().Info("Fetching Steam250 rankings from URL", zap.String("url", URL))
resp, err := utils.Request().Get(URL)
if err != nil {
zap.L().Error("Failed to fetch Steam250 rankings", zap.String("url", URL), zap.Error(err))
return nil, fmt.Errorf("failed to fetch Steam250 rankings from URL %s: %w", URL, err)
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
if err != nil {
zap.L().Error("Failed to parse Steam250 HTML document", zap.String("url", URL), zap.Error(err))
return nil, fmt.Errorf("failed to parse Steam250 HTML document for URL %s: %w", URL, err)
}
@ -53,7 +47,6 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
// Extract game name
item.Name = s.Find(".title>a").First().Text()
if item.Name == "" {
zap.L().Warn("Game name not found in Steam250 rankings", zap.String("url", URL), zap.Int("index", i))
return
}
@ -61,13 +54,11 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
idStr := s.Find(".store").AttrOr("href", "")
idSlice := regexp.MustCompile(`app/(\d+)/`).FindStringSubmatch(idStr)
if len(idSlice) < 2 {
zap.L().Warn("Failed to extract Steam ID from URL", zap.String("url", idStr), zap.Int("index", i))
return
}
steamID, err := strconv.Atoi(idSlice[1])
if err != nil {
zap.L().Warn("Failed to convert Steam ID to integer", zap.String("id", idSlice[1]), zap.Error(err))
return
}
@ -77,15 +68,11 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
})
if len(steamIDs) == 0 {
zap.L().Warn("No valid Steam IDs found in Steam250 rankings", zap.String("url", URL))
return nil, fmt.Errorf("no valid Steam IDs found in Steam250 rankings for URL %s", URL)
}
// Fetch game info from the database
zap.L().Info("Fetching game info from database", zap.Ints("steamIDs", steamIDs))
infos, err := db.GetGameInfosByPlatformIDs("steam", steamIDs)
if err != nil {
zap.L().Error("Failed to fetch game info from database", zap.Ints("steamIDs", steamIDs), zap.Error(err))
return nil, fmt.Errorf("failed to fetch game info for Steam IDs %v: %w", steamIDs, err)
}
@ -97,12 +84,7 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
// Cache the result
jsonBytes, err := json.Marshal(infos)
if err == nil {
cacheErr := cache.SetWithExpire(key, string(jsonBytes), 12*time.Hour)
if cacheErr != nil {
zap.L().Warn("Failed to cache Steam250 rankings", zap.String("url", URL), zap.Error(cacheErr))
}
} else {
zap.L().Warn("Failed to marshal Steam250 rankings for caching", zap.String("url", URL), zap.Error(err))
_ = cache.SetWithExpire(key, string(jsonBytes), 24*time.Hour)
}
return infos, nil
@ -110,31 +92,26 @@ func GetSteam250(URL string) ([]*model.GameInfo, error) {
// GetSteam250Top250 retrieves the top 250 games from Steam250.
func GetSteam250Top250() ([]*model.GameInfo, error) {
zap.L().Info("Fetching Steam250 Top 250 games")
return GetSteam250(constant.Steam250Top250URL)
}
// GetSteam250BestOfTheYear retrieves the best games of the current year from Steam250.
func GetSteam250BestOfTheYear() ([]*model.GameInfo, error) {
year := time.Now().UTC().Year()
zap.L().Info("Fetching Steam250 Best of the Year games", zap.Int("year", year))
return GetSteam250(fmt.Sprintf(constant.Steam250BestOfTheYearURL, year))
}
// GetSteam250WeekTop50 retrieves the top 50 games of the week from Steam250.
func GetSteam250WeekTop50() ([]*model.GameInfo, error) {
zap.L().Info("Fetching Steam250 Week Top 50 games")
return GetSteam250(constant.Steam250WeekTop50URL)
}
// GetSteam250MonthTop50 retrieves the top 50 games of the month from Steam250.
func GetSteam250MonthTop50() ([]*model.GameInfo, error) {
zap.L().Info("Fetching Steam250 Month Top 50 games")
return GetSteam250(constant.Steam250MonthTop50URL)
}
// GetSteam250MostPlayed retrieves the most played games from Steam250.
func GetSteam250MostPlayed() ([]*model.GameInfo, error) {
zap.L().Info("Fetching Steam250 Most Played games")
return GetSteam250(constant.Steam250MostPlayedURL)
}

41
crawler/steamrip.go
View File

@ -5,6 +5,7 @@ import (
"encoding/base64"
"errors"
"fmt"
"net/url"
"regexp"
"strings"
@ -36,7 +37,7 @@ func (c *SteamRIPCrawler) Name() string {
// CrawlByUrl crawls a single game page from SteamRIP by URL.
func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Info("Crawling game details", zap.String("URL", URL))
c.logger.Info("Crawling game", zap.String("URL", URL))
// Fetch the page content
resp, err := utils.Request().Get(URL)
@ -78,42 +79,22 @@ func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.Size = "unknown"
}
// Extract download links
item.DownloadLinks = c.extractDownloadLinks(string(resp.Body()))
if len(item.DownloadLinks) == 0 {
downloadLinks := map[string]string{}
doc.Find(".shortc-button").Each(func(i int, s *goquery.Selection) {
downloadLink, _ := s.Attr("href")
u, _ := url.Parse(downloadLink)
downloadLinks[u.Host] = downloadLink
})
item.Downloads = downloadLinks
if len(item.Downloads) == 0 {
c.logger.Warn("No download links found", zap.String("URL", URL))
return nil, errors.New("failed to find download link")
}
c.logger.Info("Crawled game details successfully", zap.String("Name", item.Name), zap.String("URL", URL))
return item, nil
}
// extractDownloadLinks extracts download links from the game page HTML.
func (c *SteamRIPCrawler) extractDownloadLinks(pageContent string) []string {
var links []string
// Match MegaDB links
megadbRegex := regexp.MustCompile(`(?i)(?:https?:)?(//megadb\.net/[^"]+)`)
if matches := megadbRegex.FindStringSubmatch(pageContent); len(matches) > 1 {
links = append(links, fmt.Sprintf("https:%s", matches[1]))
}
// Match Gofile links
gofileRegex := regexp.MustCompile(`(?i)(?:https?:)?(//gofile\.io/d/[^"]+)`)
if matches := gofileRegex.FindStringSubmatch(pageContent); len(matches) > 1 {
links = append(links, fmt.Sprintf("https:%s", matches[1]))
}
// Match Filecrypt links
filecryptRegex := regexp.MustCompile(`(?i)(?:https?:)?(//filecrypt\.co/Container/[^"]+)`)
if matches := filecryptRegex.FindStringSubmatch(pageContent); len(matches) > 1 {
links = append(links, fmt.Sprintf("https:%s", matches[1]))
}
return links
}
// Crawl crawls a limited number of games from the SteamRIP game list.
func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
c.logger.Info("Starting SteamRIP crawl", zap.Int("limit", num))

7
crawler/xatab.go
View File

@ -98,7 +98,7 @@ func (c *XatabCrawler) Crawl(page int) ([]*model.GameItem, error) {
// CrawlByUrl crawls a single game page from Xatab by URL.
func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
c.logger.Info("Crawling game details", zap.String("URL", URL))
c.logger.Info("Crawling game", zap.String("URL", URL))
// Fetch the game page
resp, err := utils.Request().Get(URL)
@ -151,9 +151,10 @@ func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
}
item.Size = size
item.DownloadLinks = []string{magnet}
item.Downloads = map[string]string{
"magnet": magnet,
}
c.logger.Info("Crawled game details successfully", zap.String("Name", item.Name), zap.String("URL", URL))
return item, nil
}

12
db/game.go
View File

@ -475,10 +475,7 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
defer cancel()
type queryRes struct {
ID struct {
RawName string `bson:"raw_name"`
Download string `bson:"download"`
} `bson:"_id"`
ID string `bson:"_id"`
Count int `bson:"count"`
IDs []primitive.ObjectID `bson:"ids"`
}
@ -489,12 +486,7 @@ func DeduplicateGameItems() ([]primitive.ObjectID, error) {
bson.D{
{Key: "$group",
Value: bson.D{
{Key: "_id",
Value: bson.D{
{Key: "raw_name", Value: "$raw_name"},
{Key: "download", Value: "$download"},
},
},
{Key: "_id", Value: bson.D{{Key: "url", Value: "$url"}}},
{Key: "count", Value: bson.D{{Key: "$sum", Value: 1}}},
{Key: "ids", Value: bson.D{{Key: "$push", Value: "$_id"}}},
},

24
model/game.go
View File

@ -41,16 +41,16 @@ type GameCollection struct {
}
type GameItem struct {
ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"speculative_name" bson:"name"`
RawName string `json:"raw_name,omitempty" bson:"raw_name"`
DownloadLinks []string `json:"download_links,omitempty" bson:"download_links"`
Size string `json:"size,omitempty" bson:"size"`
Url string `json:"url" bson:"url"`
Password string `json:"password,omitempty" bson:"password"`
Author string `json:"author,omitempty" bson:"author"`
Platform string `json:"platform,omitempty" bson:"platform"`
UpdateFlag string `json:"update_flag,omitempty" bson:"update_flag"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"speculative_name" bson:"name"`
RawName string `json:"raw_name,omitempty" bson:"raw_name"`
Downloads map[string]string `json:"downloads,omitempty" bson:"downloads"`
Size string `json:"size,omitempty" bson:"size"`
Url string `json:"url" bson:"url"`
Password string `json:"password,omitempty" bson:"password"`
Author string `json:"author,omitempty" bson:"author"`
Platform string `json:"platform,omitempty" bson:"platform"`
UpdateFlag string `json:"update_flag,omitempty" bson:"update_flag"`
CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
}

180
server/templates/game.html
View File

@ -44,19 +44,12 @@
/* Masonry Container Styles */
.masonry-container {
column-count: 3;
column-count: 2;
/* 3 Columns for Masonry */
column-gap: 1rem;
/* Adjust Gap Between Columns */
}
@media (max-width: 992px) {
.masonry-container {
column-count: 2;
/* 2 Columns on Medium Screens */
}
}
@media (max-width: 576px) {
.masonry-container {
column-count: 1;
@ -100,19 +93,23 @@
</div>
{{end}}
{{if .Developers}}
<div>
<span class="info-label">Developers:</span>
{{range .Developers}}
<span class="tag">{{.}}</span>
{{end}}
</div>
{{end}}
{{if .Publishers}}
<div>
<span class="info-label">Publishers:</span>
{{range .Publishers}}
<span class="tag">{{.}}</span>
{{end}}
</div>
{{end}}
{{if .Languages}}
<div>
@ -121,17 +118,48 @@
<span class="tag">{{.}}</span>
{{end}}
</div>
{{end}} {{if .Description}}
{{end}}
{{if .Description}}
<div>
<p>{{.Description}}</p>
</div>
{{end}} {{if .SteamID}}
{{end}}
{{if .SteamID}}
<div>
<a href="https://store.steampowered.com/app/{{.SteamID}}" target="_blank" class="btn btn-primary">
Steam
</a>
</div>
{{end}}
{{if .GameEngines}}
<div>
<span class="info-label">Engines:</span>
{{range .GameEngines}}
<span class="tag">{{.}}</span>
{{end}}
</div>
{{end}}
{{if .Genres}}
<div>
<span class="info-label">Genres:</span>
{{range .Genres}}
<span class="tag">{{.}}</span>
{{end}}
</div>
{{end}}
{{if .Themes}}
<div>
<span class="info-label">Themes:</span>
{{range .Themes}}
<span class="tag">{{.}}</span>
{{end}}
</div>
{{end}}
</div>
</div>
@ -156,53 +184,66 @@
<!-- Download Links -->
{{if .Games}}
<div class="mb-4">
<h3 class="mb-3">Download</h3>
<div class="row">
<div class="col-12">
<!-- Masonry Container -->
<div class="masonry-container">
{{range .Games}}
<div class="card download-card mb-3">
<div class="card-body">
<h5 class="card-title"><a class="text-decoration-none" href="{{.Url}}">{{.RawName}}</a></h5>
{{if .Size}}
<div class="card-text">
<small class="text-muted">Size: {{.Size}}</small>
</div>
{{end}}
{{if .Author}}
<div class="card-text">
<small class="text-muted">Source: {{.Author}}</small>
</div>
{{end}}
{{if .Platform}}
<div class="card-text">
<small class="text-muted">Platform: {{.Platform}}</small>
</div>
{{end}}
{{if .Password}}
<div class="card-text">
<small class="text-muted">Unzip password: <code>{{.Password}}</code></small>
</div>
{{end}}
{{if .UpdatedAt}}
<div class="card-text">
<small class="text-muted">Updated: {{.UpdatedAt}}</small>
</div>
{{end}}
{{range .DownloadLinks}}
<div class="input-group mb-1" style="max-width: 300px;">
<input type="text" class="form-control form-control-sm" value="{{.}}" readonly>
<button class="btn btn-outline-secondary btn-sm" type="button"
onclick="copyToClipboard(this.previousElementSibling)">
Copy
</button>
</div>
{{end}}
<div class="col-12">
<!-- Masonry Container -->
<div class="masonry-container">
{{range .Games}}
<div class="card download-card mb-3">
<div class="card-body">
<h5 class="card-title"><a class="text-decoration-none" href="{{.Url}}">{{.RawName}}</a></h5>
{{if .Size}}
<div class="card-text">
<small class="text-muted">Size: {{.Size}}</small>
</div>
{{end}}
{{if .Author}}
<div class="card-text">
<small class="text-muted">Source: {{.Author}}</small>
</div>
{{end}}
{{if .Platform}}
<div class="card-text">
<small class="text-muted">Platform: {{.Platform}}</small>
</div>
{{end}}
{{if .Password}}
<div class="card-text">
<small class="text-muted">Unzip password: <code>{{.Password}}</code></small>
</div>
{{end}}
{{if .UpdatedAt}}
<div class="card-text">
<small class="text-muted">Updated: {{.UpdatedAt}}</small>
</div>
{{end}}
<table class="table table-striped">
<thead>
<tr>
<th>Label</th>
<th>Link</th>
</tr>
</thead>
<tbody>
{{range $key, $value := .Downloads}}
<tr>
<td>{{$key}}</td>
<td>
<div class="input-group mb-3">
<input class="form-control" type="text" value="{{$value}}" readonly>
<button class="btn btn-outline-secondary btn-sm" type="button"
onclick="copyToClipboard(this, '{{$value}}')">
Copy
</button>
</div>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{end}}
</div>
{{end}}
</div>
</div>
</div>
@ -227,25 +268,20 @@
},
});
function copyToClipboard(input) {
input.select();
input.setSelectionRange(0, 99999);
function copyToClipboard(button, text) {
const el = document.createElement("textarea");
el.value = text;
document.body.appendChild(el);
el.select();
document.execCommand("copy");
document.body.removeChild(el);
navigator.clipboard.writeText(input.value).then(() => {
const button = input.nextElementSibling;
const originalText = button.textContent;
button.textContent = 'Copied';
button.classList.remove('btn-outline-secondary');
button.classList.add('btn-success');
setTimeout(() => {
button.textContent = originalText;
button.classList.remove('btn-success');
button.classList.add('btn-outline-secondary');
}, 1500);
}).catch(err => {
console.error('Failed to copy:', err);
});
button.textContent = "Copied";
button.disabled = true;
setTimeout(() => {
button.textContent = "Copy";
button.disabled = false;
}, 2000);
}
</script>
{{end}}

2
server/templates/layouts/base.html
View File

@ -1,6 +1,6 @@
{{define "base"}}
<!DOCTYPE html>
<html lang="zh-CN" data-bs-theme="dark">
<html lang="en" data-bs-theme="dark">
<head>
<meta charset="UTF-8" />

81
utils/mgnet.go Normal file
View File

@ -0,0 +1,81 @@
package utils
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/PuerkitoBio/goquery"
)
func GetLinkFromMgnet(URL string) (string, error) {
resp, err := Request().Get(URL)
if err != nil {
return "", fmt.Errorf("Error while requesting URL: %s: %s", err, URL)
}
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(resp.Body()))
if err != nil {
return "", fmt.Errorf("Error while parsing HTML: %s: %s", err, URL)
}
ad_form_data := doc.Find("[name='ad_form_data']").AttrOr("value", "")
if ad_form_data == "" {
return "", fmt.Errorf("Failed to get ad_form_data: %s", URL)
}
token_fields := doc.Find("[name='_Token[fields]']").AttrOr("value", "")
if token_fields == "" {
return "", fmt.Errorf("Failed to get _Token[fields]: %s", URL)
}
token_unlocked := doc.Find("[name='_Token[unlocked]']").AttrOr("value", "")
if token_unlocked == "" {
return "", fmt.Errorf("Failed to get _Token[unlocked]: %s", URL)
}
cookies := resp.Cookies()
csrfToken := ""
for _, cookie := range cookies {
if cookie.Name == "csrfToken" {
csrfToken = cookie.Value
break
}
}
if csrfToken == "" {
return "", fmt.Errorf("Failed to get csrfToken: %s", URL)
}
params := url.Values{}
params.Set("_method", "POST")
params.Set("_csrfToken", csrfToken)
params.Set("ad_form_data", ad_form_data)
params.Set("_Token[fields]", token_fields)
params.Set("_Token[unlocked]", token_unlocked)
cookies = append(cookies, &http.Cookie{
Name: "ab",
Value: "2",
})
resp, err = Request().SetHeaders(map[string]string{
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"X-Requested-With": "XMLHttpRequest",
"Referer": URL,
}).SetCookies(cookies).SetBody(params.Encode()).Post("https://mgnet.site/links/go")
if err != nil {
return "", fmt.Errorf("Error while requesting URL: %s: %s", err, "https://mgnet.site/links/go")
}
type requestResult struct {
Status string `json:"status"`
Message string `json:"message"`
URL string `json:"url"`
}
res := requestResult{}
err = json.Unmarshal(resp.Body(), &res)
if err != nil {
return "", fmt.Errorf("Error while parsing JSON: %s", err)
}
if res.Status != "success" {
return "", fmt.Errorf("Failed to get link: %s: %s: %+v", res.Message, URL, res)
}
return res.URL, nil
}

2
utils/ouo.go
View File

@ -15,7 +15,7 @@ import (
"strings"
)
func OuoBypass(ouoURL string) (string, error) {
func GetLinkFromOUO(ouoURL string) (string, error) {
tempURL := strings.Replace(ouoURL, "ouo.press", "ouo.io", 1)
var res string
u, err := url.Parse(tempURL)