nite/game-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

remove model.GameItem.Download

Browse Source 
add model.GameItem.DownloadLinks
This commit is contained in:
Nite07 2024-12-26 14:26:24 +08:00
parent 39c7389a0a
commit 45f7eff8b1
14 changed files with 131 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/1337x.go
View File

@ -104,7 +104,7 @@ func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.RawName = strings.Replace(item.RawName, "Download ", "", 1) item.RawName = strings.Replace(item.RawName, "Download ", "", 1)
item.RawName = strings.TrimSpace(strings.Replace(item.RawName, "Torrent | 1337x", " ", 1)) item.RawName = strings.TrimSpace(strings.Replace(item.RawName, "Torrent | 1337x", " ", 1))
item.Name = c.formatter(item.RawName) item.Name = c.formatter(item.RawName)
item.Download = magnetRegexRes[0] item.DownloadLinks = []string{magnetRegexRes[0]}
item.Author = strings.Replace(c.source, "-torrents", "", -1) item.Author = strings.Replace(c.source, "-torrents", "", -1)
item.Platform = c.platform item.Platform = c.platform
return item, nil return item, nil

2
crawler/chovka.go
View File

@ -62,7 +62,7 @@ func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, err return nil, err
} }
item.Size = size item.Size = size
item.Download = magnet item.DownloadLinks = []string{magnet}
return item, nil return item, nil
} }

1
crawler/crawler.go
View File

@ -10,6 +10,7 @@ type Crawler interface {
Name() string Name() string
Crawl(int) ([]*model.GameItem, error) Crawl(int) ([]*model.GameItem, error)
CrawlAll() ([]*model.GameItem, error) CrawlAll() ([]*model.GameItem, error)
CrawlByUrl(string) (*model.GameItem, error)
} }
type SimpleCrawler interface { type SimpleCrawler interface {

2
crawler/fitgirl.go
View File

@ -68,7 +68,7 @@ func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.Url = URL item.Url = URL
item.Size = size item.Size = size
item.Author = "FitGirl" item.Author = "FitGirl"
item.Download = magnet item.DownloadLinks = []string{magnet}
item.Platform = "windows" item.Platform = "windows"
return item, nil return item, nil
} }

2
crawler/freegog.go
View File

@ -148,7 +148,7 @@ func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
item.Download = string(magnet) item.DownloadLinks = []string{string(magnet)}
} else { } else {
return nil, errors.New("failed to find magnet link") return nil, errors.New("failed to find magnet link")
} }

8
crawler/goggames.go
View File

@ -94,7 +94,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
} }
item.Name = name item.Name = name
item.RawName = name item.RawName = name
item.Download = strings.Join(links, ",") item.DownloadLinks = links
item.Url = URL item.Url = URL
item.Size = utils.BytesToSize(size) item.Size = utils.BytesToSize(size)
item.Author = "GOGGames" item.Author = "GOGGames"
@ -130,6 +130,12 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) {
continue continue
} }
item.UpdateFlag = updateFlags[i] item.UpdateFlag = updateFlags[i]
oldItem, err := db.GetGameItemByUrl(u)
if err == nil {
db.MergeGameItem(oldItem, item)
}
if err := db.SaveGameItem(item); err != nil { if err := db.SaveGameItem(item); err != nil {
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u)) c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
continue continue

8
crawler/onlinefix.go
View File

@ -138,7 +138,9 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(resp.Body()) magnet, size, err := utils.ConvertTorrentToMagnet(resp.Body())
item.DownloadLinks = []string{magnet}
item.Size = size
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -167,7 +169,9 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(dataBytes) magnet, size, err := utils.ConvertTorrentToMagnet(dataBytes)
item.DownloadLinks = []string{magnet}
item.Size = size
if err != nil { if err != nil {
return nil, err return nil, err
} }

32
crawler/steamrip.go
View File

@ -57,24 +57,22 @@ func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
} }
megadbRegex := regexp.MustCompile(`(?i)(?:https?:)?(//megadb\.net/[^"]+)`) megadbRegex := regexp.MustCompile(`(?i)(?:https?:)?(//megadb\.net/[^"]+)`)
megadbRegexRes := megadbRegex.FindStringSubmatch(string(resp.Body())) megadbRegexRes := megadbRegex.FindStringSubmatch(string(resp.Body()))
links := []string{}
if len(megadbRegexRes) != 0 { if len(megadbRegexRes) != 0 {
item.Download = fmt.Sprintf("https:%s", megadbRegexRes[1]) links = append(links, fmt.Sprintf("https:%s", megadbRegexRes[1]))
} }
if item.Download == "" { gofileRegex := regexp.MustCompile(`(?i)(?:https?:)?(//gofile\.io/d/[^"]+)`)
gofileRegex := regexp.MustCompile(`(?i)(?:https?:)?(//gofile\.io/d/[^"]+)`) gofileRegexRes := gofileRegex.FindStringSubmatch(string(resp.Body()))
gofileRegexRes := gofileRegex.FindStringSubmatch(string(resp.Body())) if len(gofileRegexRes) != 0 {
if len(gofileRegexRes) != 0 { links = append(links, fmt.Sprintf("https:%s", gofileRegexRes[1]))
item.Download = fmt.Sprintf("https:%s", gofileRegexRes[1])
}
} }
if item.Download == "" { filecryptRegex := regexp.MustCompile(`(?i)(?:https?:)?(//filecrypt\.co/Container/[^"]+)`)
filecryptRegex := regexp.MustCompile(`(?i)(?:https?:)?(//filecrypt\.co/Container/[^"]+)`) filecryptRegexRes := filecryptRegex.FindStringSubmatch(string(resp.Body()))
filecryptRegexRes := filecryptRegex.FindStringSubmatch(string(resp.Body())) if len(filecryptRegexRes) != 0 {
if len(filecryptRegexRes) != 0 { links = append(links, fmt.Sprintf("https:%s", filecryptRegexRes[1]))
item.Download = fmt.Sprintf("https:%s", filecryptRegexRes[1])
}
} }
if item.Download == "" { item.DownloadLinks = links
if len(item.DownloadLinks) == 0 {
return nil, errors.New("failed to find download link") return nil, errors.New("failed to find download link")
} }
@ -116,6 +114,12 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
continue continue
} }
item.UpdateFlag = updateFlags[i] item.UpdateFlag = updateFlags[i]
oldItem, err := db.GetGameItemByUrl(u)
if err == nil {
db.MergeGameItem(oldItem, item)
}
if err := db.SaveGameItem(item); err != nil { if err := db.SaveGameItem(item); err != nil {
c.logger.Warn("Failed to save item", zap.Error(err)) c.logger.Warn("Failed to save item", zap.Error(err))
continue continue

2
crawler/xatab.go
View File

@ -110,7 +110,7 @@ func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, err return nil, err
} }
item.Size = size item.Size = size
item.Download = magnet item.DownloadLinks = []string{magnet}
return item, nil return item, nil
} }

7
db/game.go
View File

@ -132,7 +132,7 @@ func SaveGameItem(item *model.GameItem) error {
} }
func SaveGameItems(items []*model.GameItem) error { func SaveGameItems(items []*model.GameItem) error {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel() defer cancel()
operations := make([]mongo.WriteModel, len(items)) operations := make([]mongo.WriteModel, len(items))
for i, item := range items { for i, item := range items {
@ -945,3 +945,8 @@ func MergeGameInfo(oldInfo *model.GameInfo, newInfo *model.GameInfo) {
newInfo.SteamID = oldInfo.SteamID newInfo.SteamID = oldInfo.SteamID
newInfo.CreatedAt = oldInfo.CreatedAt newInfo.CreatedAt = oldInfo.CreatedAt
} }
func MergeGameItem(oldItem *model.GameItem, newItem *model.GameItem) {
newItem.ID = oldItem.ID
newItem.UpdatedAt = time.Now()
}

24
model/game.go
View File

@ -41,16 +41,16 @@ type GameCollection struct {
} }
type GameItem struct { type GameItem struct {
ID primitive.ObjectID `json:"id" bson:"_id"` ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"speculative_name" bson:"name"` Name string `json:"speculative_name" bson:"name"`
RawName string `json:"raw_name,omitempty" bson:"raw_name"` RawName string `json:"raw_name,omitempty" bson:"raw_name"`
Download string `json:"download_link,omitempty" bson:"download"` DownloadLinks []string `json:"download_links,omitempty" bson:"download_links"`
Size string `json:"size,omitempty" bson:"size"` Size string `json:"size,omitempty" bson:"size"`
Url string `json:"url" bson:"url"` Url string `json:"url" bson:"url"`
Password string `json:"password,omitempty" bson:"password"` Password string `json:"password,omitempty" bson:"password"`
Author string `json:"author,omitempty" bson:"author"` Author string `json:"author,omitempty" bson:"author"`
Platform string `json:"platform,omitempty" bson:"platform"` Platform string `json:"platform,omitempty" bson:"platform"`
UpdateFlag string `json:"-" bson:"update_flag,omitempty"` UpdateFlag string `json:"-" bson:"update_flag,omitempty"`
CreatedAt time.Time `json:"created_at" bson:"created_at"` CreatedAt time.Time `json:"created_at" bson:"created_at"`
UpdatedAt time.Time `json:"updated_at" bson:"updated_at"` UpdatedAt time.Time `json:"updated_at" bson:"updated_at"`
} }

3
server/middleware/log.go
View File

@ -1,6 +1,7 @@
package middleware package middleware
import ( import (
"fmt"
"pcgamedb/log" "pcgamedb/log"
"strings" "strings"
"time" "time"
@ -30,7 +31,7 @@ func Logger() gin.HandlerFunc {
zap.String("method", c.Request.Method), zap.String("method", c.Request.Method),
zap.String("path", path), zap.String("path", path),
zap.String("ip", getRealIP(c)), zap.String("ip", getRealIP(c)),
zap.Duration("latency", time.Since(startTime)), zap.String("latency", fmt.Sprintf("%v ms", time.Since(startTime).Milliseconds())),
} }
if len(c.Errors) > 0 { if len(c.Errors) > 0 {

104
server/templates/game.html
View File

@ -41,6 +41,36 @@
border-radius: 0.25rem; border-radius: 0.25rem;
font-size: 0.875rem; font-size: 0.875rem;
} }
/* Masonry Container Styles */
.masonry-container {
column-count: 3;
/* 3 Columns for Masonry */
column-gap: 1rem;
/* Adjust Gap Between Columns */
}
@media (max-width: 992px) {
.masonry-container {
column-count: 2;
/* 2 Columns on Medium Screens */
}
}
@media (max-width: 576px) {
.masonry-container {
column-count: 1;
/* 1 Column on Small Screens */
}
}
/* Masonry Item */
.masonry-container .card {
display: inline-block;
/* Ensure Cards Behave as Block Elements in Columns */
width: 100%;
/* Make Cards Fill the Column Width */
}
</style> </style>
{{end}} {{end}}
@ -127,51 +157,53 @@
{{if .Games}} {{if .Games}}
<div class="mb-4"> <div class="mb-4">
<h3 class="mb-3">Download</h3> <h3 class="mb-3">Download</h3>
<div class="row g-4"> <div class="row">
{{range .Games}} <div class="col-12">
<div class="col-md-6"> <!-- Masonry Container -->
<div class="card download-card"> <div class="masonry-container">
<div class="card-body"> {{range .Games}}
<h5 class="card-title">{{.RawName}}</h5> <div class="card download-card mb-3">
{{if .Size}} <div class="card-body">
<div class="card-text"> <h5 class="card-title"><a class="text-decoration-none" href="{{.Url}}">{{.RawName}}</a></h5>
<small class="text-muted">Size: {{.Size}}</small> {{if .Size}}
</div> <div class="card-text">
{{end}} <small class="text-muted">Size: {{.Size}}</small>
{{if .Author}} </div>
<div class="card-text"> {{end}}
<small class="text-muted">Source: {{.Author}}</small> {{if .Author}}
</div> <div class="card-text">
{{end}} <small class="text-muted">Source: {{.Author}}</small>
{{if .Platform}} </div>
<div class="card-text"> {{end}}
<small class="text-muted">Platform: {{.Platform}}</small> {{if .Platform}}
</div> <div class="card-text">
{{end}} <small class="text-muted">Platform: {{.Platform}}</small>
{{if .Password}} </div>
<div class="card-text"> {{end}}
<small class="text-muted">Unzip password: <code>{{.Password}}</code></small> {{if .Password}}
</div> <div class="card-text">
{{end}} <small class="text-muted">Unzip password: <code>{{.Password}}</code></small>
{{if .UpdatedAt}} </div>
<div class="card-text"> {{end}}
<small class="text-muted">Updated: {{.UpdatedAt}}</small> {{if .UpdatedAt}}
</div> <div class="card-text">
{{end}} <small class="text-muted">Updated: {{.UpdatedAt}}</small>
<div class="mt-2 d-flex justify-content-between align-items-center"> </div>
<div class="input-group" style="max-width: 300px;"> {{end}}
<input type="text" class="form-control form-control-sm" value="{{.Download}}" readonly> {{range .DownloadLinks}}
<div class="input-group mb-1" style="max-width: 300px;">
<input type="text" class="form-control form-control-sm" value="{{.}}" readonly>
<button class="btn btn-outline-secondary btn-sm" type="button" <button class="btn btn-outline-secondary btn-sm" type="button"
onclick="copyToClipboard(this.previousElementSibling)"> onclick="copyToClipboard(this.previousElementSibling)">
Copy Copy
</button> </button>
</div> </div>
<a href="{{.Url}}" class="btn btn-outline-primary" target="_blank">Source Page</a> {{end}}
</div> </div>
</div> </div>
{{end}}
</div> </div>
</div> </div>
{{end}}
</div> </div>
</div> </div>
{{end}} {{end}}

7
utils/request.go
View File

@ -1,6 +1,7 @@
package utils package utils
import ( import (
"net/http"
"time" "time"
"github.com/go-resty/resty/v2" "github.com/go-resty/resty/v2"
@ -10,7 +11,11 @@ var client *resty.Client
func init() { func init() {
client = resty.New() client = resty.New()
client.SetRetryCount(3).SetRetryWaitTime(1 * time.Second) client.SetRetryCount(3).SetRetryWaitTime(3 * time.Second).AddRetryCondition(
func(r *resty.Response, err error) bool {
return err != nil || r.StatusCode() == http.StatusTooManyRequests
},
)
} }
func Request() *resty.Request { func Request() *resty.Request {