nite/game-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

remove model.GameItem.Download

Browse Source 
add model.GameItem.DownloadLinks
This commit is contained in:
Nite07 2024-12-26 14:26:24 +08:00
parent 39c7389a0a
commit 45f7eff8b1
14 changed files with 131 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/1337x.go
View File

@ -104,7 +104,7 @@ func (c *s1337xCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.RawName = strings.Replace(item.RawName, "Download ", "", 1)
item.RawName = strings.TrimSpace(strings.Replace(item.RawName, "Torrent | 1337x", " ", 1))
item.Name = c.formatter(item.RawName)
item.Download = magnetRegexRes[0]
item.DownloadLinks = []string{magnetRegexRes[0]}
item.Author = strings.Replace(c.source, "-torrents", "", -1)
item.Platform = c.platform
return item, nil

2
crawler/chovka.go
View File

@ -62,7 +62,7 @@ func (c *ChovkaCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, err
}
item.Size = size
item.Download = magnet
item.DownloadLinks = []string{magnet}
return item, nil
}

1
crawler/crawler.go
View File

@ -10,6 +10,7 @@ type Crawler interface {
Name() string
Crawl(int) ([]*model.GameItem, error)
CrawlAll() ([]*model.GameItem, error)
CrawlByUrl(string) (*model.GameItem, error)
}
type SimpleCrawler interface {

2
crawler/fitgirl.go
View File

@ -68,7 +68,7 @@ func (c *FitGirlCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
item.Url = URL
item.Size = size
item.Author = "FitGirl"
item.Download = magnet
item.DownloadLinks = []string{magnet}
item.Platform = "windows"
return item, nil
}

2
crawler/freegog.go
View File

@ -148,7 +148,7 @@ func (c *FreeGOGCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil {
return nil, err
}
item.Download = string(magnet)
item.DownloadLinks = []string{string(magnet)}
} else {
return nil, errors.New("failed to find magnet link")
}

8
crawler/goggames.go
View File

@ -94,7 +94,7 @@ func (c *GOGGamesCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
}
item.Name = name
item.RawName = name
item.Download = strings.Join(links, ",")
item.DownloadLinks = links
item.Url = URL
item.Size = utils.BytesToSize(size)
item.Author = "GOGGames"
@ -130,6 +130,12 @@ func (c *GOGGamesCrawler) Crawl(page int) ([]*model.GameItem, error) {
continue
}
item.UpdateFlag = updateFlags[i]
oldItem, err := db.GetGameItemByUrl(u)
if err == nil {
db.MergeGameItem(oldItem, item)
}
if err := db.SaveGameItem(item); err != nil {
c.logger.Warn("Failed to save", zap.Error(err), zap.String("URL", u))
continue

8
crawler/onlinefix.go
View File

@ -138,7 +138,9 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil {
return nil, err
}
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(resp.Body())
magnet, size, err := utils.ConvertTorrentToMagnet(resp.Body())
item.DownloadLinks = []string{magnet}
item.Size = size
if err != nil {
return nil, err
}
@ -167,7 +169,9 @@ func (c *OnlineFixCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
if err != nil {
return nil, err
}
item.Download, item.Size, err = utils.ConvertTorrentToMagnet(dataBytes)
magnet, size, err := utils.ConvertTorrentToMagnet(dataBytes)
item.DownloadLinks = []string{magnet}
item.Size = size
if err != nil {
return nil, err
}

20
crawler/steamrip.go
View File

@ -57,24 +57,22 @@ func (c *SteamRIPCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
}
megadbRegex := regexp.MustCompile(`(?i)(?:https?:)?(//megadb\.net/[^"]+)`)
megadbRegexRes := megadbRegex.FindStringSubmatch(string(resp.Body()))
links := []string{}
if len(megadbRegexRes) != 0 {
item.Download = fmt.Sprintf("https:%s", megadbRegexRes[1])
links = append(links, fmt.Sprintf("https:%s", megadbRegexRes[1]))
}
if item.Download == "" {
gofileRegex := regexp.MustCompile(`(?i)(?:https?:)?(//gofile\.io/d/[^"]+)`)
gofileRegexRes := gofileRegex.FindStringSubmatch(string(resp.Body()))
if len(gofileRegexRes) != 0 {
item.Download = fmt.Sprintf("https:%s", gofileRegexRes[1])
links = append(links, fmt.Sprintf("https:%s", gofileRegexRes[1]))
}
}
if item.Download == "" {
filecryptRegex := regexp.MustCompile(`(?i)(?:https?:)?(//filecrypt\.co/Container/[^"]+)`)
filecryptRegexRes := filecryptRegex.FindStringSubmatch(string(resp.Body()))
if len(filecryptRegexRes) != 0 {
item.Download = fmt.Sprintf("https:%s", filecryptRegexRes[1])
links = append(links, fmt.Sprintf("https:%s", filecryptRegexRes[1]))
}
}
if item.Download == "" {
item.DownloadLinks = links
if len(item.DownloadLinks) == 0 {
return nil, errors.New("failed to find download link")
}
@ -116,6 +114,12 @@ func (c *SteamRIPCrawler) Crawl(num int) ([]*model.GameItem, error) {
continue
}
item.UpdateFlag = updateFlags[i]
oldItem, err := db.GetGameItemByUrl(u)
if err == nil {
db.MergeGameItem(oldItem, item)
}
if err := db.SaveGameItem(item); err != nil {
c.logger.Warn("Failed to save item", zap.Error(err))
continue

2
crawler/xatab.go
View File

@ -110,7 +110,7 @@ func (c *XatabCrawler) CrawlByUrl(URL string) (*model.GameItem, error) {
return nil, err
}
item.Size = size
item.Download = magnet
item.DownloadLinks = []string{magnet}
return item, nil
}

7
db/game.go
View File

@ -132,7 +132,7 @@ func SaveGameItem(item *model.GameItem) error {
}
func SaveGameItems(items []*model.GameItem) error {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
operations := make([]mongo.WriteModel, len(items))
for i, item := range items {
@ -945,3 +945,8 @@ func MergeGameInfo(oldInfo *model.GameInfo, newInfo *model.GameInfo) {
newInfo.SteamID = oldInfo.SteamID
newInfo.CreatedAt = oldInfo.CreatedAt
}
func MergeGameItem(oldItem *model.GameItem, newItem *model.GameItem) {
newItem.ID = oldItem.ID
newItem.UpdatedAt = time.Now()
}

2
model/game.go
View File

@ -44,7 +44,7 @@ type GameItem struct {
ID primitive.ObjectID `json:"id" bson:"_id"`
Name string `json:"speculative_name" bson:"name"`
RawName string `json:"raw_name,omitempty" bson:"raw_name"`
Download string `json:"download_link,omitempty" bson:"download"`
DownloadLinks []string `json:"download_links,omitempty" bson:"download_links"`
Size string `json:"size,omitempty" bson:"size"`
Url string `json:"url" bson:"url"`
Password string `json:"password,omitempty" bson:"password"`

3
server/middleware/log.go
View File

@ -1,6 +1,7 @@
package middleware
import (
"fmt"
"pcgamedb/log"
"strings"
"time"
@ -30,7 +31,7 @@ func Logger() gin.HandlerFunc {
zap.String("method", c.Request.Method),
zap.String("path", path),
zap.String("ip", getRealIP(c)),
zap.Duration("latency", time.Since(startTime)),
zap.String("latency", fmt.Sprintf("%v ms", time.Since(startTime).Milliseconds())),
}
if len(c.Errors) > 0 {

52
server/templates/game.html
View File

@ -41,6 +41,36 @@
border-radius: 0.25rem;
font-size: 0.875rem;
}
/* Masonry Container Styles */
.masonry-container {
column-count: 3;
/* 3 Columns for Masonry */
column-gap: 1rem;
/* Adjust Gap Between Columns */
}
@media (max-width: 992px) {
.masonry-container {
column-count: 2;
/* 2 Columns on Medium Screens */
}
}
@media (max-width: 576px) {
.masonry-container {
column-count: 1;
/* 1 Column on Small Screens */
}
}
/* Masonry Item */
.masonry-container .card {
display: inline-block;
/* Ensure Cards Behave as Block Elements in Columns */
width: 100%;
/* Make Cards Fill the Column Width */
}
</style>
{{end}}
@ -127,12 +157,14 @@
{{if .Games}}
<div class="mb-4">
<h3 class="mb-3">Download</h3>
<div class="row g-4">
<div class="row">
<div class="col-12">
<!-- Masonry Container -->
<div class="masonry-container">
{{range .Games}}
<div class="col-md-6">
<div class="card download-card">
<div class="card download-card mb-3">
<div class="card-body">
<h5 class="card-title">{{.RawName}}</h5>
<h5 class="card-title"><a class="text-decoration-none" href="{{.Url}}">{{.RawName}}</a></h5>
{{if .Size}}
<div class="card-text">
<small class="text-muted">Size: {{.Size}}</small>
@ -158,22 +190,22 @@
<small class="text-muted">Updated: {{.UpdatedAt}}</small>
</div>
{{end}}
<div class="mt-2 d-flex justify-content-between align-items-center">
<div class="input-group" style="max-width: 300px;">
<input type="text" class="form-control form-control-sm" value="{{.Download}}" readonly>
{{range .DownloadLinks}}
<div class="input-group mb-1" style="max-width: 300px;">
<input type="text" class="form-control form-control-sm" value="{{.}}" readonly>
<button class="btn btn-outline-secondary btn-sm" type="button"
onclick="copyToClipboard(this.previousElementSibling)">
Copy
</button>
</div>
<a href="{{.Url}}" class="btn btn-outline-primary" target="_blank">Source Page</a>
</div>
</div>
{{end}}
</div>
</div>
{{end}}
</div>
</div>
</div>
</div>
{{end}}
</div>
{{end}}

7
utils/request.go
View File

@ -1,6 +1,7 @@
package utils
import (
"net/http"
"time"
"github.com/go-resty/resty/v2"
@ -10,7 +11,11 @@ var client *resty.Client
func init() {
client = resty.New()
client.SetRetryCount(3).SetRetryWaitTime(1 * time.Second)
client.SetRetryCount(3).SetRetryWaitTime(3 * time.Second).AddRetryCondition(
func(r *resty.Response, err error) bool {
return err != nil || r.StatusCode() == http.StatusTooManyRequests
},
)
}
func Request() *resty.Request {