nite/pcgamedb
1
0
Fork 0
Code Issues Pull Requests Actions Releases 5 Activity

remove format cmd

Browse Source 
modify README
remove armgddn crawler
This commit is contained in:
Nite07 2024-11-21 01:40:45 +08:00
parent 08a4aafaf9
commit ee7db03e24
4 changed files with 3 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -9,11 +9,10 @@ pcgamedb is a powerful command-line tool designed to scrape and manage repack ga
- Fitgirl
- KaOSKrew
- DODI
- ~~FreeGOG~~
- GOGGames
- FreeGOG
- ~~GOGGames~~
- OnlineFix
- Xatab
- ~~ARMGDDN~~
- SteamRIP
- Chovka

138
cmd/format.go
View File

@ -1,138 +0,0 @@
package cmd
import (
"strings"
"pcgamedb/crawler"
"pcgamedb/db"
"pcgamedb/log"
"github.com/spf13/cobra"
"go.uber.org/zap"
)
var formatCmd = &cobra.Command{
Use: "format",
Short: "Format game downloads name by formatter",
Long: "Format game downloads name by formatter",
Run: formatRun,
}
type FormatCommandConfig struct {
Source string
}
var formatCmdCfg FormatCommandConfig
func init() {
formatCmd.Flags().StringVarP(&formatCmdCfg.Source, "source", "s", "", "source to fix (fitgirl/dodi/kaoskrew/freegog/xatab/onlinefix/armgddn)")
RootCmd.AddCommand(formatCmd)
}
func formatRun(cmd *cobra.Command, args []string) {
formatCmdCfg.Source = strings.ToLower(formatCmdCfg.Source)
switch formatCmdCfg.Source {
case "dodi":
items, err := db.GetDODIGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
}
for _, item := range items {
oldName := item.Name
item.Name = crawler.DODIFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "kaoskrew":
items, err := db.GetKaOsKrewGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
}
for _, item := range items {
oldName := item.Name
item.Name = crawler.KaOsKrewFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "freegog":
items, err := db.GetFreeGOGGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
}
for _, item := range items {
oldName := item.Name
item.Name = crawler.FreeGOGFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "xatab":
items, err := db.GetXatabGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
}
for _, item := range items {
oldName := item.Name
item.Name = crawler.XatabFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "onlinefix":
items, err := db.GetOnlineFixGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
}
for _, item := range items {
oldName := item.Name
item.Name = crawler.OnlineFixFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
case "armgddn":
items, err := db.GetARMGDDNGameItems()
if err != nil {
log.Logger.Error("Failed to get games", zap.Error(err))
return
}
for _, item := range items {
oldName := item.Name
item.Name = crawler.ARMGDDNFormatter(item.RawName)
if oldName != item.Name {
log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
err := db.SaveGameItem(item)
if err != nil {
log.Logger.Error("Failed to update item", zap.Error(err))
}
}
}
}
}

214
crawler/armgddn.go
View File

@ -1,214 +0,0 @@
package crawler
import (
"crypto/tls"
"encoding/json"
"fmt"
"io"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"pcgamedb/db"
"pcgamedb/model"
"pcgamedb/utils"
"github.com/jlaffaye/ftp"
"go.uber.org/zap"
)
const (
ftpAddress = "72.21.17.26:13017"
ftpUsername = "ARMGDDNGames"
ftpPassword = "ARMGDDNGames"
)
type GameData struct {
NumberOfGame string `json:"Number of game"`
AppID string `json:"appid"`
FolderName string `json:"foldername"`
}
type ARMGDDNCrawler struct {
logger zap.Logger
conn *ftp.ServerConn
}
// Deprecated: ARMGDDN has changed resource distribution method
func NewARMGDDNCrawler(logger *zap.Logger) *ARMGDDNCrawler {
return &ARMGDDNCrawler{
logger: *logger,
}
}
func (c *ARMGDDNCrawler) connectFTP() error {
var err error
tlsConfig := &tls.Config{InsecureSkipVerify: true}
c.conn, err = ftp.Dial(ftpAddress, ftp.DialWithTimeout(5*time.Second), ftp.DialWithExplicitTLS(tlsConfig))
if err != nil {
return err
}
if err = c.conn.Login(ftpUsername, ftpPassword); err != nil {
return err
}
return nil
}
func (c *ARMGDDNCrawler) fetchAndParseFTPData(filePath string) ([]GameData, error) {
r, err := c.conn.Retr(filePath)
if err != nil {
return nil, err
}
defer r.Close()
buf, err := io.ReadAll(r)
if err != nil {
return nil, err
}
var data []GameData
if err = json.Unmarshal(buf, &data); err != nil {
return nil, err
}
return data, nil
}
func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameItem, error) {
count := 0
var res []*model.GameItem
modTimeMap := make(map[string]time.Time)
entries, err := c.conn.List(fmt.Sprintf("/%s", platform))
if err != nil {
return nil, err
}
for _, entry := range entries {
if entry.Type == ftp.EntryTypeFolder {
modTimeMap[entry.Name] = entry.Time
}
}
for _, v := range data {
if count == num {
break
}
path := fmt.Sprintf("/%s/%s", platform, v.FolderName)
u := fmt.Sprintf("ARMGDDNGames/%s/%s", platform, v.NumberOfGame)
modTime, ok := modTimeMap[v.FolderName]
if !ok {
c.logger.Warn("mod time not found", zap.String("url", u))
continue
}
updateFlag := fmt.Sprintf("ARMGDDNGames/%s/%s/%s", platform, v.NumberOfGame, modTime.UTC().String())
if db.IsARMGDDNCrawled(updateFlag) {
continue
}
c.logger.Info("Crawling", zap.String("url", u))
walker := c.conn.Walk(path)
size := uint64(0)
for walker.Next() {
if walker.Stat().Type == ftp.EntryTypeFile {
fileSize, err := c.conn.FileSize(walker.Path())
if err != nil {
c.logger.Warn("file size error", zap.Error(err))
break
}
size += uint64(fileSize)
}
}
item, err := db.GetGameItemByUrl(u)
if err != nil {
continue
}
item.Url = u
item.Name = ARMGDDNFormatter(v.FolderName)
item.UpdateFlag = updateFlag
item.Size = utils.BytesToSize(size)
item.RawName = v.FolderName
item.Author = "ARMGDDN"
item.Download = fmt.Sprintf("ftpes://%s:%s@%s/%s/%s", ftpUsername, ftpPassword, ftpAddress, platform, url.QueryEscape(v.FolderName))
if err := db.SaveGameItem(item); err != nil {
continue
}
res = append(res, item)
count++
var id int
var info *model.GameInfo
if v.AppID != "NONSTEAM" {
id, err = strconv.Atoi(v.AppID)
if err != nil {
c.logger.Warn("strconv error", zap.Error(err))
continue
}
info, err = OrganizeGameItemWithSteam(id, item)
if err != nil {
continue
}
} else {
info, err = OrganizeGameItem(item)
if err != nil {
continue
}
}
err = db.SaveGameInfo(info)
if err != nil {
c.logger.Warn("save game info error", zap.Error(err))
continue
}
}
return res, nil
}
func ARMGDDNFormatter(name string) string {
cleanedName := strings.ReplaceAll(strings.TrimSpace(name), "-ARMGDDN", "")
matchIndex := regexp.MustCompile(`v\d`).FindStringIndex(cleanedName)
if matchIndex == nil {
return cleanedName
}
return strings.TrimSpace(cleanedName[:matchIndex[0]])
}
func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameItem, error) {
return c.crawlPlatform("/PC/currentserverPC-FTP.json", "PC", num)
}
func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameItem, error) {
return c.crawlPlatform("/PCVR/currentserverPCVR-FTP.json", "PCVR", num)
}
func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameItem, error) {
num1 := num / 2
num2 := num - num1
if num == -1 {
num1 = -1
num2 = -1
}
res1, err := c.CrawlPC(num1)
if err != nil {
return nil, err
}
res2, err := c.CrawlPCVR(num2)
if err != nil {
return nil, err
}
return append(res1, res2...), nil
}
func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameItem, error) {
return c.Crawl(-1)
}
func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameItem, error) {
err := c.connectFTP()
if err != nil {
return nil, err
}
defer func() { _ = c.conn.Quit() }()
data, err := c.fetchAndParseFTPData(jsonFile)
if err != nil {
return nil, err
}
return c.crawlGames(data, platform, num)
}

3
crawler/crawler.go
View File

@ -31,8 +31,7 @@ func BuildCrawlerMap(logger *zap.Logger) map[string]Crawler {
"xatab": NewXatabCrawler(logger),
"onlinefix": NewOnlineFixCrawler(logger),
"steamrip": NewSteamRIPCrawler(logger),
// "armgddn": NewARMGDDNCrawler(logger),
"chovka": NewChovkaCrawler(logger),
"chovka": NewChovkaCrawler(logger),
// "gnarly": NewGnarlyCrawler(logger),
// "goggames": NewGOGGamesCrawler(logger),
}