From ee7db03e243461fd48a53c2e5f3ef001663cb568 Mon Sep 17 00:00:00 2001 From: nite07 Date: Thu, 21 Nov 2024 01:40:45 +0800 Subject: [PATCH] remove format cmd modify README remove armgddn crawler --- README.md | 5 +- cmd/format.go | 138 ----------------------------- crawler/armgddn.go | 214 --------------------------------------------- crawler/crawler.go | 3 +- 4 files changed, 3 insertions(+), 357 deletions(-) delete mode 100644 cmd/format.go delete mode 100644 crawler/armgddn.go diff --git a/README.md b/README.md index 2a81ff9..275227f 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,10 @@ pcgamedb is a powerful command-line tool designed to scrape and manage repack ga - Fitgirl - KaOSKrew - DODI - - ~~FreeGOG~~ - - GOGGames + - FreeGOG + - ~~GOGGames~~ - OnlineFix - Xatab - - ~~ARMGDDN~~ - SteamRIP - Chovka diff --git a/cmd/format.go b/cmd/format.go deleted file mode 100644 index fa1e18e..0000000 --- a/cmd/format.go +++ /dev/null @@ -1,138 +0,0 @@ -package cmd - -import ( - "strings" - - "pcgamedb/crawler" - "pcgamedb/db" - "pcgamedb/log" - - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var formatCmd = &cobra.Command{ - Use: "format", - Short: "Format game downloads name by formatter", - Long: "Format game downloads name by formatter", - Run: formatRun, -} - -type FormatCommandConfig struct { - Source string -} - -var formatCmdCfg FormatCommandConfig - -func init() { - formatCmd.Flags().StringVarP(&formatCmdCfg.Source, "source", "s", "", "source to fix (fitgirl/dodi/kaoskrew/freegog/xatab/onlinefix/armgddn)") - RootCmd.AddCommand(formatCmd) -} - -func formatRun(cmd *cobra.Command, args []string) { - formatCmdCfg.Source = strings.ToLower(formatCmdCfg.Source) - switch formatCmdCfg.Source { - case "dodi": - items, err := db.GetDODIGameItems() - if err != nil { - log.Logger.Error("Failed to get games", zap.Error(err)) - return - } - for _, item := range items { - oldName := item.Name - item.Name = crawler.DODIFormatter(item.RawName) - if oldName != item.Name { - log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name)) - err := db.SaveGameItem(item) - if err != nil { - log.Logger.Error("Failed to update item", zap.Error(err)) - } - } - } - case "kaoskrew": - items, err := db.GetKaOsKrewGameItems() - if err != nil { - log.Logger.Error("Failed to get games", zap.Error(err)) - return - } - for _, item := range items { - oldName := item.Name - item.Name = crawler.KaOsKrewFormatter(item.RawName) - if oldName != item.Name { - log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name)) - err := db.SaveGameItem(item) - if err != nil { - log.Logger.Error("Failed to update item", zap.Error(err)) - } - } - } - case "freegog": - items, err := db.GetFreeGOGGameItems() - if err != nil { - log.Logger.Error("Failed to get games", zap.Error(err)) - return - } - for _, item := range items { - oldName := item.Name - item.Name = crawler.FreeGOGFormatter(item.RawName) - if oldName != item.Name { - log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name)) - err := db.SaveGameItem(item) - if err != nil { - log.Logger.Error("Failed to update item", zap.Error(err)) - } - } - } - case "xatab": - items, err := db.GetXatabGameItems() - if err != nil { - log.Logger.Error("Failed to get games", zap.Error(err)) - return - } - for _, item := range items { - oldName := item.Name - item.Name = crawler.XatabFormatter(item.RawName) - if oldName != item.Name { - log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name)) - err := db.SaveGameItem(item) - if err != nil { - log.Logger.Error("Failed to update item", zap.Error(err)) - } - } - } - case "onlinefix": - items, err := db.GetOnlineFixGameItems() - if err != nil { - log.Logger.Error("Failed to get games", zap.Error(err)) - return - } - for _, item := range items { - oldName := item.Name - item.Name = crawler.OnlineFixFormatter(item.RawName) - if oldName != item.Name { - log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name)) - err := db.SaveGameItem(item) - if err != nil { - log.Logger.Error("Failed to update item", zap.Error(err)) - } - } - } - case "armgddn": - items, err := db.GetARMGDDNGameItems() - if err != nil { - log.Logger.Error("Failed to get games", zap.Error(err)) - return - } - for _, item := range items { - oldName := item.Name - item.Name = crawler.ARMGDDNFormatter(item.RawName) - if oldName != item.Name { - log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name)) - err := db.SaveGameItem(item) - if err != nil { - log.Logger.Error("Failed to update item", zap.Error(err)) - } - } - } - } -} diff --git a/crawler/armgddn.go b/crawler/armgddn.go deleted file mode 100644 index de8aaef..0000000 --- a/crawler/armgddn.go +++ /dev/null @@ -1,214 +0,0 @@ -package crawler - -import ( - "crypto/tls" - "encoding/json" - "fmt" - "io" - "net/url" - "regexp" - "strconv" - "strings" - "time" - - "pcgamedb/db" - "pcgamedb/model" - "pcgamedb/utils" - - "github.com/jlaffaye/ftp" - "go.uber.org/zap" -) - -const ( - ftpAddress = "72.21.17.26:13017" - ftpUsername = "ARMGDDNGames" - ftpPassword = "ARMGDDNGames" -) - -type GameData struct { - NumberOfGame string `json:"Number of game"` - AppID string `json:"appid"` - FolderName string `json:"foldername"` -} - -type ARMGDDNCrawler struct { - logger zap.Logger - conn *ftp.ServerConn -} - -// Deprecated: ARMGDDN has changed resource distribution method -func NewARMGDDNCrawler(logger *zap.Logger) *ARMGDDNCrawler { - return &ARMGDDNCrawler{ - logger: *logger, - } -} - -func (c *ARMGDDNCrawler) connectFTP() error { - var err error - tlsConfig := &tls.Config{InsecureSkipVerify: true} - c.conn, err = ftp.Dial(ftpAddress, ftp.DialWithTimeout(5*time.Second), ftp.DialWithExplicitTLS(tlsConfig)) - if err != nil { - return err - } - if err = c.conn.Login(ftpUsername, ftpPassword); err != nil { - return err - } - return nil -} - -func (c *ARMGDDNCrawler) fetchAndParseFTPData(filePath string) ([]GameData, error) { - r, err := c.conn.Retr(filePath) - if err != nil { - return nil, err - } - defer r.Close() - - buf, err := io.ReadAll(r) - if err != nil { - return nil, err - } - - var data []GameData - if err = json.Unmarshal(buf, &data); err != nil { - return nil, err - } - return data, nil -} - -func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameItem, error) { - count := 0 - var res []*model.GameItem - modTimeMap := make(map[string]time.Time) - entries, err := c.conn.List(fmt.Sprintf("/%s", platform)) - if err != nil { - return nil, err - } - for _, entry := range entries { - if entry.Type == ftp.EntryTypeFolder { - modTimeMap[entry.Name] = entry.Time - } - } - for _, v := range data { - if count == num { - break - } - path := fmt.Sprintf("/%s/%s", platform, v.FolderName) - u := fmt.Sprintf("ARMGDDNGames/%s/%s", platform, v.NumberOfGame) - modTime, ok := modTimeMap[v.FolderName] - if !ok { - c.logger.Warn("mod time not found", zap.String("url", u)) - continue - } - updateFlag := fmt.Sprintf("ARMGDDNGames/%s/%s/%s", platform, v.NumberOfGame, modTime.UTC().String()) - if db.IsARMGDDNCrawled(updateFlag) { - continue - } - c.logger.Info("Crawling", zap.String("url", u)) - walker := c.conn.Walk(path) - size := uint64(0) - for walker.Next() { - if walker.Stat().Type == ftp.EntryTypeFile { - fileSize, err := c.conn.FileSize(walker.Path()) - if err != nil { - c.logger.Warn("file size error", zap.Error(err)) - break - } - size += uint64(fileSize) - } - } - item, err := db.GetGameItemByUrl(u) - if err != nil { - continue - } - item.Url = u - item.Name = ARMGDDNFormatter(v.FolderName) - item.UpdateFlag = updateFlag - item.Size = utils.BytesToSize(size) - item.RawName = v.FolderName - item.Author = "ARMGDDN" - item.Download = fmt.Sprintf("ftpes://%s:%s@%s/%s/%s", ftpUsername, ftpPassword, ftpAddress, platform, url.QueryEscape(v.FolderName)) - if err := db.SaveGameItem(item); err != nil { - continue - } - res = append(res, item) - count++ - var id int - var info *model.GameInfo - if v.AppID != "NONSTEAM" { - id, err = strconv.Atoi(v.AppID) - if err != nil { - c.logger.Warn("strconv error", zap.Error(err)) - continue - } - info, err = OrganizeGameItemWithSteam(id, item) - if err != nil { - continue - } - } else { - info, err = OrganizeGameItem(item) - if err != nil { - continue - } - } - err = db.SaveGameInfo(info) - if err != nil { - c.logger.Warn("save game info error", zap.Error(err)) - continue - } - } - return res, nil -} - -func ARMGDDNFormatter(name string) string { - cleanedName := strings.ReplaceAll(strings.TrimSpace(name), "-ARMGDDN", "") - matchIndex := regexp.MustCompile(`v\d`).FindStringIndex(cleanedName) - if matchIndex == nil { - return cleanedName - } - return strings.TrimSpace(cleanedName[:matchIndex[0]]) -} - -func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameItem, error) { - return c.crawlPlatform("/PC/currentserverPC-FTP.json", "PC", num) -} - -func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameItem, error) { - return c.crawlPlatform("/PCVR/currentserverPCVR-FTP.json", "PCVR", num) -} - -func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameItem, error) { - num1 := num / 2 - num2 := num - num1 - if num == -1 { - num1 = -1 - num2 = -1 - } - res1, err := c.CrawlPC(num1) - if err != nil { - return nil, err - } - res2, err := c.CrawlPCVR(num2) - if err != nil { - return nil, err - } - return append(res1, res2...), nil -} - -func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameItem, error) { - return c.Crawl(-1) -} - -func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameItem, error) { - err := c.connectFTP() - if err != nil { - return nil, err - } - defer func() { _ = c.conn.Quit() }() - - data, err := c.fetchAndParseFTPData(jsonFile) - if err != nil { - return nil, err - } - - return c.crawlGames(data, platform, num) -} diff --git a/crawler/crawler.go b/crawler/crawler.go index c596a43..b3d8911 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -31,8 +31,7 @@ func BuildCrawlerMap(logger *zap.Logger) map[string]Crawler { "xatab": NewXatabCrawler(logger), "onlinefix": NewOnlineFixCrawler(logger), "steamrip": NewSteamRIPCrawler(logger), - // "armgddn": NewARMGDDNCrawler(logger), - "chovka": NewChovkaCrawler(logger), + "chovka": NewChovkaCrawler(logger), // "gnarly": NewGnarlyCrawler(logger), // "goggames": NewGOGGamesCrawler(logger), }