remove format cmd

modify README
remove armgddn crawler

README.md

@@ -9,11 +9,10 @@ pcgamedb is a powerful command-line tool designed to scrape and manage repack ga
   - Fitgirl
   - KaOSKrew
   - DODI
-  - ~~FreeGOG~~
+  - FreeGOG
-  - GOGGames
+  - ~~GOGGames~~
   - OnlineFix
   - Xatab
-  - ~~ARMGDDN~~
   - SteamRIP
   - Chovka
 

cmd/format.go

@@ -1,138 +0,0 @@
-package cmd
-
-import (
-	"strings"
-
-	"pcgamedb/crawler"
-	"pcgamedb/db"
-	"pcgamedb/log"
-
-	"github.com/spf13/cobra"
-	"go.uber.org/zap"
-)
-
-var formatCmd = &cobra.Command{
-	Use:   "format",
-	Short: "Format game downloads name by formatter",
-	Long:  "Format game downloads name by formatter",
-	Run:   formatRun,
-}
-
-type FormatCommandConfig struct {
-	Source string
-}
-
-var formatCmdCfg FormatCommandConfig
-
-func init() {
-	formatCmd.Flags().StringVarP(&formatCmdCfg.Source, "source", "s", "", "source to fix (fitgirl/dodi/kaoskrew/freegog/xatab/onlinefix/armgddn)")
-	RootCmd.AddCommand(formatCmd)
-}
-
-func formatRun(cmd *cobra.Command, args []string) {
-	formatCmdCfg.Source = strings.ToLower(formatCmdCfg.Source)
-	switch formatCmdCfg.Source {
-	case "dodi":
-		items, err := db.GetDODIGameItems()
-		if err != nil {
-			log.Logger.Error("Failed to get games", zap.Error(err))
-			return
-		}
-		for _, item := range items {
-			oldName := item.Name
-			item.Name = crawler.DODIFormatter(item.RawName)
-			if oldName != item.Name {
-				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameItem(item)
-				if err != nil {
-					log.Logger.Error("Failed to update item", zap.Error(err))
-				}
-			}
-		}
-	case "kaoskrew":
-		items, err := db.GetKaOsKrewGameItems()
-		if err != nil {
-			log.Logger.Error("Failed to get games", zap.Error(err))
-			return
-		}
-		for _, item := range items {
-			oldName := item.Name
-			item.Name = crawler.KaOsKrewFormatter(item.RawName)
-			if oldName != item.Name {
-				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameItem(item)
-				if err != nil {
-					log.Logger.Error("Failed to update item", zap.Error(err))
-				}
-			}
-		}
-	case "freegog":
-		items, err := db.GetFreeGOGGameItems()
-		if err != nil {
-			log.Logger.Error("Failed to get games", zap.Error(err))
-			return
-		}
-		for _, item := range items {
-			oldName := item.Name
-			item.Name = crawler.FreeGOGFormatter(item.RawName)
-			if oldName != item.Name {
-				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameItem(item)
-				if err != nil {
-					log.Logger.Error("Failed to update item", zap.Error(err))
-				}
-			}
-		}
-	case "xatab":
-		items, err := db.GetXatabGameItems()
-		if err != nil {
-			log.Logger.Error("Failed to get games", zap.Error(err))
-			return
-		}
-		for _, item := range items {
-			oldName := item.Name
-			item.Name = crawler.XatabFormatter(item.RawName)
-			if oldName != item.Name {
-				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameItem(item)
-				if err != nil {
-					log.Logger.Error("Failed to update item", zap.Error(err))
-				}
-			}
-		}
-	case "onlinefix":
-		items, err := db.GetOnlineFixGameItems()
-		if err != nil {
-			log.Logger.Error("Failed to get games", zap.Error(err))
-			return
-		}
-		for _, item := range items {
-			oldName := item.Name
-			item.Name = crawler.OnlineFixFormatter(item.RawName)
-			if oldName != item.Name {
-				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameItem(item)
-				if err != nil {
-					log.Logger.Error("Failed to update item", zap.Error(err))
-				}
-			}
-		}
-	case "armgddn":
-		items, err := db.GetARMGDDNGameItems()
-		if err != nil {
-			log.Logger.Error("Failed to get games", zap.Error(err))
-			return
-		}
-		for _, item := range items {
-			oldName := item.Name
-			item.Name = crawler.ARMGDDNFormatter(item.RawName)
-			if oldName != item.Name {
-				log.Logger.Info("Fix name", zap.String("old", oldName), zap.String("raw", item.RawName), zap.String("name", item.Name))
-				err := db.SaveGameItem(item)
-				if err != nil {
-					log.Logger.Error("Failed to update item", zap.Error(err))
-				}
-			}
-		}
-	}
-}

crawler/armgddn.go

@@ -1,214 +0,0 @@
-package crawler
-
-import (
-	"crypto/tls"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/url"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"pcgamedb/db"
-	"pcgamedb/model"
-	"pcgamedb/utils"
-
-	"github.com/jlaffaye/ftp"
-	"go.uber.org/zap"
-)
-
-const (
-	ftpAddress  = "72.21.17.26:13017"
-	ftpUsername = "ARMGDDNGames"
-	ftpPassword = "ARMGDDNGames"
-)
-
-type GameData struct {
-	NumberOfGame string `json:"Number of game"`
-	AppID        string `json:"appid"`
-	FolderName   string `json:"foldername"`
-}
-
-type ARMGDDNCrawler struct {
-	logger zap.Logger
-	conn   *ftp.ServerConn
-}
-
-// Deprecated: ARMGDDN has changed resource distribution method
-func NewARMGDDNCrawler(logger *zap.Logger) *ARMGDDNCrawler {
-	return &ARMGDDNCrawler{
-		logger: *logger,
-	}
-}
-
-func (c *ARMGDDNCrawler) connectFTP() error {
-	var err error
-	tlsConfig := &tls.Config{InsecureSkipVerify: true}
-	c.conn, err = ftp.Dial(ftpAddress, ftp.DialWithTimeout(5*time.Second), ftp.DialWithExplicitTLS(tlsConfig))
-	if err != nil {
-		return err
-	}
-	if err = c.conn.Login(ftpUsername, ftpPassword); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (c *ARMGDDNCrawler) fetchAndParseFTPData(filePath string) ([]GameData, error) {
-	r, err := c.conn.Retr(filePath)
-	if err != nil {
-		return nil, err
-	}
-	defer r.Close()
-
-	buf, err := io.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
-
-	var data []GameData
-	if err = json.Unmarshal(buf, &data); err != nil {
-		return nil, err
-	}
-	return data, nil
-}
-
-func (c *ARMGDDNCrawler) crawlGames(data []GameData, platform string, num int) ([]*model.GameItem, error) {
-	count := 0
-	var res []*model.GameItem
-	modTimeMap := make(map[string]time.Time)
-	entries, err := c.conn.List(fmt.Sprintf("/%s", platform))
-	if err != nil {
-		return nil, err
-	}
-	for _, entry := range entries {
-		if entry.Type == ftp.EntryTypeFolder {
-			modTimeMap[entry.Name] = entry.Time
-		}
-	}
-	for _, v := range data {
-		if count == num {
-			break
-		}
-		path := fmt.Sprintf("/%s/%s", platform, v.FolderName)
-		u := fmt.Sprintf("ARMGDDNGames/%s/%s", platform, v.NumberOfGame)
-		modTime, ok := modTimeMap[v.FolderName]
-		if !ok {
-			c.logger.Warn("mod time not found", zap.String("url", u))
-			continue
-		}
-		updateFlag := fmt.Sprintf("ARMGDDNGames/%s/%s/%s", platform, v.NumberOfGame, modTime.UTC().String())
-		if db.IsARMGDDNCrawled(updateFlag) {
-			continue
-		}
-		c.logger.Info("Crawling", zap.String("url", u))
-		walker := c.conn.Walk(path)
-		size := uint64(0)
-		for walker.Next() {
-			if walker.Stat().Type == ftp.EntryTypeFile {
-				fileSize, err := c.conn.FileSize(walker.Path())
-				if err != nil {
-					c.logger.Warn("file size error", zap.Error(err))
-					break
-				}
-				size += uint64(fileSize)
-			}
-		}
-		item, err := db.GetGameItemByUrl(u)
-		if err != nil {
-			continue
-		}
-		item.Url = u
-		item.Name = ARMGDDNFormatter(v.FolderName)
-		item.UpdateFlag = updateFlag
-		item.Size = utils.BytesToSize(size)
-		item.RawName = v.FolderName
-		item.Author = "ARMGDDN"
-		item.Download = fmt.Sprintf("ftpes://%s:%s@%s/%s/%s", ftpUsername, ftpPassword, ftpAddress, platform, url.QueryEscape(v.FolderName))
-		if err := db.SaveGameItem(item); err != nil {
-			continue
-		}
-		res = append(res, item)
-		count++
-		var id int
-		var info *model.GameInfo
-		if v.AppID != "NONSTEAM" {
-			id, err = strconv.Atoi(v.AppID)
-			if err != nil {
-				c.logger.Warn("strconv error", zap.Error(err))
-				continue
-			}
-			info, err = OrganizeGameItemWithSteam(id, item)
-			if err != nil {
-				continue
-			}
-		} else {
-			info, err = OrganizeGameItem(item)
-			if err != nil {
-				continue
-			}
-		}
-		err = db.SaveGameInfo(info)
-		if err != nil {
-			c.logger.Warn("save game info error", zap.Error(err))
-			continue
-		}
-	}
-	return res, nil
-}
-
-func ARMGDDNFormatter(name string) string {
-	cleanedName := strings.ReplaceAll(strings.TrimSpace(name), "-ARMGDDN", "")
-	matchIndex := regexp.MustCompile(`v\d`).FindStringIndex(cleanedName)
-	if matchIndex == nil {
-		return cleanedName
-	}
-	return strings.TrimSpace(cleanedName[:matchIndex[0]])
-}
-
-func (c *ARMGDDNCrawler) CrawlPC(num int) ([]*model.GameItem, error) {
-	return c.crawlPlatform("/PC/currentserverPC-FTP.json", "PC", num)
-}
-
-func (c *ARMGDDNCrawler) CrawlPCVR(num int) ([]*model.GameItem, error) {
-	return c.crawlPlatform("/PCVR/currentserverPCVR-FTP.json", "PCVR", num)
-}
-
-func (c *ARMGDDNCrawler) Crawl(num int) ([]*model.GameItem, error) {
-	num1 := num / 2
-	num2 := num - num1
-	if num == -1 {
-		num1 = -1
-		num2 = -1
-	}
-	res1, err := c.CrawlPC(num1)
-	if err != nil {
-		return nil, err
-	}
-	res2, err := c.CrawlPCVR(num2)
-	if err != nil {
-		return nil, err
-	}
-	return append(res1, res2...), nil
-}
-
-func (c *ARMGDDNCrawler) CrawlAll() ([]*model.GameItem, error) {
-	return c.Crawl(-1)
-}
-
-func (c *ARMGDDNCrawler) crawlPlatform(jsonFile, platform string, num int) ([]*model.GameItem, error) {
-	err := c.connectFTP()
-	if err != nil {
-		return nil, err
-	}
-	defer func() { _ = c.conn.Quit() }()
-
-	data, err := c.fetchAndParseFTPData(jsonFile)
-	if err != nil {
-		return nil, err
-	}
-
-	return c.crawlGames(data, platform, num)
-}

crawler/crawler.go

@@ -31,8 +31,7 @@ func BuildCrawlerMap(logger *zap.Logger) map[string]Crawler {
 		"xatab":     NewXatabCrawler(logger),
 		"onlinefix": NewOnlineFixCrawler(logger),
 		"steamrip":  NewSteamRIPCrawler(logger),
-		// "armgddn":   NewARMGDDNCrawler(logger),
+		"chovka":    NewChovkaCrawler(logger),
-		"chovka": NewChovkaCrawler(logger),
 		// "gnarly":   NewGnarlyCrawler(logger),
 		// "goggames": NewGOGGamesCrawler(logger),
 	}