[ADD] new torrent site

cmd/angel/main.go

@@ -54,6 +54,7 @@ func main() {
 					&ktorrent.TorrentQQ{},
 					&ktorrent.TorrentSome{},
 					&ktorrent.TorrentRJ{},
+					&ktorrent.TorrentTop{},
 				}
 				s = common.GetAvailableSites(s)
 				fmt.Printf("[*] Angel found %d available site(s) ...\n", len(s))

common/common.go

@@ -50,6 +50,7 @@ var (
 		"torrentsome":   "https://torrentsome150.com",
 		"ktxtorrent":    "https://ktxtorrent37.com",
 		"torrentrj":     "https://torrentrj156.com",
+		"torrenttop":    "https://torrenttop118.com",
 	}
 	UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36"
 )
@@ -204,6 +205,7 @@ func GetAvailableSites(oldItems []Scraping) []Scraping {
 		"torrentmobile", "torrentview", "torrentsir",
 		"torrentj", "torrentsee", "jujutorrent",
 		"torrentqq", "torrentsome", "torrentrj",
+		"torrenttop",
 	}
 	ch := make(chan int, len(items))
 	var wg sync.WaitGroup

ktorrent/torrenttop.go

@@ -0,0 +1,90 @@
+package ktorrent
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/daite/angel/common"
+)
+
+// TorrentTop struct is for TorrentSee torrent web site
+type TorrentTop struct {
+	Name        string
+	Keyword     string
+	SearchURL   string
+	ScrapedData *sync.Map
+}
+
+// initialize method set keyword and URL based on default url
+func (t *TorrentTop) initialize(keyword string) {
+	t.Keyword = keyword
+	t.Name = "torrenttop"
+	t.SearchURL = common.TorrentURL[t.Name] + "/search/index?keywords=" + url.QueryEscape(t.Keyword)
+}
+
+// Crawl torrent data from web site
+func (t *TorrentTop) Crawl(keyword string) map[string]string {
+	t.initialize(keyword)
+	data := t.getData(t.SearchURL)
+	if data == nil {
+		return nil
+	}
+	m := map[string]string{}
+	data.Range(
+		func(key, value interface{}) bool {
+			m[fmt.Sprint(key)] = fmt.Sprint(value)
+			return true
+		})
+	return m
+}
+
+// GetData method returns map(title, bbs url)
+func (t *TorrentTop) getData(url string) *sync.Map {
+	var wg sync.WaitGroup
+	m := &sync.Map{}
+	resp, ok := common.GetResponseFromURL(url)
+	if !ok {
+		return nil
+	}
+	defer resp.Body.Close()
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return nil
+	}
+	doc.Find("l.py-4.flex.flex-row.border-b.topic-item a").Each(func(i int, s *goquery.Selection) {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			title, _ := s.Attr("title")
+			title = strings.TrimSpace(title)
+			link, _ := s.Attr("href")
+			link = strings.TrimSpace(common.URLJoin(common.TorrentURL[t.Name], link))
+			magnet := t.GetMagnet(link)
+			m.Store(title, magnet)
+		}()
+	})
+	wg.Wait()
+	t.ScrapedData = m
+	return m
+}
+
+// GetMagnet method returns torrent magnet
+func (t *TorrentTop) GetMagnet(url string) string {
+	resp, ok := common.GetResponseFromURL(url)
+	if !ok {
+		return "failed to fetch magnet"
+	}
+	defer resp.Body.Close()
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		return err.Error()
+	}
+	magnet, _ := doc.Find(".fas.fa-magnet + a").Attr("href")
+	if magnet == "" {
+		return "no magnet"
+	}
+	return magnet
+}