extend links range

wedojava · wedojava · commit 89c2441baa71 · 2020-08-26T02:41:29.000+08:00
diff --git a/internal/fetcher/links.go b/internal/fetcher/links.go
@@ -35,7 +35,11 @@ func (f *Fetcher) SetLinks() error {
 		f.Links = LinksFilter(links, `.*?/.*?-\d*.html`)
 		KickOutLinksMatchPath(&f.Links, "about")
 	case "www.zaobao.com":
-		f.Links = LinksFilter(links, `.*?/world/.*`)
+		newsWorld := LinksFilter(links, `.*?/news/world/.*`)
+		newsChina := LinksFilter(links, `.*?/news/china/.*`)
+		realtimeWorld := LinksFilter(links, `.*?/realtime/world/.*`)
+		realtimeChina := LinksFilter(links, `.*?/realtime/china/.*`)
+		f.Links = append(append(append(newsWorld, newsChina...), realtimeWorld...), realtimeChina...)
 	}
 	return nil
 }
diff --git a/internal/fetcher/links_test.go b/internal/fetcher/links_test.go
@@ -1,7 +1,6 @@
 package fetcher
 
 import (
-	"fmt"
 	"net/url"
 	"testing"
 )
@@ -18,23 +17,29 @@ func TestKickOutLinksMatchPath(t *testing.T) {
 }
 
 func TestSetLinks(t *testing.T) {
-	u, err := url.Parse("https://www.zaobao.com/realtime/world")
+	u, err := url.Parse("https://www.zaobao.com/realtime/")
 	if err != nil {
 		t.Errorf("Url Parse fail!\n%s", err)
 	}
 	var f = &Fetcher{
 		Entrance: u,
 	}
 	f.SetLinks()
-	assertLink := "https://www.zaobao.com/realtime/world/story20200825-1079575"
+	assertLinks := []string{
+		"https://www.zaobao.com/realtime/china/story20200825-1079597",
+		"https://www.zaobao.com/realtime/world/story20200825-1079575",
+		"https://www.zaobao.com/news/china/story20200825-1079462",
+		"https://www.zaobao.com/news/world/story20200825-1079476",
+	}
 	shot := 0
 	for _, link := range f.Links {
-		fmt.Println(link)
-		if link == assertLink {
-			shot++
+		for _, v := range assertLinks {
+			if link == v {
+				shot++
+			}
 		}
 	}
 	if shot == 0 {
-		t.Errorf("want: %v, got: %v", 1, shot)
+		t.Errorf("want: %v, got: %v", len(assertLinks), shot)
 	}
 }

Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,11 @@ func (f *Fetcher) SetLinks() error {`
`35`	`35`	f.Links = LinksFilter(links, `.?/.?-\d*.html`)
`36`	`36`	`KickOutLinksMatchPath(&f.Links, "about")`
`37`	`37`	`case "www.zaobao.com":`
`38`		- f.Links = LinksFilter(links, `.?/world/.`)
	`38`	+ newsWorld := LinksFilter(links, `.?/news/world/.`)
	`39`	+ newsChina := LinksFilter(links, `.?/news/china/.`)
	`40`	+ realtimeWorld := LinksFilter(links, `.?/realtime/world/.`)
	`41`	+ realtimeChina := LinksFilter(links, `.?/realtime/china/.`)
	`42`	`+ f.Links = append(append(append(newsWorld, newsChina...), realtimeWorld...), realtimeChina...)`
`39`	`43`	`}`
`40`	`44`	`return nil`
`41`	`45`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,6 @@`
`1`	`1`	`package fetcher`
`2`	`2`
`3`	`3`	`import (`
`4`		`- "fmt"`
`5`	`4`	`"net/url"`
`6`	`5`	`"testing"`
`7`	`6`	`)`
`@@ -18,23 +17,29 @@ func TestKickOutLinksMatchPath(t *testing.T) {`
`18`	`17`	`}`
`19`	`18`
`20`	`19`	`func TestSetLinks(t *testing.T) {`
`21`		`- u, err := url.Parse("https://www.zaobao.com/realtime/world")`
	`20`	`+ u, err := url.Parse("https://www.zaobao.com/realtime/")`
`22`	`21`	`if err != nil {`
`23`	`22`	`t.Errorf("Url Parse fail!\n%s", err)`
`24`	`23`	`}`
`25`	`24`	`var f = &Fetcher{`
`26`	`25`	`Entrance: u,`
`27`	`26`	`}`
`28`	`27`	`f.SetLinks()`
`29`		`- assertLink := "https://www.zaobao.com/realtime/world/story20200825-1079575"`
	`28`	`+ assertLinks := []string{`
	`29`	`+ "https://www.zaobao.com/realtime/china/story20200825-1079597",`
	`30`	`+ "https://www.zaobao.com/realtime/world/story20200825-1079575",`
	`31`	`+ "https://www.zaobao.com/news/china/story20200825-1079462",`
	`32`	`+ "https://www.zaobao.com/news/world/story20200825-1079476",`
	`33`	`+ }`
`30`	`34`	`shot := 0`
`31`	`35`	`for _, link := range f.Links {`
`32`		`- fmt.Println(link)`
`33`		`- if link == assertLink {`
`34`		`- shot++`
	`36`	`+ for _, v := range assertLinks {`
	`37`	`+ if link == v {`
	`38`	`+ shot++`
	`39`	`+ }`
`35`	`40`	`}`
`36`	`41`	`}`
`37`	`42`	`if shot == 0 {`
`38`		`- t.Errorf("want: %v, got: %v", 1, shot)`
	`43`	`+ t.Errorf("want: %v, got: %v", len(assertLinks), shot)`
`39`	`44`	`}`
`40`	`45`	`}`