rfa body fetching test passed.

wedojava · wedojava · commit b0555b38ef1c · 2020-07-06T15:13:15.000+08:00
diff --git a/internal/fetcher/body.go b/internal/fetcher/body.go
@@ -23,6 +23,10 @@ func (p *Post) SetBody() error {
 		if err := p.FmtBody(Voa); err != nil {
 			return err
 		}
+	case "www.rfa.org":
+		if err := p.FmtBody(Voa); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -128,3 +132,22 @@ func Voa(p *Post) (string, error) {
 	body = strings.ReplaceAll(body, "br  \n", "")
 	return body, nil
 }
+
+func Rfa(p *Post) (string, error) {
+	doc := p.DOC
+	body := ""
+	// Fetch content nodes
+	articleDoc := ElementsByTagAndClass(doc, "div", "wsw")
+	if len(articleDoc) == 0 {
+		return "", errors.New(`[-] There is no element match '<div class="wsw">'`)
+	}
+	plist := ElementsByTagName(articleDoc[0], "p")
+	for _, v := range plist {
+		body += v.FirstChild.Data + "  \n"
+	}
+	body = strings.ReplaceAll(body, "strong  \n", "")
+	body = strings.ReplaceAll(body, "span  \n", "")
+	body = strings.ReplaceAll(body, "br  \n", "")
+	return body, nil
+
+}
diff --git a/internal/fetcher/node.go b/internal/fetcher/node.go
@@ -40,6 +40,26 @@ func ElementsByTagAndClass(doc *html.Node, tag, class string) []*html.Node {
 	return nodes
 }
 
+func ElementsByTagAndId(doc *html.Node, tag, id string) []*html.Node {
+	var nodes []*html.Node
+	if tag == "" || id == "" {
+		return nil
+	}
+	if doc.Type == html.ElementNode {
+		if tag == doc.Data {
+			for _, a := range doc.Attr {
+				if a.Key == "id" && a.Val == id {
+					nodes = append(nodes, doc)
+				}
+			}
+		}
+	}
+	for c := doc.FirstChild; c != nil; c = c.NextSibling {
+		nodes = append(nodes, ElementsByTagAndId(c, tag, id)...)
+	}
+	return nodes
+}
+
 func ForEachNode(n *html.Node, pre, post func(n *html.Node)) {
 	if pre != nil {
 		pre(n)
diff --git a/internal/fetcher/node_test.go b/internal/fetcher/node_test.go
@@ -20,3 +20,24 @@ func TestElementsByTagAndClass(t *testing.T) {
 		fmt.Println(v.FirstChild.Data)
 	}
 }
+
+func TestElementsByTagAndId(t *testing.T) {
+	p := PostFactory("https://www.rfa.org/mandarin/yataibaodao/junshiwaijiao/jt-07022020105416.html")
+	raw, doc, err := GetRawAndDoc(p.URL, 1*time.Minute)
+	if err != nil {
+		t.Errorf("GetDOC error: %v", err)
+	}
+	p.DOC = doc
+	p.Raw = raw
+	tc := ElementsByTagAndId(doc, "div", "storytext")
+	plist := ElementsByTagName(tc[0], "p")
+	for _, v := range plist {
+		if v.FirstChild != nil {
+			if v.FirstChild.Data == "b" {
+				fmt.Println("**" + v.FirstChild.FirstChild.Data + "**")
+			} else {
+				fmt.Println(v.FirstChild.Data)
+			}
+		}
+	}
+}
diff --git a/internal/fetcher/post.go b/internal/fetcher/post.go
@@ -45,8 +45,7 @@ func (p *Post) SetPost() error {
 	if err != nil {
 		return err
 	}
-	p.Raw = raw
-	p.DOC = doc
+	p.Raw, p.DOC = raw, doc
 	// set Date
 	if err := p.SetDate(); err != nil {
 		return err
