Added: downloading and returning wikipedia articles

internal/wikipediadl/const.go

@@ -0,0 +1,6 @@
+package wikipediadl
+
+const (
+	WikipediaDumpDomain = "dumps.wikimedia.org"
+	WikipediaDumpUrl    = "https://" + WikipediaDumpDomain + "/plwiki/latest/"
+)

internal/wikipediadl/downloadarticles.go

@@ -1,79 +0,0 @@
-package wikipediadl
-
-import (
-	"compress/bzip2"
-	"encoding/xml"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-)
-
-type Page struct {
-	Title    string   `xml:"title"`
-	Revision Revision `xml:"revision"`
-}
-
-type Revision struct {
-	Text string `xml:"text"`
-}
-
-func DownloadArticles() {
-	url := "https://dumps.wikimedia.org/plwiki/latest/plwiki-latest-pages-articles1.xml-p1p187037.bz2"
-
-	limiter := flag.Int("limit", 10, "Number of articles to process; 0 means all")
-	flag.Parse()
-
-	log.Printf("Downloading chunk: %s", url)
-	resp, err := http.Get(url)
-	if err != nil {
-		log.Fatalf("Failed to download chunk: %v", err)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
-		log.Fatalf("Bad response status: %s\nBody: %s", resp.Status, string(body))
-	}
-
-	bz2Reader := bzip2.NewReader(resp.Body)
-	dec := xml.NewDecoder(bz2Reader)
-
-	count := 0
-	for {
-		tok, err := dec.Token()
-		if err != nil {
-			if err == io.EOF {
-				log.Println("Reached end of chunk")
-				break
-			}
-			log.Fatalf("XML token error: %v", err)
-		}
-
-		switch se := tok.(type) {
-		case xml.StartElement:
-			if se.Name.Local == "page" {
-				var p Page
-				if err := dec.DecodeElement(&p, &se); err != nil {
-					log.Printf("Error decoding page: %v", err)
-					continue
-				}
-
-				count++
-				fmt.Printf("---- Article %d ----\n", count)
-				fmt.Printf("Title: %s\n", p.Title)
-				// fmt.Println("Content:")
-				// fmt.Println(p.Revision.Text)
-				fmt.Println("--------------------\n")
-
-				if *limiter > 0 && count >= *limiter {
-					log.Printf("Reached limit of %d articles, stopping.", *limiter)
-					return
-				}
-			}
-		}
-	}
-
-	log.Printf("Done. Total articles processed: %d", count)
-}

internal/wikipediadl/error.go

@@ -3,5 +3,6 @@ package wikipediadl
 import "errors"
 
 var (
-	ErrArticleFetchFailed = errors.New("wikipediadl: failed to fetch articles")
+	ErrArticleBundleFetchFailed = errors.New("wikipediadl: failed to fetch article bundles")
+	ErrArticleDownloadFailed    = errors.New("wikipediadl: failed to extract articles")
 )

internal/wikipediadl/extractarticles.go

@@ -0,0 +1,75 @@
+package wikipediadl
+
+import (
+	"compress/bzip2"
+	"encoding/xml"
+	"errors"
+	"io"
+	"log"
+	"net/http"
+)
+
+type WikiArticle struct {
+	Title    string   `xml:"title"`
+	Revision Revision `xml:"revision"`
+}
+
+type Revision struct {
+	Text string `xml:"text"`
+}
+
+func ExtractArticles(bundle string) ([]WikiArticle, error) {
+	url := WikipediaDumpUrl + bundle
+
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Println(err.Error())
+		return nil, errors.New("wikipediadl: failed load articles")
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, errors.New("wikipediadl: bad response status")
+	}
+
+	bz2Reader := bzip2.NewReader(resp.Body)
+	xmlDec := xml.NewDecoder(bz2Reader)
+
+	count := 0
+
+	articles := []WikiArticle{}
+Loop:
+	for {
+		tok, err := xmlDec.Token()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+
+			return nil, errors.New("XML token error")
+		}
+
+		switch se := tok.(type) {
+		case xml.StartElement:
+			if count == 2 { // XXX: remove later
+				break Loop
+			}
+
+			if se.Name.Local != "page" {
+				continue
+			}
+
+			var p WikiArticle
+			if err := xmlDec.DecodeElement(&p, &se); err != nil {
+				log.Println(err.Error())
+				continue
+			}
+
+			articles = append(articles, p)
+			count++ // XXX: remove later
+		}
+	}
+
+	return articles, nil
+}

internal/wikipediadl/fetcharticles.go

@@ -1,7 +1,6 @@
 package wikipediadl
 
 import (
-	"fmt"
 	"log"
 	"strings"
 
@@ -13,12 +12,17 @@ const (
 	DumpUrl    = "https://dumps.wikimedia.org/plwiki/latest/"
 )
 
+func FetchArticleBundles() ([]string, error) {
+	scraper := getScraper()
+
+	articles := getAllArticles(scraper)
+	return articles, nil
+}
+
 func getScraper() *colly.Collector {
-	s := colly.NewCollector(
+	return colly.NewCollector(
 		colly.AllowedDomains(DumpDomain),
 	)
-
-	return s
 }
 
 func getAllArticles(s *colly.Collector) []string {
@@ -58,14 +62,3 @@ func isValidArticle(a string) bool {
 	articleIndex := article[0]
 	return articleIndex >= 48 && articleIndex <= 57
 }
-
-func FetchArticles() error {
-	scraper := getScraper()
-
-	articles := getAllArticles(scraper)
-	for _, a := range articles {
-		fmt.Println(a)
-	}
-
-	return nil
-}