140 lines
3.4 KiB
Go
140 lines
3.4 KiB
Go
package collectors
|
|
|
|
import (
|
|
"crowsnest/internal/model"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/gocolly/colly/v2"
|
|
)
|
|
|
|
func (c *Collector) CollectSpiegel() {
|
|
collycollector := colly.NewCollector(
|
|
colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
|
|
colly.CacheDir("./persistence/spiegel_cache"),
|
|
colly.MaxDepth(5),
|
|
)
|
|
|
|
// store articles
|
|
collycollector.OnResponse(func(r *colly.Response) {
|
|
url := r.Request.URL.String()
|
|
err := c.ExtractSpiegel(url, r.Body)
|
|
if err == nil {
|
|
log.Println("added article", url)
|
|
} else {
|
|
log.Println("failed to add article:", err, "("+url+")")
|
|
}
|
|
})
|
|
|
|
// cascade
|
|
collycollector.OnHTML("a[href]", func(e *colly.HTMLElement) {
|
|
url := e.Attr("href")
|
|
|
|
if !strings.HasPrefix(url, "http") {
|
|
return
|
|
}
|
|
e.Request.Visit(url)
|
|
})
|
|
|
|
// go through archive
|
|
startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
currentDate := time.Now()
|
|
|
|
for date := startDate; date.Before(currentDate) || date.Equal(currentDate); date = date.AddDate(0, 0, 1) {
|
|
urlDate := date.Format("02.01.2006")
|
|
url := fmt.Sprintf("https://www.spiegel.de/nachrichtenarchiv/artikel-%s.html", urlDate)
|
|
|
|
collycollector.Visit(url)
|
|
}
|
|
}
|
|
|
|
func (c *Collector) ExtractSpiegel(url string, body []byte) error {
|
|
paywall_pattern := regexp.MustCompile(`"paywall":{"attributes":{"is_active":true`)
|
|
url_pattern := regexp.MustCompile(`^https://(www\.)?spiegel.de.*`)
|
|
whitespace := regexp.MustCompile(`\s+`)
|
|
|
|
var exists bool
|
|
var pagetype, title, content, datestr string
|
|
var tag *goquery.Selection
|
|
var date time.Time
|
|
|
|
// check url url pattern
|
|
if !url_pattern.Match([]byte(url)) {
|
|
return errors.New("invalid url pattern")
|
|
}
|
|
|
|
// check for paywall
|
|
if paywall_pattern.Match(body) {
|
|
return errors.New("unable to extract article due to paywal")
|
|
}
|
|
|
|
// construct goquery doc
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(body)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// check for article type
|
|
tag = doc.Find("meta[property='og:type']")
|
|
pagetype, exists = tag.Attr("content")
|
|
if !exists || pagetype != "article" {
|
|
return errors.New("unable to extract article, not of type article")
|
|
}
|
|
|
|
// get title
|
|
tag = doc.Find("meta[property='og:title']")
|
|
title, exists = tag.Attr("content")
|
|
if !exists {
|
|
return errors.New("unable to extract article, no title tag")
|
|
}
|
|
|
|
// prepend description to content of article
|
|
tag = doc.Find("meta[name='description']")
|
|
content, exists = tag.Attr("content")
|
|
content += " "
|
|
if !exists {
|
|
return errors.New("unable to extract article, no description tag")
|
|
}
|
|
|
|
// get publishing date
|
|
tag = doc.Find("meta[name='date']")
|
|
datestr, exists = tag.Attr("content")
|
|
if !exists {
|
|
return errors.New("unable to extract article, no date tag")
|
|
}
|
|
|
|
date, err = time.Parse("2006-01-02T15:04:05-07:00", datestr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// get content
|
|
tag = doc.Find("main[id='Inhalt'] div > p")
|
|
|
|
tag.Each(func(index int, p *goquery.Selection) {
|
|
content += " " + p.Text()
|
|
})
|
|
|
|
// clean up content string
|
|
content = string(whitespace.ReplaceAll([]byte(content), []byte(" ")))
|
|
content = strings.ReplaceAll(content, "»", "\"")
|
|
content = strings.ReplaceAll(content, "«", "\"")
|
|
|
|
// insert new article
|
|
article := model.Article{
|
|
SourceUrl: url,
|
|
PublishDate: date,
|
|
FetchDate: time.Now(),
|
|
Title: title,
|
|
Content: content,
|
|
}
|
|
|
|
err = c.Articles.Insert(&article)
|
|
return err
|
|
}
|