crowsnest/cmd/crawler/collectors/spiegel.go

package collectors

import (
	"crowsnest/internal/model"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/gocolly/colly/v2"
)

func (c *Collector) Spiegel() {
	collycollector := colly.NewCollector(
		colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
		colly.CacheDir("./persistence/spiegel_cache"),
		colly.MaxDepth(2),
	)

	// cache
	collycollector.OnRequest(func(r *colly.Request) {
		url := r.URL.String()
		exists, err := c.Responses.UrlExists(url)
		if err == nil && !exists {
			c.Responses.Insert(url, nil)
			log.Println("request", url)
		} else {
			r.Abort()
		}
	})
	collycollector.OnResponse(func(r *colly.Response) {
		url := r.Request.URL.String()
		c.Responses.Update(&model.Response{Url: url, Content: r.Body, FetchDate: time.Now(), Processed: false})
		log.Println("response cached", url)
	})

	// cascade
	collycollector.OnHTML("a[href]", func(e *colly.HTMLElement) {
		url := e.Attr("href")
		log.Println("found", url)

		if !strings.HasPrefix(url, "http") {
			return
		}
		log.Println("visiting", url)
		e.Request.Visit(url)
	})

	// go through archive
	startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
	currentDate := time.Now()

	for date := startDate; date.Before(currentDate) || date.Equal(currentDate); date = date.AddDate(0, 0, 1) {
		urlDate := date.Format("02.01.2006")
		url := fmt.Sprintf("https://www.spiegel.de/nachrichtenarchiv/artikel-%s.html", urlDate)

		collycollector.Visit(url)
	}
}