eliaskohout/crowsnest
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

change db from sqlite3 to postgresql

Browse Source
This commit is contained in:
Elias Kohout
2025-01-07 09:32:57 +01:00
parent f719c73b46
commit 9302e982c6
20 changed files with 309 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
cmd/crawler/collectors/spiegel.go
View File

@@ -1,110 +1,59 @@
package collectors
import (
//"crowsnest/internal/model"
//"regexp"
//"time"
//"strings"
"crowsnest/internal/model"
"fmt"
"log"
"strings"
"time"
"github.com/gocolly/colly/v2"
)
func (c *Collector) Spiegel() {
collycollector := colly.NewCollector(
colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
colly.CacheDir("./persistence/spiegel_cache"),
colly.MaxDepth(3),
colly.MaxDepth(2),
)
// cache
collycollector.OnRequest(func(r *colly.Request) {
url := r.URL.String()
exists, err := c.Responses.UrlExists(url)
if err == nil && !exists {
c.Responses.Insert(url, nil)
log.Println("request", url)
} else {
r.Abort()
}
})
collycollector.OnResponse(func(r *colly.Response) {
url := r.Request.URL.String()
c.Responses.Update(&model.Response{Url: url, Content: r.Body, FetchDate: time.Now(), Processed: false})
log.Println("response cached", url)
})
// cascade
collycollector.OnHTML("a[href]", func(e *colly.HTMLElement) {
e.Request.Visit(e.Attr("href"))
url := e.Attr("href")
log.Println("found", url)
if !strings.HasPrefix(url, "http") {
return
}
log.Println("visiting", url)
e.Request.Visit(url)
})
// cache
collycollector.OnScraped(func(r *colly.Response) {
c.Responses.Insert(r.Request.URL.String(), string(r.Body))
})
// go through archive
startDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)
// go through archive
startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
currentDate := time.Now()
for date := startDate; date.Before(currentDate) || date.Equal(currentDate); date = date.AddDate(0, 0, 1) {
urlDate := date.Format("02.01.2006")
url := fmt.Sprintf("https://www.spiegel.de/nachrichtenarchiv/artikel-%s.html", urlDate)
collycollector.Visit(url)
collycollector.Visit(url)
}
//// create entry if not behind paywall
//paywall_false_pattern := regexp.MustCompile("\"paywall\":{\"attributes\":{\"is_active\":false")
//collycollector.OnResponse(func(r *colly.Response) {
// if paywall_false_pattern.Match(r.Body) {
// url := r.Request.URL.String()
// (*results)[url] = &model.Article{
// SourceUrl: url,
// FetchDate: time.Now(),
// Content: "",
// }
// }
//})
//// check for article type
//collycollector.OnHTML("meta[property='og:type']", func(e *colly.HTMLElement) {
// if e.Attr("content") != "article" {
// delete(*results, e.Request.URL.String())
// }
//})
//// add title
//collycollector.OnHTML("meta[property='og:title']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// val.Title = e.Attr("content")
// }
//})
//// prepend description to content of article
//collycollector.OnHTML("meta[name='description']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// val.Content = e.Attr("content") + val.Content
// }
//})
//// add publishing date
//collycollector.OnHTML("meta[name='date']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// t, err := time.Parse("2006-01-02T15:04:05-07:00", e.Attr("content"))
// if err != nil {
// panic(err)
// }
// val.PublishDate = t
// }
//})
//// add author
//collycollector.OnHTML("meta[name='author']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// val.Author = e.Attr("content")
// }
//})
//// add content
//collycollector.OnHTML("main[id='Inhalt'] div > p", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// cont := val.Content
// pattern := regexp.MustCompile("\\s+")
// cont = string(pattern.ReplaceAll([]byte(cont), []byte(" ")))
// cont = strings.ReplaceAll(cont, "»", "\"")
// cont = strings.ReplaceAll(cont, "«", "\"")
// val.Content = cont + " " + e.Text
// }
//})
}