115 lines
2.7 KiB
Go
115 lines
2.7 KiB
Go
package main
|
|
|
|
import (
|
|
"crowsnest/internal/model"
|
|
"crowsnest/internal/model/sqlite"
|
|
"database/sql"
|
|
"fmt"
|
|
"log"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/gocolly/colly/v2"
|
|
_ "github.com/mattn/go-sqlite3"
|
|
)
|
|
|
|
func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
|
|
c := colly.NewCollector(
|
|
colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
|
|
colly.CacheDir("./persistence/spiegel_cache"),
|
|
colly.MaxDepth(5),
|
|
)
|
|
|
|
// create entry if not behind paywall
|
|
paywall_false_pattern := regexp.MustCompile("\"paywall\":{\"attributes\":{\"is_active\":false")
|
|
c.OnResponse(func(r *colly.Response) {
|
|
if paywall_false_pattern.Match(r.Body) {
|
|
url := r.Request.URL.String()
|
|
(*results)[url] = &model.Article{
|
|
SourceUrl: url,
|
|
FetchDate: time.Now(),
|
|
Content: "",
|
|
}
|
|
}
|
|
|
|
})
|
|
|
|
// check for article type
|
|
c.OnHTML("meta[property='og:type']", func(e *colly.HTMLElement) {
|
|
if e.Attr("content") != "article" {
|
|
}
|
|
})
|
|
|
|
// add title
|
|
c.OnHTML("meta[property='og:title']", func(e *colly.HTMLElement) {
|
|
if val, ok := (*results)[e.Request.URL.String()]; ok {
|
|
val.Title = e.Attr("content")
|
|
}
|
|
})
|
|
|
|
// prepend description to content of article
|
|
c.OnHTML("meta[name='description']", func(e *colly.HTMLElement) {
|
|
if val, ok := (*results)[e.Request.URL.String()]; ok {
|
|
val.Content = e.Attr("content") + val.Content
|
|
}
|
|
})
|
|
|
|
// add publishing date
|
|
c.OnHTML("meta[name='date']", func(e *colly.HTMLElement) {
|
|
if val, ok := (*results)[e.Request.URL.String()]; ok {
|
|
t, err := time.Parse("2006-01-02T15:04:05-07:00", e.Attr("content"))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
val.PublishDate = t
|
|
}
|
|
})
|
|
|
|
// add author
|
|
c.OnHTML("meta[name='author']", func(e *colly.HTMLElement) {
|
|
if val, ok := (*results)[e.Request.URL.String()]; ok {
|
|
val.Author = e.Attr("content")
|
|
}
|
|
})
|
|
|
|
// add content
|
|
c.OnHTML("main[id='Inhalt'] div > p", func(e *colly.HTMLElement) {
|
|
if val, ok := (*results)[e.Request.URL.String()]; ok {
|
|
cont := val.Content
|
|
|
|
pattern := regexp.MustCompile("\\s+")
|
|
cont = string(pattern.ReplaceAll([]byte(cont), []byte(" ")))
|
|
cont = strings.ReplaceAll(cont, "»", "\"")
|
|
cont = strings.ReplaceAll(cont, "«", "\"")
|
|
val.Content = cont + " " + e.Text
|
|
}
|
|
})
|
|
|
|
// cascade
|
|
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
|
|
e.Request.Visit(e.Attr("href"))
|
|
})
|
|
|
|
return c
|
|
}
|
|
|
|
func main() {
|
|
res := make(map[string]*model.Article)
|
|
c := spiegelCollector(&res)
|
|
|
|
c.Visit("https://www.spiegel.de/")
|
|
|
|
db, err := sql.Open("sqlite3", "./persistence/app.db")
|
|
if err != nil { log.Fatal(err) }
|
|
|
|
db_articles := &sqlite.ArticleModel{ DB: db }
|
|
|
|
counter := 0
|
|
for _, val := range res {
|
|
counter++
|
|
db_articles.Insert(val)
|
|
}
|
|
fmt.Println(counter)
|
|
}
|