src/internal/crawler/SpiegelConverter.go

@@ -1,96 +0,0 @@
-package crawler
-
-import (
-	"crowsnest/internal/model"
-	"errors"
-	"regexp"
-	"strings"
-	"time"
-
-	"github.com/PuerkitoBio/goquery"
-)
-
-type ConverterSpiegel struct {
-	pattern_paywall    *regexp.Regexp
-	pattern_url        *regexp.Regexp
-	pattern_whitespace *regexp.Regexp
-}
-
-func (c *ConverterSpiegel) Init() {
-	c.pattern_paywall = regexp.MustCompile(`"paywall":{"attributes":{"is_active":true`)
-	c.pattern_url = regexp.MustCompile(`^https://(www\.)?spiegel.de.*`)
-	c.pattern_whitespace = regexp.MustCompile(`\s+`)
-}
-
-func (c *ConverterSpiegel) Convert(res *Resource) (*model.Article, error) {
-	// check url url pattern
-	if !c.pattern_url.Match([]byte(res.Url)) {
-		return nil, errors.New("invalid url pattern")
-	}
-
-	// check for paywall
-	if c.pattern_paywall.Match([]byte(res.Body)) {
-		return nil, errors.New("unable to extract article due to paywal")
-	}
-
-	// construct goquery doc
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(res.Body))
-	if err != nil {
-		return nil, err
-	}
-
-	// check for article type
-	tag := doc.Find("meta[property='og:type']")
-	pagetype, exists := tag.Attr("content")
-	if !exists || pagetype != "article" {
-		return nil, errors.New("unable to extract article, not of type article")
-	}
-
-	// get title
-	tag = doc.Find("meta[property='og:title']")
-	title, exists := tag.Attr("content")
-	if !exists {
-		return nil, errors.New("unable to extract article, no title tag")
-	}
-
-	// prepend description to content of article
-	tag = doc.Find("meta[name='description']")
-	content, exists := tag.Attr("content")
-	content += " "
-	if !exists {
-		return nil, errors.New("unable to extract article, no description tag")
-	}
-
-	// get publishing date
-	tag = doc.Find("meta[name='date']")
-	datestr, exists := tag.Attr("content")
-	if !exists {
-		return nil, errors.New("unable to extract article, no date tag")
-	}
-
-	date, err := time.Parse("2006-01-02T15:04:05-07:00", datestr)
-	if err != nil {
-		return nil, err
-	}
-
-	// get content
-	tag = doc.Find("main[id='Inhalt'] div > p")
-
-	tag.Each(func(index int, p *goquery.Selection) {
-		content += " " + p.Text()
-	})
-
-	// clean up content string
-	content = string(c.pattern_whitespace.ReplaceAll([]byte(content), []byte(" ")))
-	content = strings.ReplaceAll(content, "»", "\"")
-	content = strings.ReplaceAll(content, "«", "\"")
-
-	// create new article
-	return &model.Article{
-		SourceUrl:   res.Url,
-		PublishDate: date,
-		FetchDate:   time.Now(),
-		Title:       title,
-		Content:     content,
-	}, nil
-}