rename spiegel converter

This commit is contained in:
2025-03-24 13:34:37 +01:00
parent 9687f327fe
commit b7b4e74eff
3 changed files with 48 additions and 44 deletions

View File

@@ -67,7 +67,7 @@ func main() {
// define app
webapp := app.NewApp(db)
// create middle
// create middleware
stack := middleware.CreateStack(
middleware.Logging,
)

View File

@@ -28,13 +28,15 @@ func (cf *CrawlerFacade) Init() {
colly.MaxDepth(1),
)
sf_feed := sf.Feed()
sf_converter := ConverterSpiegel{}
sf_converter := SpiegelConverter{}
sf_converter.Init()
go func() {
for val := range sf_feed {
article, err := sf_converter.Convert(val)
if err != nil { continue }
if err != nil {
continue
}
cf.spiegelFeedDistributer.Publish(article)
}
}()
@@ -54,7 +56,9 @@ func (cf *CrawlerFacade) Init() {
go func() {
for val := range zf_feed {
article, err := zf_converter.Convert(val)
if err != nil { continue }
if err != nil {
continue
}
cf.zeitFeedDistributer.Publish(article)
}
}()

View File

@@ -10,19 +10,19 @@ import (
"github.com/PuerkitoBio/goquery"
)
type ConverterSpiegel struct {
type SpiegelConverter struct {
pattern_paywall *regexp.Regexp
pattern_url *regexp.Regexp
pattern_whitespace *regexp.Regexp
}
func (c *ConverterSpiegel) Init() {
func (c *SpiegelConverter) Init() {
c.pattern_paywall = regexp.MustCompile(`"paywall":{"attributes":{"is_active":true`)
c.pattern_url = regexp.MustCompile(`^https://(www\.)?spiegel.de.*`)
c.pattern_whitespace = regexp.MustCompile(`\s+`)
}
func (c *ConverterSpiegel) Convert(res *Resource) (*model.Article, error) {
func (c *SpiegelConverter) Convert(res *Resource) (*model.Article, error) {
// check url url pattern
if !c.pattern_url.Match([]byte(res.Url)) {
return nil, errors.New("invalid url pattern")