rename spiegel converter
This commit is contained in:
@@ -67,7 +67,7 @@ func main() {
|
||||
// define app
|
||||
webapp := app.NewApp(db)
|
||||
|
||||
// create middle
|
||||
// create middleware
|
||||
stack := middleware.CreateStack(
|
||||
middleware.Logging,
|
||||
)
|
||||
|
||||
@@ -28,13 +28,15 @@ func (cf *CrawlerFacade) Init() {
|
||||
colly.MaxDepth(1),
|
||||
)
|
||||
sf_feed := sf.Feed()
|
||||
sf_converter := ConverterSpiegel{}
|
||||
sf_converter := SpiegelConverter{}
|
||||
sf_converter.Init()
|
||||
|
||||
go func() {
|
||||
for val := range sf_feed {
|
||||
article, err := sf_converter.Convert(val)
|
||||
if err != nil { continue }
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
cf.spiegelFeedDistributer.Publish(article)
|
||||
}
|
||||
}()
|
||||
@@ -54,7 +56,9 @@ func (cf *CrawlerFacade) Init() {
|
||||
go func() {
|
||||
for val := range zf_feed {
|
||||
article, err := zf_converter.Convert(val)
|
||||
if err != nil { continue }
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
cf.zeitFeedDistributer.Publish(article)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -10,19 +10,19 @@ import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
type ConverterSpiegel struct {
|
||||
type SpiegelConverter struct {
|
||||
pattern_paywall *regexp.Regexp
|
||||
pattern_url *regexp.Regexp
|
||||
pattern_whitespace *regexp.Regexp
|
||||
}
|
||||
|
||||
func (c *ConverterSpiegel) Init() {
|
||||
func (c *SpiegelConverter) Init() {
|
||||
c.pattern_paywall = regexp.MustCompile(`"paywall":{"attributes":{"is_active":true`)
|
||||
c.pattern_url = regexp.MustCompile(`^https://(www\.)?spiegel.de.*`)
|
||||
c.pattern_whitespace = regexp.MustCompile(`\s+`)
|
||||
}
|
||||
|
||||
func (c *ConverterSpiegel) Convert(res *Resource) (*model.Article, error) {
|
||||
func (c *SpiegelConverter) Convert(res *Resource) (*model.Article, error) {
|
||||
// check url url pattern
|
||||
if !c.pattern_url.Match([]byte(res.Url)) {
|
||||
return nil, errors.New("invalid url pattern")
|
||||
|
||||
Reference in New Issue
Block a user