eliaskohout/crowsnest
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

remove model files for responses

Browse Source
This commit is contained in:
Elias Kohout
2025-01-07 12:32:38 +01:00
parent ce10e1e62b
commit 321ccfe44d
6 changed files with 20 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/crawler/collectors/collector.go
View File

@@ -3,6 +3,5 @@ package collectors
import "crowsnest/internal/model/database"
type Collector struct {
Responses *database.ResponseModel
Articles *database.ArticleModel
Articles *database.ArticleModel
}

4
cmd/crawler/collectors/spiegel.go
View File

@@ -17,7 +17,7 @@ func (c *Collector) CollectSpiegel() {
collycollector := colly.NewCollector(
colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
colly.CacheDir("./persistence/spiegel_cache"),
colly.MaxDepth(5),
colly.MaxDepth(3),
)
// store articles
@@ -42,7 +42,7 @@ func (c *Collector) CollectSpiegel() {
})
// go through archive
startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
startDate := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
currentDate := time.Now()
for date := startDate; date.Before(currentDate) || date.Equal(currentDate); date = date.AddDate(0, 0, 1) {

4
cmd/crawler/collectors/zeit.go
View File

@@ -19,7 +19,7 @@ func (c *Collector) CollectZeit() {
collycollector := colly.NewCollector(
colly.AllowedDomains("www.zeit.de", "zeit.de"),
colly.CacheDir("./persistence/zeit_cache"),
colly.MaxDepth(5),
colly.MaxDepth(3),
)
// store articles
@@ -44,7 +44,7 @@ func (c *Collector) CollectZeit() {
})
// go through archive
startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
startDate := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
//startDate := time.Date(1946, time.January, 1, 0, 0, 0, 0, time.UTC)
currentDate := time.Now()