eliaskohout/crowsnest
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

remove model files for responses

Browse Source
This commit is contained in:
Elias Kohout
2025-01-07 12:32:38 +01:00
parent ce10e1e62b
commit 321ccfe44d
6 changed files with 20 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/crawler/collectors/zeit.go
View File

@@ -19,7 +19,7 @@ func (c *Collector) CollectZeit() {
collycollector := colly.NewCollector(
colly.AllowedDomains("www.zeit.de", "zeit.de"),
colly.CacheDir("./persistence/zeit_cache"),
colly.MaxDepth(5),
colly.MaxDepth(3),
)
// store articles
@@ -44,7 +44,7 @@ func (c *Collector) CollectZeit() {
})
// go through archive
startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
startDate := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
//startDate := time.Date(1946, time.January, 1, 0, 0, 0, 0, time.UTC)
currentDate := time.Now()