remove model files for responses

cmd/crawler/collectors/collector.go

@@ -3,6 +3,5 @@ package collectors
 import "crowsnest/internal/model/database"
 
 type Collector struct {
-	Responses *database.ResponseModel
-	Articles  *database.ArticleModel
+	Articles *database.ArticleModel
 }

cmd/crawler/collectors/spiegel.go

@@ -17,7 +17,7 @@ func (c *Collector) CollectSpiegel() {
 	collycollector := colly.NewCollector(
 		colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
 		colly.CacheDir("./persistence/spiegel_cache"),
-		colly.MaxDepth(5),
+		colly.MaxDepth(3),
 	)
 
 	// store articles
@@ -42,7 +42,7 @@ func (c *Collector) CollectSpiegel() {
 	})
 
 	// go through archive
-	startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
+	startDate := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
 	currentDate := time.Now()
 
 	for date := startDate; date.Before(currentDate) || date.Equal(currentDate); date = date.AddDate(0, 0, 1) {

cmd/crawler/collectors/zeit.go

@@ -19,7 +19,7 @@ func (c *Collector) CollectZeit() {
 	collycollector := colly.NewCollector(
 		colly.AllowedDomains("www.zeit.de", "zeit.de"),
 		colly.CacheDir("./persistence/zeit_cache"),
-		colly.MaxDepth(5),
+		colly.MaxDepth(3),
 	)
 
 	// store articles
@@ -44,7 +44,7 @@ func (c *Collector) CollectZeit() {
 	})
 
 	// go through archive
-	startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
+	startDate := time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC)
 	//startDate := time.Date(1946, time.January, 1, 0, 0, 0, 0, time.UTC)
 	currentDate := time.Now()
 

cmd/crawler/main.go

@@ -6,6 +6,7 @@ import (
 	"database/sql"
 	"log"
 	"os"
+	"sync"
 
 	_ "github.com/lib/pq"
 )
@@ -23,10 +24,20 @@ func main() {
 
 	// collect websites
 	coll := collectors.Collector{
-		Responses: &database.ResponseModel{DB: db},
-		Articles:  &database.ArticleModel{DB: db},
+		Articles: &database.ArticleModel{DB: db},
 	}
 
-	coll.SpiegelCollect()
-	//coll.Zeit()
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		coll.CollectSpiegel()
+	}()
+	go func() {
+		defer wg.Done()
+		go coll.CollectZeit()
+	}()
+
+	wg.Wait()
 }