restructuring; setting up basic webserver and templating

cmd/crawler/main.go

@@ -14,8 +14,8 @@ import (
 func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	c := colly.NewCollector(
 		colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
-		colly.CacheDir("./spiegel_cache"),
-		colly.MaxDepth(2),
+		colly.CacheDir("./persistence/spiegel_cache"),
+		colly.MaxDepth(5),
 	)
 
 	// create entry if not behind paywall
@@ -36,8 +36,7 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	// check for article type
 	c.OnHTML("meta[property='og:type']", func(e *colly.HTMLElement) {
 		if e.Attr("content") != "article" {
-			delete((*results), e.Request.URL.String())
-		}
+		} 
 	})
 
 	// add title
@@ -87,6 +86,7 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 
 	// cascade
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
+        fmt.Println(e.Attr("href"))
 		e.Request.Visit(e.Attr("href"))
 	})
 
@@ -100,7 +100,7 @@ func main() {
 	c.Visit("https://www.spiegel.de/")
 
     // data store
-    fds, _ := data.NewFileDatastore("spiegel.json")
+    fds, _ := data.NewFileDatastore("./persistence/spiegel.json")
     repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
 
     counter := 0