restructuring; setting up basic webserver and templating

cmd/crawler/main.go

@@ -14,8 +14,8 @@ import (
 func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	c := colly.NewCollector(
 		colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
-		colly.CacheDir("./spiegel_cache"),
-		colly.MaxDepth(2),
+		colly.CacheDir("./persistence/spiegel_cache"),
+		colly.MaxDepth(5),
 	)
 
 	// create entry if not behind paywall
@@ -36,8 +36,7 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	// check for article type
 	c.OnHTML("meta[property='og:type']", func(e *colly.HTMLElement) {
 		if e.Attr("content") != "article" {
-			delete((*results), e.Request.URL.String())
-		}
+		} 
 	})
 
 	// add title
@@ -87,6 +86,7 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 
 	// cascade
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
+        fmt.Println(e.Attr("href"))
 		e.Request.Visit(e.Attr("href"))
 	})
 
@@ -100,7 +100,7 @@ func main() {
 	c.Visit("https://www.spiegel.de/")
 
     // data store
-    fds, _ := data.NewFileDatastore("spiegel.json")
+    fds, _ := data.NewFileDatastore("./persistence/spiegel.json")
     repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
 
     counter := 0

cmd/frontend/main.go

@@ -1,41 +1,35 @@
 package main
 
 import (
-    "fmt"
-    "net/http"
-    "crowsnest/internal/model"
-    "crowsnest/internal/data"
+	//"fmt"
+	"crowsnest/internal/data"
+	"crowsnest/internal/model"
+	"fmt"
+	"html/template"
+	"net/http"
 )
 
-func test(w http.ResponseWriter, req *http.Request) {
-    for name, headers := range req.Header {
-        for _, h := range headers {
-            fmt.Fprintf(w, "%v: %v\n", name, h)
-        }
-    }
-}
-
-func PrintAllKeys(ds data.IDatastore) {
-    m, err := ds.GetAllKeys()
-    fmt.Println(m, err)
-}
-
-func main() {
-    http.HandleFunc("/test", test)
-
-    // Serve files from the "static" directory
-	fs := http.FileServer(http.Dir("./static"))
-	http.Handle("/", http.StripPrefix("/", fs))
-
-
-    //a := &model.Article{}
-    
-    fds, _ := data.NewFileDatastore("data.json")
-    fmt.Println(fds.GetAll())
+func index(w http.ResponseWriter, req *http.Request) {
+    fds, _ := data.NewFileDatastore("./persistence/spiegel100.json")
     
     repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
     articles, _ := repo.GetAll()
-    fmt.Println(articles[0])
 
-    //http.ListenAndServe(":8090", nil)
+    t := template.Must(template.ParseFiles("templates/article.html", "templates/layout.html"))
+    _ = t.ExecuteTemplate(w, "base", articles)
+}
+
+func main() {
+    // routes
+    http.HandleFunc("/", index)
+
+    // serve files from the "static" directory
+	fs := http.FileServer(http.Dir("./static"))
+	http.Handle("/static", http.StripPrefix("/", fs))
+
+    t := template.Must(template.ParseFiles("templates/article.html"))
+    fmt.Println(t)
+
+    // start server
+    http.ListenAndServe(":8080", nil)
 }