restructuring; setting up basic webserver and templating

cmd/crawler/main.go

@@ -14,8 +14,8 @@ import (
 func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	c := colly.NewCollector(
 		colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
-		colly.CacheDir("./spiegel_cache"),
-		colly.MaxDepth(2),
+		colly.CacheDir("./persistence/spiegel_cache"),
+		colly.MaxDepth(5),
 	)
 
 	// create entry if not behind paywall
@@ -36,8 +36,7 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	// check for article type
 	c.OnHTML("meta[property='og:type']", func(e *colly.HTMLElement) {
 		if e.Attr("content") != "article" {
-			delete((*results), e.Request.URL.String())
-		}
+		} 
 	})
 
 	// add title
@@ -87,6 +86,7 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 
 	// cascade
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
+        fmt.Println(e.Attr("href"))
 		e.Request.Visit(e.Attr("href"))
 	})
 
@@ -100,7 +100,7 @@ func main() {
 	c.Visit("https://www.spiegel.de/")
 
     // data store
-    fds, _ := data.NewFileDatastore("spiegel.json")
+    fds, _ := data.NewFileDatastore("./persistence/spiegel.json")
     repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
 
     counter := 0

cmd/frontend/main.go

@@ -1,41 +1,35 @@
 package main
 
 import (
-    "fmt"
-    "net/http"
-    "crowsnest/internal/model"
-    "crowsnest/internal/data"
+	//"fmt"
+	"crowsnest/internal/data"
+	"crowsnest/internal/model"
+	"fmt"
+	"html/template"
+	"net/http"
 )
 
-func test(w http.ResponseWriter, req *http.Request) {
-    for name, headers := range req.Header {
-        for _, h := range headers {
-            fmt.Fprintf(w, "%v: %v\n", name, h)
-        }
-    }
-}
-
-func PrintAllKeys(ds data.IDatastore) {
-    m, err := ds.GetAllKeys()
-    fmt.Println(m, err)
-}
-
-func main() {
-    http.HandleFunc("/test", test)
-
-    // Serve files from the "static" directory
-	fs := http.FileServer(http.Dir("./static"))
-	http.Handle("/", http.StripPrefix("/", fs))
-
-
-    //a := &model.Article{}
-    
-    fds, _ := data.NewFileDatastore("data.json")
-    fmt.Println(fds.GetAll())
+func index(w http.ResponseWriter, req *http.Request) {
+    fds, _ := data.NewFileDatastore("./persistence/spiegel100.json")
     
     repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
     articles, _ := repo.GetAll()
-    fmt.Println(articles[0])
 
-    //http.ListenAndServe(":8090", nil)
+    t := template.Must(template.ParseFiles("templates/article.html", "templates/layout.html"))
+    _ = t.ExecuteTemplate(w, "base", articles)
+}
+
+func main() {
+    // routes
+    http.HandleFunc("/", index)
+
+    // serve files from the "static" directory
+	fs := http.FileServer(http.Dir("./static"))
+	http.Handle("/static", http.StripPrefix("/", fs))
+
+    t := template.Must(template.ParseFiles("templates/article.html"))
+    fmt.Println(t)
+
+    // start server
+    http.ListenAndServe(":8080", nil)
 }

data.json

@@ -1 +0,0 @@
-{"article:0":"{\"Identifier\":0,\"SourceUrl\":\"\",\"PublishDate\":\"0001-01-01T00:00:00Z\",\"FetchDate\":\"0001-01-01T00:00:00Z\",\"Title\":\"\",\"Content\":\"\"}"}

internal/data/FileDatastore.go

@@ -12,6 +12,7 @@ import (
 // a simple text file to the data as json.
 type FileDatastore struct {
     path string
+    data map[string]string
 }
 
 // Creates a new FileDatastore object, creating the storage file in the
@@ -30,6 +31,10 @@ func NewFileDatastore(path string) (*FileDatastore, error) {
 // an error, if the file does not exit or the file content can not be
 // converted.
 func (fds *FileDatastore) readMapObj() (map[string]string, error) {
+    if fds.data != nil {
+        return fds.data, nil
+    }
+
     dat, err := os.ReadFile(fds.path)
     if err != nil { return nil, err }
 
@@ -50,6 +55,8 @@ func (fds *FileDatastore) writeMapObj(m map[string]string) error {
     encoder := json.NewEncoder(file)
     if err := encoder.Encode(m); err != nil { return err }
 
+    fds.data = m
+
     return nil
 }
 

templates/article.html

@@ -0,0 +1,14 @@
+{{ define "content" }}
+
+{{ range . }}
+<div class="card" style="">
+  <div class="card-body">
+      <h5 class="card-title">{{ .Title }}</h5>
+      <h6 class="card-subtitle mb-2 text-body-secondary">{{ .Author }}</h6>
+      <p class="card-text">{{ .Content }}</p>
+    <a href="{{ .SourceUrl }}" class="card-link">Link</a>
+  </div>
+</div>
+{{ end }}
+
+{{ end }}

templates/layout.html

@@ -0,0 +1,13 @@
+{{ define "base" }}
+<!DOCTYPE html>
+<html lang="de">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
+    <body>
+    
+    {{ template "content" . }}
+
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script>
+    </body>
+</html>
+{{ end }}