move from file storage to sqlite3

cmd/crawler/main.go

@@ -2,13 +2,16 @@ package main
 
 import (
 	"crowsnest/internal/model"
-	"crowsnest/internal/data"
+	"crowsnest/internal/model/sqlite"
+	"database/sql"
 	"fmt"
+	"log"
 	"regexp"
 	"strings"
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	_ "github.com/mattn/go-sqlite3"
 )
 
 func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
@@ -21,7 +24,6 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	// create entry if not behind paywall
 	paywall_false_pattern := regexp.MustCompile("\"paywall\":{\"attributes\":{\"is_active\":false")
 	c.OnResponse(func(r *colly.Response) {
-
 		if paywall_false_pattern.Match(r.Body) {
 			url := r.Request.URL.String()
 			(*results)[url] = &model.Article{
@@ -86,7 +88,6 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 
 	// cascade
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
-        fmt.Println(e.Attr("href"))
 		e.Request.Visit(e.Attr("href"))
 	})
 
@@ -99,14 +100,15 @@ func main() {
 
 	c.Visit("https://www.spiegel.de/")
 
-    // data store
-    fds, _ := data.NewFileDatastore("./persistence/spiegel.json")
-    repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
+    db, err := sql.Open("sqlite3", "./persistence/app.db")
+    if err != nil { log.Fatal(err) }
+
+    db_articles := &sqlite.ArticleModel{ DB: db }
 
     counter := 0
 	for _, val := range res {
         counter++
-        repo.Create(val)
+        db_articles.Insert(val)
 	}
     fmt.Println(counter)
 }

cmd/frontend/Index.go

@@ -1,27 +1,16 @@
 package main
 
 import (
-	"crowsnest/internal/data"
 	"crowsnest/internal/model"
 	"html/template"
 	"net/http"
 )
 
-// sort criteria
-type articleDateOrder struct {}
-func (ord articleDateOrder) Weight(a *model.Article) int {
-    return int(a.PublishDate.Unix())
-}
-
 // List the latest articles using the base template. 
 func (app *App) Index(w http.ResponseWriter, req *http.Request) {
-    // retrieve from repo
-    fds, err := data.NewFileDatastore("persistence/spiegel100.json")
-    if err != nil { http.Error(w, "Failed to load datastore", http.StatusInternalServerError); return; }
-    repo, err := data.NewDefaultRepository[*model.Article](fds, "article")
-    if err != nil { http.Error(w, "Failed to create repository", http.StatusInternalServerError); return; }
-    articles, err := repo.GetByCriteria(articleDateOrder{})
-    if err != nil { http.Error(w, "Failed to get articles", http.StatusInternalServerError); return; }
+    // get articles
+    articles, err := app.articles.All()
+    if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError); return; }
 
     // truncate
     if len(articles) > 10 {

cmd/frontend/UpSearch.go

@@ -1,58 +1,23 @@
 package main
 
 import (
-	"crowsnest/internal/data"
 	"crowsnest/internal/model"
 	"html/template"
 	"net/http"
-	"regexp"
-	"strings"
 )
 
-// sort criteria
-type articleTermFrequency struct {
-    terms []string
-}
-
-func (ord articleTermFrequency) Weight(a *model.Article) int {
-    score := 0
-    for _, term := range ord.terms {
-        score += strings.Count(a.Content, term)
-    }
-    return score
-}
-
 // Enpoint that returns a list of articles given search terms in the post
 // request of a search form. Uses the content template.
 func (app *App) UpSearch(w http.ResponseWriter, req *http.Request) {
-    // parse the form data
-    err := req.ParseForm()
-    if err != nil { http.Error(w, "Unable to parse form", http.StatusBadRequest); return; }
-
-    // collect search terms
-    p := regexp.MustCompile("\\s+")
-    searchTerms := make([]string, 0)
-    for _, elem := range p.Split(req.FormValue("search"), -1) {
-        elem = strings.TrimSpace(elem)
-        if elem == "" { continue }
-        searchTerms = append(searchTerms, elem)
-    }
-
-    // retrieve from repo
-    fds, _ := data.NewFileDatastore("persistence/spiegel100.json")
-    if err != nil { http.Error(w, "Failed to read datastore", http.StatusInternalServerError); return; }
-
-    repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
-    if err != nil { http.Error(w, "Failed to create repository", http.StatusInternalServerError); return; }
-
-    var articles []*model.Article
-    if len(searchTerms) != 0 {
-        articles, err = repo.GetByCriteria(articleTermFrequency{ terms: searchTerms })
-    } else {
+    searchTerms := req.FormValue("search")
+    if searchTerms == "" {
         app.Index(w, req)
         return
     }
-    if err != nil { http.Error(w, "Failed to get articles from repo", http.StatusInternalServerError); return; }
+    
+    // get articles
+    articles, err := app.articles.Search(searchTerms)
+    if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError); return; }
 
     // convert to viewmodel
     articleVMs := make([]*model.ArticleViewModel, 0, len(articles))

cmd/frontend/main.go

@@ -1,21 +1,32 @@
 package main
 
 import (
+	"crowsnest/internal/model/sqlite"
+	"database/sql"
 	"log"
 	"net/http"
+
+	_ "github.com/mattn/go-sqlite3"
 )
 
 
-type App struct {}
+type App struct {
+    articles *sqlite.ArticleModel
+}
 
 func main() {
-    app := &App{}
+    db, err := sql.Open("sqlite3", "./persistence/app.db")
+    if err != nil { log.Fatal(err) }
+
+    app := &App{
+        articles: &sqlite.ArticleModel{ DB: db },
+    }
 
     server := http.Server{
         Addr:    ":8080",
         Handler: app.routes(),
     }
 
-    server.ListenAndServe()
     log.Println("server started, listening on :8080")
+    server.ListenAndServe()
 }