move from file storage to sqlite3

cmd/crawler/main.go

@@ -2,13 +2,16 @@ package main
 
 import (
 	"crowsnest/internal/model"
-	"crowsnest/internal/data"
+	"crowsnest/internal/model/sqlite"
+	"database/sql"
 	"fmt"
+	"log"
 	"regexp"
 	"strings"
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	_ "github.com/mattn/go-sqlite3"
 )
 
 func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
@@ -21,7 +24,6 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 	// create entry if not behind paywall
 	paywall_false_pattern := regexp.MustCompile("\"paywall\":{\"attributes\":{\"is_active\":false")
 	c.OnResponse(func(r *colly.Response) {
-
 		if paywall_false_pattern.Match(r.Body) {
 			url := r.Request.URL.String()
 			(*results)[url] = &model.Article{
@@ -86,7 +88,6 @@ func spiegelCollector(results *map[string]*model.Article) *colly.Collector {
 
 	// cascade
 	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
-        fmt.Println(e.Attr("href"))
 		e.Request.Visit(e.Attr("href"))
 	})
 
@@ -99,14 +100,15 @@ func main() {
 
 	c.Visit("https://www.spiegel.de/")
 
-    // data store
-    fds, _ := data.NewFileDatastore("./persistence/spiegel.json")
-    repo, _ := data.NewDefaultRepository[*model.Article](fds, "article")
+    db, err := sql.Open("sqlite3", "./persistence/app.db")
+    if err != nil { log.Fatal(err) }
+
+    db_articles := &sqlite.ArticleModel{ DB: db }
 
     counter := 0
 	for _, val := range res {
         counter++
-        repo.Create(val)
+        db_articles.Insert(val)
 	}
     fmt.Println(counter)
 }