move collection and extraction of articles into their own file; add custom response cache

internal/model/response.go

@@ -0,0 +1,11 @@
+package model
+
+import "time"
+
+// A simple cache for requests.
+type Response struct {
+	Url       string
+	Content   string
+    Processed bool
+	FetchDate time.Time
+}

internal/model/sqlite/articles.go

@@ -85,3 +85,25 @@ func (m *ArticleModel) Insert(a *model.Article) error {
     _, err = m.DB.Exec(stmt, lastId, a.Title, a.Author, a.Content)
     return err
 }
+
+// TODO docstring
+func (m *ArticleModel) Update(a *model.Article) error {
+    // begin transaction 
+    _, err := m.DB.Begin()
+    if err != nil { return err }
+
+    // insert article
+    stmt := `UPDATE articles 
+             SET title = ?, sourceUrl = ?, author = ?, content = ?, publishDate = ?, fetchDate = ?
+             WHERE id = ?
+    `
+    _, err = m.DB.Exec(stmt, a.Title, a.SourceUrl, a.Author, a.Content, a.PublishDate, a.FetchDate, a.Identifier)
+    if err != nil { return err }
+
+    // insert into fts_articles for full-text search
+    stmt = `INSERT INTO fts_articles (id, content)
+             VALUES (?, ? || '\n' || ? || '\n' || ?)
+    `
+    _, err = m.DB.Exec(stmt, a.Identifier, a.Title, a.Author, a.Content)
+    return err
+}

internal/model/sqlite/responeses.go

@@ -0,0 +1,91 @@
+package sqlite
+
+import (
+	"crowsnest/internal/model"
+	"database/sql"
+)
+
+type ResponseModel struct {
+    DB *sql.DB
+}
+
+
+// TODO docstring
+func (m *ResponseModel) All() ([]model.Response, error) {
+    stmt := `
+        SELECT url, content, fetchDate, processed
+        FROM responses 
+    `
+    rows, err := m.DB.Query(stmt)
+    if err != nil { return nil, err }
+
+    responses := []model.Response{}
+    for rows.Next() {
+        r := model.Response{}
+        err := rows.Scan(&r.Url, &r.Content, &r.FetchDate, &r.Processed)
+        if err != nil { return nil, err }
+
+        responses = append(responses, r)
+    }
+    
+    if err = rows.Err(); err != nil { return nil,  err }
+
+    return responses, nil
+}
+
+// TODO docstring
+func (m *ResponseModel) UnprocessedUrls() ([]string, error) {
+    stmt := `
+        SELECT url 
+        FROM responses 
+        WHERE NOT processed 
+    `
+    rows, err := m.DB.Query(stmt)
+    if err != nil { return nil, err }
+
+    urls := make([]string, 0)
+    for rows.Next() {
+        r := ""
+        err := rows.Scan(&r)
+        if err != nil { return nil, err }
+
+        urls = append(urls, r)
+    }
+    
+    if err = rows.Err(); err != nil { return nil,  err }
+
+    return urls, nil
+}
+
+// TODO docstring
+func (m *ResponseModel) GetByUrl(url string) (model.Response, error) {
+    stmt := `
+        SELECT url, content, fetchDate, processed
+        FROM responses 
+        WHERE url = ?
+    `
+
+    res := model.Response{}
+    row := m.DB.QueryRow(stmt, url)
+    err := row.Scan(&res.Url, &res.Content, &res.FetchDate, &res.Processed)
+
+    return res, err
+}
+
+// TODO docstring
+func (m *ResponseModel) Insert(url string, content string) error {
+    // insert response
+    stmt := `INSERT INTO responses (url, content) VALUES (?, ?)`
+    _, err := m.DB.Exec(stmt, url, content)
+
+    return err
+}
+
+// TODO docstring
+func (m *ResponseModel) Processed(url string) error {
+    // insert response
+    stmt := `UPDATE responses SET processed = true WHERE url = ?`
+    _, err := m.DB.Exec(stmt, url)
+
+    return err
+}