eliaskohout/crowsnest
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

move from file storage to sqlite3

Browse Source
This commit is contained in:
Elias Kohout
2025-01-03 01:00:06 +01:00
parent fbca771479
commit 98655fd1fb
16 changed files with 150 additions and 389 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
internal/data/DefaultRepository.go
View File

@@ -1,134 +0,0 @@
package data
import (
"encoding/json"
"errors"
"slices"
"strings"
)
// Default implementation of IRepository using an IDatastore.
type DefaultRepository[T IIdentifiable] struct {
ds IDatastore
prefix string
}
// Creates a new DefaultRepository for a generic type T given a IDatastore and
// a prefix. The type T must implement the IIdentifiable interface. The prefix
// will be used for the key for every entry using the created repository. The
// prefix should be not yet be in use in the given datastore and not contain a
// ':' character.
func NewDefaultRepository[T IIdentifiable](ds IDatastore, prefix string) (*DefaultRepository[T], error) {
if strings.Contains(prefix, ":") { return nil, errors.New("prefix should not contain ':'") }
return &DefaultRepository[T]{ ds: ds, prefix: prefix }, nil
}
// Creates a new entry in the repository with the given object t. Throws an
// error if there already exists an entry with the same id, the json encoding
// fails or the connection to the IDatastore fails.
func (repo *DefaultRepository[T]) Create(t T) error {
key := repo.prefix + ":" + t.Id()
exists, err := repo.ds.KeyExists(key)
if err != nil { return err }
if exists { return errors.New("entry with given id already exists") }
d, err := json.Marshal(t)
if err != nil { return err }
err = repo.ds.Set(key, string(d))
if err != nil { return err }
return nil
}
// Updates the entry with the same id as t in the repository with the values of
// t. Trows an error if the json encoding fails or the connection to the
// IDatastore fails.
func (repo *DefaultRepository[T]) Update(t T) error {
key := repo.prefix + ":" + t.Id()
exists, err := repo.ds.KeyExists(key)
if err != nil { return err }
if !exists { return errors.New("no entry with given id") }
d, err := json.Marshal(t)
if err != nil { return err }
err = repo.ds.Set(key, string(d))
if err != nil { return err }
return nil
}
// Delete the entry with the same id as t in the repository. Trows an error if
// the connection to the IDatastore fails or the key of t does not exist.
func (repo *DefaultRepository[T]) Delete(t T) error {
key := repo.prefix + ":" + t.Id()
exists, err := repo.ds.KeyExists(key)
if err != nil { return err }
if !exists { return errors.New("no entry with given id") }
err = repo.ds.Delete(key)
if err != nil { return err }
return nil
}
// Get all the objects of type T from the repository as a list. Trows an error
// if the connection to the IDatastore fails.
func (repo *DefaultRepository[T]) GetAll() ([]T, error) {
out := make([]T, 0)
allkeys, err := repo.ds.GetAllKeys()
if err != nil { return nil, err }
for key, _ := range allkeys {
splitkey := strings.Split(key, ":")
if splitkey[0] == repo.prefix {
// retrieve the object
obj, err := repo.GetById(splitkey[1])
if err != nil { return nil, err }
out = append(out, obj)
}
}
return out, nil
}
// Get the objects of type T from the repository that has the given id. Trows an error
// if the connection to the IDatastore or the decoding process fails.
func (repo *DefaultRepository[T]) GetById(id string) (T, error) {
var obj T
key := repo.prefix + ":" + id
value, err := repo.ds.Get(key)
if err != nil { return obj, err }
err = json.Unmarshal([]byte(value), &obj)
if err != nil { return obj, err }
return obj, nil
}
// Returns a slice of all elememts in the repo that have a
// ISearchCriteria.Weight greater than 0 sort by that weight. Throws an error
// when the elememts cannot be retrieved from the repo.
func (repo *DefaultRepository[T]) GetByCriteria(c ISortCriteria[T]) ([]T, error) {
all, err := repo.GetAll()
if err != nil { return nil, err }
filtered := make([]T, 0)
for _, elem := range all {
if c.Weight(elem) > 0 { filtered = append(filtered, elem) }
}
slices.SortFunc(filtered, func(a, b T) int {
wa, wb := c.Weight(a), c.Weight(b)
if wa > wb { return -1 }
return 1
})
return filtered, nil
}

139
internal/data/FileDatastore.go
View File

@@ -1,139 +0,0 @@
package data
import (
"os"
"maps"
"errors"
"encoding/json"
)
// A very simple datastructure, implementing the IDatastore interface. It uses
// a simple text file to the data as json.
type FileDatastore struct {
path string
data map[string]string
}
// Creates a new FileDatastore object, creating the storage file in the
// process.
func NewFileDatastore(path string) (*FileDatastore, error) {
fds := &FileDatastore{ path: path }
if _, err := fds.readMapObj(); err != nil {
if err := fds.writeMapObj(make(map[string]string)); err != nil { return nil, err }
}
return fds, nil
}
// Read the contents of the storage file and convert to a map object. May throw
// an error, if the file does not exit or the file content can not be
// converted.
func (fds *FileDatastore) readMapObj() (map[string]string, error) {
if fds.data != nil {
return fds.data, nil
}
dat, err := os.ReadFile(fds.path)
if err != nil { return nil, err }
var mapobj map[string]string
err = json.Unmarshal(dat, &mapobj)
if err != nil { return nil, err }
return mapobj, nil
}
// Write the map object to the storage file. Will overwrite the content of the
// file. May throw an error, if the file cannot be created or written to.
func (fds *FileDatastore) writeMapObj(m map[string]string) error {
file, err := os.Create(fds.path)
if err != nil { return err }
defer file.Close()
encoder := json.NewEncoder(file)
if err := encoder.Encode(m); err != nil { return err }
fds.data = m
return nil
}
// --- implement IDatastore interface ---
// Sets the key value pair given, overwriting if the key already exists. May
// through an error if the file cannot be opened or the contents cannot be
// decoded correctly.
func (fds *FileDatastore) Set(key string, val string) error {
m, err := fds.readMapObj()
if err != nil { return err }
m[key] = val
err = fds.writeMapObj(m)
if err != nil { return err }
return nil
}
// Check if for the given key a entry does exit. May through an error if the
// file cannot be opened or the contents cannot be decoded correctly.
func (fds *FileDatastore) KeyExists(key string) (bool, error) {
m, err := fds.readMapObj()
if err != nil { return false, err }
_, ok := m[key]
return ok, nil
}
// Gets the value for the given key. May through an error if the key does not
// exit, the file cannot be opened or the contents cannot be decoded
// correctly.
func (fds *FileDatastore) Get(key string) (string, error) {
m, err := fds.readMapObj()
if err != nil { return "", err }
val, ok := m[key]
if !ok { return "", errors.New("key not found") }
return val, nil
}
// Gets all the key value pairs from the file and returns them as a map object.
// May through an error if the file cannot be opened or the contents cannot be
// decoded correctly.
func (fds *FileDatastore) GetAll() (map[string]string, error) {
return fds.readMapObj()
}
// Gets all the key the file and returns them as a map object. May through an
// error if the file cannot be opened or the contents cannot be decoded
// correctly.
func (fds *FileDatastore) GetAllKeys() (map[string]bool, error) {
m, err := fds.readMapObj()
if err != nil { return nil, err }
out := make(map[string]bool)
for key := range maps.Keys(m) {
out[key] = true
}
return out, nil
}
// Deletes the entry with the given key. May through an error if the file
// cannot be opened or the contents cannot be decoded or encoded correctly.
func (fds *FileDatastore) Delete(key string) error {
m, err := fds.readMapObj()
if err != nil { return err }
delete(m, key)
err = fds.writeMapObj(m)
if err != nil { return err }
return nil
}

12
internal/data/IDatastore.go
View File

@@ -1,12 +0,0 @@
package data
// Defines the first layer of abstraction on the interface to a persistent data
// store. This may be a file or database.
type IDatastore interface {
Set(key string, val string) error
KeyExists(key string) (bool, error)
Get(key string) (string, error)
GetAll() (map[string]string, error)
GetAllKeys() (map[string]bool, error)
Delete(key string) error
}

7
internal/data/IIdentifiable.go
View File

@@ -1,7 +0,0 @@
package data
// Defines an Id function that uniquely identifies an object. This may be used
// as a primary key in a database/ datastore.
type IIdentifiable interface {
Id() string // not allowed to contain a ':'
}

13
internal/data/IRepository.go
View File

@@ -1,13 +0,0 @@
package data
// An interface to manage generic structure objects persistently. Should use an
// IDatastore as the interface that actually stores and retrieves the data from
// an external source.
type IRepository[T IIdentifiable] interface {
Create(t T) error
Update(t T) error
Delete(t T) error
GetAll() ([]T, error)
GetById(id string) (T, error)
GetByCriteria(c ISortCriteria[T]) ([]T, error)
}

7
internal/data/ISearchCriteria.go
View File

@@ -1,7 +0,0 @@
package data
// Defines a Weight function that determines a order on type T. As an example
// this may be used to sort article by date or filter for a search term.
type ISortCriteria[T any] interface {
Weight(t T) int
}

13
internal/model/article.go
View File

@@ -2,13 +2,12 @@ package model
import (
"time"
"crypto/sha256"
"encoding/hex"
)
// TODO docstring
type Article struct {
Identifier int
SourceUrl string
PublishDate time.Time
FetchDate time.Time
@@ -42,13 +41,3 @@ func (a *Article) ViewModel() *ArticleViewModel {
Summary: summary,
}
}
// --- implement IIdentifiable interface ---
// Generates a hash based on the source url of the article. Can be used to
// identify the article.
func (article *Article) Id() string {
hash := sha256.Sum256([]byte(article.SourceUrl))
return hex.EncodeToString(hash[:])
}

87
internal/model/sqlite/articles.go Normal file
View File

@@ -0,0 +1,87 @@
package sqlite
import (
"crowsnest/internal/model"
"database/sql"
)
// TODO docstring
type ArticleModel struct {
DB *sql.DB
}
// TODO docstring
func (m *ArticleModel) All() ([]model.Article, error) {
stmt := `
SELECT id, title, sourceUrl, author, content, publishDate, fetchDate
FROM articles
ORDER BY publishDate DESC
`
rows, err := m.DB.Query(stmt)
if err != nil { return nil, err }
articles := []model.Article{}
for rows.Next() {
a := model.Article{}
err := rows.Scan(&a.Identifier, &a.Title, &a.SourceUrl, &a.Author, &a.Content, &a.PublishDate, &a.FetchDate)
if err != nil { return nil, err }
articles = append(articles, a)
}
if err = rows.Err(); err != nil { return nil, err }
return articles, nil
}
// TODO docstring
func (m *ArticleModel) Search(query string) ([]model.Article, error) {
stmt := `
SELECT id, title, sourceUrl, author, content, publishDate, fetchDate
FROM articles JOIN (
SELECT id as id2, rank FROM fts_articles WHERE content MATCH ?
) ON id = id2
ORDER BY rank ASC, publishDate DESC
LIMIT 10
`
rows, err := m.DB.Query(stmt, query)
if err != nil { return nil, err }
articles := []model.Article{}
for rows.Next() {
a := model.Article{}
err := rows.Scan(&a.Identifier, &a.Title, &a.SourceUrl, &a.Author, &a.Content, &a.PublishDate, &a.FetchDate)
if err != nil { return nil, err }
articles = append(articles, a)
}
if err = rows.Err(); err != nil { return nil, err }
return articles, nil
}
// Inserts a new article into the database. The id attribute of the given
// article will be ignored. May throw an error if the execution of the database
// query fails.
func (m *ArticleModel) Insert(a *model.Article) error {
// begin transaction
_, err := m.DB.Begin()
if err != nil { return err }
// insert article
stmt := `INSERT INTO articles (title, sourceUrl, author, content, publishDate, fetchDate)
VALUES (?, ?, ?, ?, ?, ?)
`
result, err := m.DB.Exec(stmt, a.Title, a.SourceUrl, a.Author, a.Content, a.PublishDate, a.FetchDate)
if err != nil { return err }
lastId, err := result.LastInsertId()
if err != nil { return err }
// insert into fts_articles for full-text search
stmt = `INSERT INTO fts_articles (id, content)
VALUES (?, ? || '\n' || ? || '\n' || ?)
`
_, err = m.DB.Exec(stmt, lastId, a.Title, a.Author, a.Content)
return err
}