eliaskohout/crowsnest
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

saving 25ms-50ms + move summary column

Browse Source
This commit is contained in:
Elias Kohout
2025-01-20 11:31:32 +01:00
parent 47299d6ef3
commit 0594fb8aeb
6 changed files with 92 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/assets/migrations/20250120084318_mv_ai_summary.sql Normal file
View File

@@ -0,0 +1,19 @@
-- +goose Up
-- +goose StatementBegin
BEGIN;
ALTER TABLE articles DROP COLUMN IF EXISTS aisummary;
ALTER TABLE documents ADD COLUMN summary TEXT DEFAULT '';
COMMIT;
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
BEGIN;
ALTER TABLE articles ADD COLUMN aisummary TEXT DEFAULT '';
ALTER TABLE documents DROP COLUMN IF EXISTS summary;
COMMIT;
-- +goose StatementEnd

2
src/cmd/frontend/main.go
View File

@@ -19,7 +19,7 @@ func main() {
}
// run web crawlers
articles := &database.ArticleModel{DB: db}
articles := &database.ArticleRepository{DB: db}
crawler := crawler.CrawlerFacade{}
crawler.Init()

8
src/internal/app/Index.go
View File

@@ -20,7 +20,7 @@ func (app *App) Index(w http.ResponseWriter, req *http.Request) {
}
// get articles
articles, err := app.articles.All(int(limit), int(offset))
articleVMs, err := app.articles.AllArticleViewModels(int(limit), int(offset))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -34,12 +34,6 @@ func (app *App) Index(w http.ResponseWriter, req *http.Request) {
}
totalCount /= pageSize
// convert to viewmodel
articleVMs := make([]*model.ArticleViewModel, 0, len(articles))
for _, a := range articles {
articleVMs = append(articleVMs, a.ViewModel())
}
// render template
t := template.Must(template.ParseFiles(
"assets/templates/article.html",

4
src/internal/app/app.go
View File

@@ -7,12 +7,12 @@ import (
)
type App struct {
articles *database.ArticleModel
articles *database.ArticleRepository
}
func NewApp(db *sql.DB) *App {
return &App{
articles: &database.ArticleModel{DB: db},
articles: &database.ArticleRepository{DB: db},
}
}

22
src/internal/model/article.go
View File

@@ -5,7 +5,6 @@ import (
"time"
)
// TODO docstring
type Article struct {
Id int
SourceUrl string
@@ -13,7 +12,6 @@ type Article struct {
FetchDate time.Time
Title string
Content string
AiSummary string
}
func (a *Article) Clone() *Article {
@@ -24,11 +22,9 @@ func (a *Article) Clone() *Article {
FetchDate: a.FetchDate,
Title: a.Title,
Content: a.Content,
AiSummary: a.AiSummary,
}
}
// TODO docstring
type ArticleViewModel struct {
Id int
Title string
@@ -49,13 +45,11 @@ type ArticlePageViewModel struct {
// TODO docstring
func (a *Article) ViewModel() *ArticleViewModel {
summary := a.AiSummary
if summary == "" {
if len(a.Content) > 200 {
summary = a.Content[:200]
} else {
summary = a.Content
}
var summary string
if len(a.Content) > 200 {
summary = a.Content[:200]
} else {
summary = a.Content
}
short_url := ""
@@ -70,15 +64,11 @@ func (a *Article) ViewModel() *ArticleViewModel {
PublishDate: a.PublishDate.Local().Format("02.01.2006"),
ShortSource: short_url,
Summary: summary,
AiSummarized: a.AiSummary != "",
}
}
func (a *Article) PageViewModel() *ArticlePageViewModel {
summary := a.AiSummary
if summary == "" {
summary = "N/A"
}
summary := "N/A"
short_url := ""
parsedURL, err := url.Parse(a.SourceUrl)

76
src/internal/model/database/articles.go → src/internal/model/database/articleRepository.go
View File

@@ -3,18 +3,19 @@ package database
import (
"crowsnest/internal/model"
"database/sql"
"net/url"
"strings"
)
type ArticleModel struct {
type ArticleRepository struct {
DB *sql.DB
}
// Gets all the article objects from the database. This may throw an error if
// the connection to the database fails.
func (m *ArticleModel) All(limit int, offset int) ([]model.Article, error) {
func (m *ArticleRepository) All(limit int, offset int) ([]model.Article, error) {
stmt := `
SELECT id, title, sourceUrl, content, publishDate, fetchDate, aisummary
SELECT id, title, sourceUrl, content, publishDate, fetchDate
FROM articles
ORDER BY publishDate DESC
LIMIT $1 OFFSET $2
@@ -27,7 +28,7 @@ func (m *ArticleModel) All(limit int, offset int) ([]model.Article, error) {
articles := []model.Article{}
for rows.Next() {
a := model.Article{}
err := rows.Scan(&a.Id, &a.Title, &a.SourceUrl, &a.Content, &a.PublishDate, &a.FetchDate, &a.AiSummary)
err := rows.Scan(&a.Id, &a.Title, &a.SourceUrl, &a.Content, &a.PublishDate, &a.FetchDate)
if err != nil {
return nil, err
}
@@ -42,9 +43,58 @@ func (m *ArticleModel) All(limit int, offset int) ([]model.Article, error) {
return articles, nil
}
func (m *ArticleRepository) AllArticleViewModels(limit int, offset int) ([]*model.ArticleViewModel, error) {
stmt := `
SELECT a.id, a.title, a.sourceUrl, a.publishDate, d.summary
FROM articles a JOIN documents d ON a.document_id = d.id
ORDER BY a.publishDate DESC
LIMIT $1 OFFSET $2
`
rows, err := m.DB.Query(stmt, limit, offset)
if err != nil {
return nil, err
}
articleVMs := []*model.ArticleViewModel{}
var sourceUrl string
for rows.Next() {
a := model.ArticleViewModel{}
err := rows.Scan(&a.Id, &a.Title, &sourceUrl, &a.PublishDate, &a.Summary)
if err != nil {
return nil, err
}
// summary
if a.Summary == "" {
a.Summary = "N/A"
}
// short url
parsedURL, err := url.Parse(sourceUrl)
if err == nil {
a.ShortSource = parsedURL.Hostname()
} else {
a.ShortSource = ""
}
// ai summary always false
a.AiSummarized = false
articleVMs = append(articleVMs, &a)
}
if err = rows.Err(); err != nil {
return nil, err
}
return articleVMs, nil
}
// Counts all articles in the database. This may throw an error if the
// connection to the database fails.
func (m *ArticleModel) CountAll() (uint, error) {
func (m *ArticleRepository) CountAll() (uint, error) {
stmt := `SELECT count(id) FROM articles `
rows := m.DB.QueryRow(stmt)
@@ -60,7 +110,7 @@ func (m *ArticleModel) CountAll() (uint, error) {
// Will use the full-text search features of the underlying database to search
// articles for a given search query. This may fail if the connection to the
// database fails.
func (m *ArticleModel) Search(query string) ([]model.Article, error) {
func (m *ArticleRepository) Search(query string) ([]model.Article, error) {
stmt := `
SELECT a.id, a.title, a.sourceurl, a.content, a.publishdate, a.fetchDate
FROM articles a JOIN documents d ON a.document_id = d.id
@@ -94,7 +144,7 @@ func (m *ArticleModel) Search(query string) ([]model.Article, error) {
// Will return an article given an id. This may fail if the connection to the
// database fails or there is no aritcle with the given id.
func (m *ArticleModel) ById(id int) (*model.Article, error) {
func (m *ArticleRepository) ById(id int) (*model.Article, error) {
stmt := `
SELECT a.id, a.title, a.sourceurl, a.content, a.publishdate, a.fetchDate
FROM articles a
@@ -114,21 +164,21 @@ func (m *ArticleModel) ById(id int) (*model.Article, error) {
// Inserts a new article into the database. The id attribute of the given
// article will be ignored. May throw an error if the execution of the database
// query fails.
func (m *ArticleModel) Insert(a *model.Article) error {
func (m *ArticleRepository) Insert(a *model.Article) error {
// insert article
stmt := `INSERT INTO articles (title, sourceUrl, content, publishDate, fetchDate, aisummary)
stmt := `INSERT INTO articles (title, sourceUrl, content, publishDate, fetchDate)
VALUES ($1, $2, $3, $4, $5, $6)
`
_, err := m.DB.Exec(stmt, a.Title, a.SourceUrl, a.Content, a.PublishDate, a.FetchDate, a.AiSummary)
_, err := m.DB.Exec(stmt, a.Title, a.SourceUrl, a.Content, a.PublishDate, a.FetchDate)
return err
}
// TODO docstring
func (m *ArticleModel) Update(a *model.Article) error {
func (m *ArticleRepository) Update(a *model.Article) error {
stmt := `UPDATE articles
SET title = $1, sourceUrl = $2, content = $4, publishDate = $5, fetchDate = $6, aisummary = $7
SET title = $1, sourceUrl = $2, content = $4, publishDate = $5, fetchDate = $6
WHERE id = $8
`
_, err := m.DB.Exec(stmt, a.Title, a.SourceUrl, a.Content, a.PublishDate, a.FetchDate, a.AiSummary, a.Id)
_, err := m.DB.Exec(stmt, a.Title, a.SourceUrl, a.Content, a.PublishDate, a.FetchDate, a.Id)
return err
}