eliaskohout/crowsnest
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

change db from sqlite3 to postgresql

Browse Source
This commit is contained in:
Elias Kohout
2025-01-07 09:32:57 +01:00
parent f719c73b46
commit 9302e982c6
20 changed files with 309 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
cmd/crawler/collectors/spiegel.go
View File

@@ -1,110 +1,59 @@
package collectors
import (
//"crowsnest/internal/model"
//"regexp"
//"time"
//"strings"
"crowsnest/internal/model"
"fmt"
"log"
"strings"
"time"
"github.com/gocolly/colly/v2"
)
func (c *Collector) Spiegel() {
collycollector := colly.NewCollector(
colly.AllowedDomains("www.spiegel.de", "spiegel.de"),
colly.CacheDir("./persistence/spiegel_cache"),
colly.MaxDepth(3),
colly.MaxDepth(2),
)
// cache
collycollector.OnRequest(func(r *colly.Request) {
url := r.URL.String()
exists, err := c.Responses.UrlExists(url)
if err == nil && !exists {
c.Responses.Insert(url, nil)
log.Println("request", url)
} else {
r.Abort()
}
})
collycollector.OnResponse(func(r *colly.Response) {
url := r.Request.URL.String()
c.Responses.Update(&model.Response{Url: url, Content: r.Body, FetchDate: time.Now(), Processed: false})
log.Println("response cached", url)
})
// cascade
collycollector.OnHTML("a[href]", func(e *colly.HTMLElement) {
e.Request.Visit(e.Attr("href"))
url := e.Attr("href")
log.Println("found", url)
if !strings.HasPrefix(url, "http") {
return
}
log.Println("visiting", url)
e.Request.Visit(url)
})
// cache
collycollector.OnScraped(func(r *colly.Response) {
c.Responses.Insert(r.Request.URL.String(), string(r.Body))
})
// go through archive
startDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC)
// go through archive
startDate := time.Date(2025, time.January, 1, 0, 0, 0, 0, time.UTC)
currentDate := time.Now()
for date := startDate; date.Before(currentDate) || date.Equal(currentDate); date = date.AddDate(0, 0, 1) {
urlDate := date.Format("02.01.2006")
url := fmt.Sprintf("https://www.spiegel.de/nachrichtenarchiv/artikel-%s.html", urlDate)
collycollector.Visit(url)
collycollector.Visit(url)
}
//// create entry if not behind paywall
//paywall_false_pattern := regexp.MustCompile("\"paywall\":{\"attributes\":{\"is_active\":false")
//collycollector.OnResponse(func(r *colly.Response) {
// if paywall_false_pattern.Match(r.Body) {
// url := r.Request.URL.String()
// (*results)[url] = &model.Article{
// SourceUrl: url,
// FetchDate: time.Now(),
// Content: "",
// }
// }
//})
//// check for article type
//collycollector.OnHTML("meta[property='og:type']", func(e *colly.HTMLElement) {
// if e.Attr("content") != "article" {
// delete(*results, e.Request.URL.String())
// }
//})
//// add title
//collycollector.OnHTML("meta[property='og:title']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// val.Title = e.Attr("content")
// }
//})
//// prepend description to content of article
//collycollector.OnHTML("meta[name='description']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// val.Content = e.Attr("content") + val.Content
// }
//})
//// add publishing date
//collycollector.OnHTML("meta[name='date']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// t, err := time.Parse("2006-01-02T15:04:05-07:00", e.Attr("content"))
// if err != nil {
// panic(err)
// }
// val.PublishDate = t
// }
//})
//// add author
//collycollector.OnHTML("meta[name='author']", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// val.Author = e.Attr("content")
// }
//})
//// add content
//collycollector.OnHTML("main[id='Inhalt'] div > p", func(e *colly.HTMLElement) {
// if val, ok := (*results)[e.Request.URL.String()]; ok {
// cont := val.Content
// pattern := regexp.MustCompile("\\s+")
// cont = string(pattern.ReplaceAll([]byte(cont), []byte(" ")))
// cont = strings.ReplaceAll(cont, "»", "\"")
// cont = strings.ReplaceAll(cont, "«", "\"")
// val.Content = cont + " " + e.Text
// }
//})
}

2
cmd/crawler/collectors/zeit.go
View File

@@ -23,7 +23,7 @@ func (c *Collector) Zeit() {
// cache
collycollector.OnScraped(func(r *colly.Response) {
c.Responses.Insert(r.Request.URL.String(), string(r.Body))
c.Responses.Insert(r.Request.URL.String(), r.Body)
})
// go through archive

191
cmd/crawler/extractors/spiegel.go
View File

@@ -10,104 +10,125 @@ import (
"github.com/PuerkitoBio/goquery"
)
func (extractor *Extractor) Spiegel() error {
// get urls to process
urls, err := extractor.Responses.UnprocessedUrls()
if err != nil { return err }
// get urls to process
urls, err := extractor.Responses.UnprocessedUrls()
if err != nil {
return err
}
paywall_false_pattern := regexp.MustCompile("\"paywall\":{\"attributes\":{\"is_active\":false")
url_pattern := regexp.MustCompile("^https://(www\\.)?spiegel.de.*")
whitespace := regexp.MustCompile("\\s+")
url_pattern := regexp.MustCompile("^https://(www\\.)?spiegel.de.*")
whitespace := regexp.MustCompile("\\s+")
var exists bool
var pagetype, title, content, datestr, author string
var tag *goquery.Selection
var date time.Time
var exists bool
var pagetype, title, content, datestr, author string
var tag *goquery.Selection
var date time.Time
for _, url := range urls {
// check url url pattern
if !url_pattern.Match([]byte(url)) { continue }
for _, url := range urls {
// check url url pattern
if !url_pattern.Match([]byte(url)) {
continue
}
// get response
res, err := extractor.Responses.GetByUrl(url)
if err != nil {
log.Println("failed to process url", url, "with", err)
continue
}
// check for paywall
if !paywall_false_pattern.Match([]byte(res.Content)) {
extractor.Responses.Processed(url)
continue
}
// construct goquery doc
doc, err := goquery.NewDocumentFromReader(strings.NewReader(res.Content))
if err != nil {
log.Println("failed to process url", url, "with", err)
continue
}
// check for article type
tag = doc.Find("meta[property='og:type']")
pagetype, exists = tag.Attr("content")
if !exists || pagetype != "article" { extractor.Responses.Processed(url); continue; }
// get response
res, err := extractor.Responses.GetByUrl(url)
if err != nil {
log.Println("failed to process url", url, "with", err)
continue
}
// get title
tag = doc.Find("meta[property='og:title']")
title, exists = tag.Attr("content")
if !exists { extractor.Responses.Processed(url); continue; }
// check for paywall
if !paywall_false_pattern.Match([]byte(res.Content)) {
extractor.Responses.Processed(url)
continue
}
// prepend description to content of article
tag = doc.Find("meta[name='description']")
content, exists = tag.Attr("content")
content += " "
if !exists { extractor.Responses.Processed(url); continue; }
// construct goquery doc
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(res.Content)))
if err != nil {
log.Println("failed to process url", url, "with", err)
continue
}
// get publishing date
tag = doc.Find("meta[name='date']")
datestr, exists = tag.Attr("content")
if !exists { extractor.Responses.Processed(url); continue; }
// check for article type
tag = doc.Find("meta[property='og:type']")
pagetype, exists = tag.Attr("content")
if !exists || pagetype != "article" {
extractor.Responses.Processed(url)
continue
}
// get title
tag = doc.Find("meta[property='og:title']")
title, exists = tag.Attr("content")
if !exists {
extractor.Responses.Processed(url)
continue
}
// prepend description to content of article
tag = doc.Find("meta[name='description']")
content, exists = tag.Attr("content")
content += " "
if !exists {
extractor.Responses.Processed(url)
continue
}
// get publishing date
tag = doc.Find("meta[name='date']")
datestr, exists = tag.Attr("content")
if !exists {
extractor.Responses.Processed(url)
continue
}
date, err = time.Parse("2006-01-02T15:04:05-07:00", datestr)
if err != nil { extractor.Responses.Processed(url); continue; }
if err != nil {
extractor.Responses.Processed(url)
continue
}
// get author
tag = doc.Find("meta[name='author']")
author, exists = tag.Attr("content")
if !exists { extractor.Responses.Processed(url); continue; }
// get author
tag = doc.Find("meta[name='author']")
author, exists = tag.Attr("content")
if !exists {
extractor.Responses.Processed(url)
continue
}
// get content
tag = doc.Find("main[id='Inhalt'] div > p")
// get content
tag = doc.Find("main[id='Inhalt'] div > p")
tag.Each(func(index int, p *goquery.Selection) {
content += " " + p.Text()
})
// clean up content string
content = string(whitespace.ReplaceAll([]byte(content), []byte(" ")))
content = strings.ReplaceAll(content, "»", "\"")
content = strings.ReplaceAll(content, "«", "\"")
// insert new article
article := model.Article{
SourceUrl: url,
PublishDate: date,
FetchDate: res.FetchDate,
Title: title,
Content: content,
Author: author,
}
err = extractor.Articles.Insert(&article)
if err != nil {
log.Println("failed to insert", article)
} else {
extractor.Responses.Processed(url)
log.Println("found article at", url)
}
}
tag.Each(func(index int, p *goquery.Selection) {
content += " " + p.Text()
})
return nil
// clean up content string
content = string(whitespace.ReplaceAll([]byte(content), []byte(" ")))
content = strings.ReplaceAll(content, "»", "\"")
content = strings.ReplaceAll(content, "«", "\"")
// insert new article
article := model.Article{
SourceUrl: url,
PublishDate: date,
FetchDate: res.FetchDate,
Title: title,
Content: content,
Author: author,
}
err = extractor.Articles.Insert(&article)
if err != nil {
log.Println("failed to insert", article)
} else {
extractor.Responses.Processed(url)
log.Println("found article at", url)
}
}
return nil
}

24
cmd/crawler/main.go
View File

@@ -1,26 +1,31 @@
package main
import (
"crowsnest/cmd/crawler/collectors"
"crowsnest/cmd/crawler/extractors"
"crowsnest/internal/model/database"
"database/sql"
"log"
"os"
"time"
_ "github.com/mattn/go-sqlite3"
_ "github.com/lib/pq"
)
func main() {
// open database
db, err := sql.Open("sqlite3", "./persistence/app.db")
// collect environement variables
databaseURL := os.Getenv("DB_URL")
// connect to database
db, err := sql.Open("postgres", databaseURL)
if err != nil {
log.Fatal(err)
}
defer db.Close()
// collect websites
_ = collectors.Collector{
Responses: &database.ResponseModel{DB: db},
}
//coll := collectors.Collector{
// Responses: &database.ResponseModel{DB: db},
//}
//coll.Spiegel()
//coll.Zeit()
@@ -31,5 +36,8 @@ func main() {
Articles: &database.ArticleModel{DB: db},
}
extr.Spiegel()
for {
extr.Spiegel()
time.Sleep(5 * time.Second)
}
}

51
cmd/frontend/UpSearch.go
View File

@@ -9,29 +9,32 @@ import (
// Enpoint that returns a list of articles given search terms in the post
// request of a search form. Uses the content template.
func (app *App) UpSearch(w http.ResponseWriter, req *http.Request) {
// construct search query
searchTerms := req.FormValue("search")
if searchTerms == "" {
app.Index(w, req)
return
}
// get articles
articles, err := app.articles.Search(searchTerms)
if err != nil {
// treat as no result
//http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// construct search query
searchTerms := req.FormValue("search")
if searchTerms == "" {
app.Index(w, req)
return
}
// convert to viewmodel
articleVMs := make([]*model.ArticleViewModel, 0, len(articles))
for _, a := range articles {
articleVMs = append(articleVMs, a.ViewModel())
}
// render template
t := template.Must(template.ParseFiles("assets/templates/article.html"))
err = t.ExecuteTemplate(w, "content", articleVMs)
if err != nil { http.Error(w, "Failed to render template", http.StatusInternalServerError); return; }
// get articles
articles, err := app.articles.Search(searchTerms)
if err != nil {
// treat as no result
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// convert to viewmodel
articleVMs := make([]*model.ArticleViewModel, 0, len(articles))
for _, a := range articles {
articleVMs = append(articleVMs, a.ViewModel())
}
// render template
t := template.Must(template.ParseFiles("assets/templates/article.html"))
err = t.ExecuteTemplate(w, "content", articleVMs)
if err != nil {
http.Error(w, "Failed to render template", http.StatusInternalServerError)
return
}
}

19
cmd/frontend/main.go
View File

@@ -3,13 +3,11 @@ package main
import (
"crowsnest/internal/model/database"
"database/sql"
"errors"
"log"
"net/http"
"os"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
)
type App struct {
@@ -19,24 +17,17 @@ type App struct {
func main() {
// collect environement variables
databaseURL := os.Getenv("DB_URL")
dbDriver := os.Getenv("DB_DRIVER")
// connect to database
var db *sql.DB
var err error
switch {
case dbDriver == "sqlite3":
db, err = sql.Open("sqlite3", databaseURL)
if err != nil {
log.Fatal(err)
}
default:
log.Fatal(errors.New("given DB_DRIVER is not supported"))
db, err := sql.Open("postgres", databaseURL)
if err != nil {
log.Fatal(err)
}
defer db.Close()
// define app
app := &App{
articles: &database.ArticleModel{DB: db, DbDriver: dbDriver},
articles: &database.ArticleModel{DB: db},
}
// start web server