44 lines
734 B
Go
44 lines
734 B
Go
package main
|
|
|
|
import (
|
|
"crowsnest/cmd/crawler/extractors"
|
|
"crowsnest/internal/model/database"
|
|
"database/sql"
|
|
"log"
|
|
"os"
|
|
"time"
|
|
|
|
_ "github.com/lib/pq"
|
|
)
|
|
|
|
func main() {
|
|
// collect environement variables
|
|
databaseURL := os.Getenv("DB_URL")
|
|
|
|
// connect to database
|
|
db, err := sql.Open("postgres", databaseURL)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
|
|
// collect websites
|
|
//coll := collectors.Collector{
|
|
// Responses: &database.ResponseModel{DB: db},
|
|
//}
|
|
|
|
//coll.Spiegel()
|
|
//coll.Zeit()
|
|
|
|
// extract articles from websites
|
|
extr := extractors.Extractor{
|
|
Responses: &database.ResponseModel{DB: db},
|
|
Articles: &database.ArticleModel{DB: db},
|
|
}
|
|
|
|
for {
|
|
extr.Spiegel()
|
|
time.Sleep(5 * time.Second)
|
|
}
|
|
}
|