44 lines
641 B
Go
44 lines
641 B
Go
package main
|
|
|
|
import (
|
|
"crowsnest/cmd/crawler/collectors"
|
|
"crowsnest/internal/model/database"
|
|
"database/sql"
|
|
"log"
|
|
"os"
|
|
"sync"
|
|
|
|
_ "github.com/lib/pq"
|
|
)
|
|
|
|
func main() {
|
|
// collect environement variables
|
|
databaseURL := os.Getenv("DB_URL")
|
|
|
|
// connect to database
|
|
db, err := sql.Open("postgres", databaseURL)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer db.Close()
|
|
|
|
// collect websites
|
|
coll := collectors.Collector{
|
|
Articles: &database.ArticleModel{DB: db},
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
coll.CollectSpiegel()
|
|
}()
|
|
go func() {
|
|
defer wg.Done()
|
|
go coll.CollectZeit()
|
|
}()
|
|
|
|
wg.Wait()
|
|
}
|