crowsnest/cmd/crawler/main.go

package main

import (
	"crowsnest/cmd/crawler/extractors"
	"crowsnest/internal/model/database"
	"database/sql"
	"log"
	"os"
	"time"

	_ "github.com/lib/pq"
)

func main() {
	// collect environement variables
	databaseURL := os.Getenv("DB_URL")

	// connect to database
	db, err := sql.Open("postgres", databaseURL)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// collect websites
	//coll := collectors.Collector{
	//	Responses: &database.ResponseModel{DB: db},
	//}

	//coll.Spiegel()
	//coll.Zeit()

	// extract articles from websites
	extr := extractors.Extractor{
		Responses: &database.ResponseModel{DB: db},
		Articles:  &database.ArticleModel{DB: db},
	}

	for {
		extr.Spiegel()
		time.Sleep(5 * time.Second)
	}
}