crowsnest/golang/cmd/crawler/main.go

package main

import (
	"crowsnest/cmd/crawler/collectors"
	"crowsnest/internal/model/database"
	"database/sql"
	"log"
	"os"
	"sync"

	_ "github.com/lib/pq"
)

func main() {
	// collect environement variables
	databaseURL := os.Getenv("DB_URL")

	// connect to database
	db, err := sql.Open("postgres", databaseURL)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// collect websites
	coll := collectors.Collector{
		Articles: &database.ArticleModel{DB: db},
	}

	var wg sync.WaitGroup
	wg.Add(2)

	go func() {
		defer wg.Done()
		coll.CollectSpiegel()
	}()
	go func() {
		defer wg.Done()
		go coll.CollectZeit()
	}()

	wg.Wait()
}