103 lines
2.6 KiB
Go
103 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
// "context"
|
|
"encoding/json"
|
|
"log"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
// "time"
|
|
|
|
"github.com/jobs-scraper/infrastructure"
|
|
localNats "github.com/jobs-scraper/infrastructure/nats"
|
|
"github.com/jobs-scraper/internal/domain"
|
|
"github.com/nats-io/nats.go"
|
|
|
|
// "github.com/jobs-scraper/internal/domain"
|
|
// "github.com/jobs-scraper/internal/pipeline"
|
|
// "github.com/jobs-scraper/internal/repo"
|
|
"github.com/joho/godotenv"
|
|
)
|
|
|
|
func main() {
|
|
// Try to load .local.env first, then fallback to .env
|
|
if err := godotenv.Load(".local.env"); err != nil {
|
|
log.Println("No .local.env file found, trying .env")
|
|
if err := godotenv.Load(); err != nil {
|
|
log.Println("No .env file found, using system environment variables")
|
|
}
|
|
}
|
|
|
|
dbConfig := infrastructure.LoadConfigFromEnv()
|
|
db, err := infrastructure.NewConnection(dbConfig)
|
|
if err != nil {
|
|
log.Fatal("Error connecting to db")
|
|
}
|
|
|
|
err = db.Ping()
|
|
if err != nil {
|
|
log.Fatal("Error pinging db")
|
|
}
|
|
|
|
log.Println("Successfully connected to db")
|
|
|
|
nc, err := localNats.NewNatsClient()
|
|
if err != nil {
|
|
log.Fatal("Error connecting to nats")
|
|
}
|
|
|
|
// scraper := pipeline.NewScraper(pipeline.Config{
|
|
// SortBy: "R",
|
|
// MaxRetries: 3,
|
|
// BaseDelay: 1 * time.Second,
|
|
// MaxDelay: 30 * time.Second,
|
|
// RequestTimeout: 30 * time.Second,
|
|
// })
|
|
|
|
// jobRepo := repo.NewJobRepository(db)
|
|
// jobDescriptionRepo := repo.NewJobDescriptionRepository(db)
|
|
|
|
// jobPipeline := pipeline.NewJobPipeline(scraper, 5, 1*time.Second) // 5 workers, 1 second rate limit
|
|
|
|
// ctx := context.Background()
|
|
|
|
// searchParams := domain.SearchQuery{
|
|
// Keywords: "Javascript",
|
|
// Location: "US",
|
|
// FWT: "2,3",
|
|
// }
|
|
|
|
// Subscribe to LinkedIn topic
|
|
sub, err := nc.Subscribe(localNats.LinkedInSubTopic, "scraper-consumer", func(msg *nats.Msg) {
|
|
var data domain.SearchQuery
|
|
err := json.Unmarshal(msg.Data, &data)
|
|
|
|
if err != nil {
|
|
log.Printf("Error processing message: %v", err)
|
|
return
|
|
|
|
}
|
|
log.Printf("Received message on %s: %s", msg.Subject, string(msg.Data))
|
|
// err = jobPipeline.ProcessJobsStreaming(ctx, 10, jobRepo, jobDescriptionRepo, searchParams)
|
|
|
|
msg.Ack()
|
|
})
|
|
if err != nil {
|
|
log.Fatal("Error subscribing to LinkedIn topic:", err)
|
|
}
|
|
|
|
log.Printf("Successfully subscribed to %s", localNats.LinkedInSubTopic)
|
|
|
|
// Keep the program running to listen for messages
|
|
log.Println("Scraper is running. Press Ctrl+C to stop...")
|
|
|
|
// Wait for interrupt signal
|
|
c := make(chan os.Signal, 1)
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
<-c
|
|
sub.Unsubscribe()
|
|
nc.Close()
|
|
log.Println("Shutting down scraper...")
|
|
}
|