package main import ( // "context" "encoding/json" "log" "os" "os/signal" "syscall" // "time" "github.com/jobs-scraper/infrastructure" localNats "github.com/jobs-scraper/infrastructure/nats" "github.com/jobs-scraper/internal/domain" "github.com/nats-io/nats.go" // "github.com/jobs-scraper/internal/domain" // "github.com/jobs-scraper/internal/pipeline" // "github.com/jobs-scraper/internal/repo" "github.com/joho/godotenv" ) func main() { // Try to load .local.env first, then fallback to .env if err := godotenv.Load(".local.env"); err != nil { log.Println("No .local.env file found, trying .env") if err := godotenv.Load(); err != nil { log.Println("No .env file found, using system environment variables") } } dbConfig := infrastructure.LoadConfigFromEnv() db, err := infrastructure.NewConnection(dbConfig) if err != nil { log.Fatal("Error connecting to db") } err = db.Ping() if err != nil { log.Fatal("Error pinging db") } log.Println("Successfully connected to db") nc, err := localNats.NewNatsClient() if err != nil { log.Fatal("Error connecting to nats") } // scraper := pipeline.NewScraper(pipeline.Config{ // SortBy: "R", // MaxRetries: 3, // BaseDelay: 1 * time.Second, // MaxDelay: 30 * time.Second, // RequestTimeout: 30 * time.Second, // }) // jobRepo := repo.NewJobRepository(db) // jobDescriptionRepo := repo.NewJobDescriptionRepository(db) // jobPipeline := pipeline.NewJobPipeline(scraper, 5, 1*time.Second) // 5 workers, 1 second rate limit // ctx := context.Background() // searchParams := domain.SearchQuery{ // Keywords: "Javascript", // Location: "US", // FWT: "2,3", // } // Subscribe to LinkedIn topic sub, err := nc.Subscribe(localNats.LinkedInSubTopic, "scraper-consumer", func(msg *nats.Msg) { var data domain.SearchQuery err := json.Unmarshal(msg.Data, &data) if err != nil { log.Printf("Error processing message: %v", err) return } log.Printf("Received message on %s: %s", msg.Subject, string(msg.Data)) // err = jobPipeline.ProcessJobsStreaming(ctx, 10, jobRepo, jobDescriptionRepo, searchParams) msg.Ack() }) if err != nil { log.Fatal("Error subscribing to LinkedIn topic:", err) } log.Printf("Successfully subscribed to %s", localNats.LinkedInSubTopic) // Keep the program running to listen for messages log.Println("Scraper is running. Press Ctrl+C to stop...") // Wait for interrupt signal c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) <-c sub.Unsubscribe() nc.Close() log.Println("Shutting down scraper...") }