package main import ( "context" "encoding/json" "fmt" "log" "net/http" "os" "os/exec" "os/signal" "runtime" "syscall" "time" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/scraper-google/analyzer" si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure" "github.com/jobs-scraper/services/scraper-google/utils" "github.com/levmv/sked" ) type SearchResult struct { Title string Description string Link string CompanyName string } func main() { db, err := si.SetupInfrastructure() if err != nil { log.Fatalf("Failed to set up infrastructure: %v", err) } jobRepo := repo.NewJobRepository(db) jobDescRepo := repo.NewJobDescriptionRepository(db) ctx, cancel := context.WithCancel(context.Background()) // Launch Chrome once and share across all goroutines wsURL, chromeCmd, err := launchChrome() if err != nil { log.Fatalf("Failed to launch Chrome: %v", err) } defer chromeCmd.Process.Kill() log.Printf("Chrome launched with WebSocket URL: %s", wsURL) sched := sked.New(ctx) googleLinkStream := make(chan domain.GoogleLink, 100) pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} // pages := []int{100, 110, 120, 130} log.Println("Starting scraper with scheduler...") for _, page := range pages { go func() { log.Printf("Running initial scrape for page offset: %d, inital run", page) Work(ctx, wsURL)(page, googleLinkStream) }() } for _, page := range pages { sched.Schedule(func(ctx context.Context) { log.Printf("Running scheduled scrape for page offset: %d", page) Work(ctx, wsURL)(page, googleLinkStream) }).Every(time.Hour * 24) } go func() { for googleLink := range googleLinkStream { log.Printf("Received job: %s ", googleLink.CompanyName) exists, err := jobRepo.JobExistsByLink(googleLink.Link) if err != nil { log.Printf("Error checking if job exists for link %s: %v", googleLink.Link, err) continue } if exists { log.Printf("Job already exists, skipping: %s", googleLink.Link) continue } htmlRaw, url, err := utils.GetHTMLRaw(ctx, wsURL, googleLink) if err != nil { log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err) continue } result, err := analyzer.GetJobDescription(htmlRaw) if err != nil { log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err) continue } now := time.Now() // Save the main job job := domain.Job{ Title: googleLink.Title, Company: googleLink.CompanyName, CompanyLink: url, Location: result.Criteria["location"], JobLink: googleLink.Link, Provider: domain.Google, JobPostTime: &now, Status: domain.JobStatusCreated, } jobID, err := jobRepo.SaveJob(job) if err != nil { log.Printf("Failed to save job: %v", err) continue } // Save job description jobDescription := domain.JobDescription{ JobID: jobID, Description: result.Description, Criteria: result.Criteria, } if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil { log.Printf("Failed to save job description for job %d: %v", jobID, err) continue } log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, jobID) } }() if err := sched.Run(); err != nil { log.Fatalf("Failed to start scheduler: %v", err) } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) <-c cancel() log.Println("Shutting down scraper...") } // launchChrome starts Chrome with remote debugging and returns the WebSocket URL func launchChrome() (string, *exec.Cmd, error) { var chromePath string switch runtime.GOOS { case "darwin": chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" case "linux": chromePath = "google-chrome" case "windows": chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" default: return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS) } // Kill any existing Chrome debug instances first exec.Command("pkill", "-f", "remote-debugging-port=9222").Run() time.Sleep(500 * time.Millisecond) cmd := exec.Command(chromePath, "--remote-debugging-port=9222", "--no-first-run", "--no-default-browser-check", "--disable-background-networking", "--disable-extensions", "--disable-sync", "--disable-translate", "--metrics-recording-only", "--safebrowsing-disable-auto-update", "--user-data-dir=/tmp/chrome-debug-profile", // "--incognito", "--headless", ) if err := cmd.Start(); err != nil { return "", nil, fmt.Errorf("failed to start Chrome: %w", err) } // Poll the debug endpoint to get the WebSocket URL var wsURL string for i := 0; i < 50; i++ { time.Sleep(200 * time.Millisecond) resp, err := http.Get("http://127.0.0.1:9222/json/version") if err != nil { continue } var result struct { WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { resp.Body.Close() continue } resp.Body.Close() if result.WebSocketDebuggerURL != "" { wsURL = result.WebSocketDebuggerURL break } } if wsURL == "" { cmd.Process.Kill() return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL") } // Wait for Chrome to be fully ready to accept connections time.Sleep(1 * time.Second) return wsURL, cmd, nil }