jobs-monorepo/services/scraper-google/main.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"syscall"
	"time"

	"github.com/jobs-scraper/internal/pkg/domain"
	"github.com/jobs-scraper/libs/repo"
	"github.com/jobs-scraper/services/scraper-google/analyzer"
	si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
	"github.com/jobs-scraper/services/scraper-google/utils"
	"github.com/levmv/sked"
)

type SearchResult struct {
	Title       string
	Description string
	Link        string
	CompanyName string
}

func main() {
	db, err := si.SetupInfrastructure()
	if err != nil {
		log.Fatalf("Failed to set up infrastructure: %v", err)
	}

	jobRepo := repo.NewJobRepository(db)
	jobDescRepo := repo.NewJobDescriptionRepository(db)

	ctx, cancel := context.WithCancel(context.Background())

	// Launch Chrome once and share across all goroutines
	wsURL, chromeCmd, err := launchChrome()
	if err != nil {
		log.Fatalf("Failed to launch Chrome: %v", err)
	}
	defer chromeCmd.Process.Kill()
	log.Printf("Chrome launched with WebSocket URL: %s", wsURL)

	sched := sked.New(ctx)
	googleLinkStream := make(chan domain.GoogleLink, 100)
	// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
	pages := []int{100, 110, 120, 130}

	log.Println("Starting scraper with scheduler...")

	for _, page := range pages {
		go func() {
			log.Printf("Running initial scrape for page offset: %d, inital run", page)
			Work(ctx, wsURL)(page, googleLinkStream)
		}()
	}

	for _, page := range pages {
		sched.Schedule(func(ctx context.Context) {
			log.Printf("Running scheduled scrape for page offset: %d", page)
			Work(ctx, wsURL)(page, googleLinkStream)
		}).Every(time.Hour * 24)
	}

	go func() {
		for googleLink := range googleLinkStream {
			log.Printf("Received job: %s ", googleLink.CompanyName)

			exists, err := jobRepo.JobExistsByLink(googleLink.Link)

			if err != nil {
				log.Printf("Error checking if job exists for link %s: %v", googleLink.Link, err)
				continue
			}

			if exists {
				log.Printf("Job already exists, skipping: %s", googleLink.Link)
				continue
			}

			htmlRaw, url, err := utils.GetHTMLRaw(ctx, wsURL, googleLink)

			if err != nil {
				log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
				continue
			}

			result, err := analyzer.GetJobDescription(htmlRaw)
			if err != nil {
				log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err)
				continue
			}

			now := time.Now()

			// Save the main job
			job := domain.Job{
				Title:       googleLink.Title,
				Company:     googleLink.CompanyName,
				CompanyLink: url,
				Location:    result.Criteria["location"],
				JobLink:     googleLink.Link,
				Provider:    domain.Google,
				JobPostTime: &now,
				Status:      domain.JobStatusCreated,
			}

			jobID, err := jobRepo.SaveJob(job)
			if err != nil {
				log.Printf("Failed to save job: %v", err)
				continue
			}

			// Save job description
			jobDescription := domain.JobDescription{
				JobID:       jobID,
				Description: result.Description,
				Criteria:    result.Criteria,
			}

			if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil {
				log.Printf("Failed to save job description for job %d: %v", jobID, err)
				continue
			}

			log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, jobID)
		}
	}()

	if err := sched.Run(); err != nil {
		log.Fatalf("Failed to start scheduler: %v", err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	<-c
	cancel()
	log.Println("Shutting down scraper...")
}

// launchChrome starts Chrome with remote debugging and returns the WebSocket URL
func launchChrome() (string, *exec.Cmd, error) {
	var chromePath string
	switch runtime.GOOS {
	case "darwin":
		chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
	case "linux":
		chromePath = "google-chrome"
	case "windows":
		chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
	default:
		return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
	}

	// Kill any existing Chrome debug instances first
	exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
	time.Sleep(500 * time.Millisecond)

	cmd := exec.Command(chromePath,
		"--remote-debugging-port=9222",
		"--no-first-run",
		"--no-default-browser-check",
		"--disable-background-networking",
		"--disable-extensions",
		"--disable-sync",
		"--disable-translate",
		"--metrics-recording-only",
		"--safebrowsing-disable-auto-update",
		"--user-data-dir=/tmp/chrome-debug-profile",
		// "--incognito",
		// "--headless",
	)

	if err := cmd.Start(); err != nil {
		return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
	}

	// Poll the debug endpoint to get the WebSocket URL
	var wsURL string
	for i := 0; i < 50; i++ {
		time.Sleep(200 * time.Millisecond)
		resp, err := http.Get("http://127.0.0.1:9222/json/version")
		if err != nil {
			continue
		}

		var result struct {
			WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
		}
		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
			resp.Body.Close()
			continue
		}
		resp.Body.Close()

		if result.WebSocketDebuggerURL != "" {
			wsURL = result.WebSocketDebuggerURL
			break
		}
	}

	if wsURL == "" {
		cmd.Process.Kill()
		return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
	}

	// Wait for Chrome to be fully ready to accept connections
	time.Sleep(1 * time.Second)

	return wsURL, cmd, nil
}