jobs-monorepo/services/scraper-google/main.go

package main

import (
	"context"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/jobs-scraper/internal/browser"
	"github.com/jobs-scraper/internal/domain"
	"github.com/jobs-scraper/internal/repo"
	"github.com/jobs-scraper/services/scraper-google/analyzer"
	si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
	"github.com/jobs-scraper/services/scraper-google/utils"
	"github.com/levmv/sked"
)

type SearchResult struct {
	Title       string
	Description string
	Link        string
	CompanyName string
}

func main() {
	db, err := si.SetupInfrastructure()
	if err != nil {
		log.Fatalf("Failed to set up infrastructure: %v", err)
	}

	jobRepo := repo.NewJobRepository(db)
	jobDescRepo := repo.NewJobDescriptionRepository(db)

	ctx, cancel := context.WithCancel(context.Background())

	// Launch Chrome once and share across all goroutines
	browser, err := browser.NewBrowser(ctx)

	if err != nil {
		log.Fatalf("Failed to launch Chrome: %v", err)
	}

	defer browser.Close()

	log.Printf("Chrome launched with WebSocket URL: %s", browser.Instance.WSURL)

	sched := sked.New(ctx)
	googleLinkStream := make(chan domain.GoogleLink, 100)
	pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}

	log.Println("Starting scraper with scheduler...")

	for _, page := range pages {
		go func() {
			log.Printf("Running initial scrape for page offset: %d, inital run", page)
			Work(ctx, browser)(page, googleLinkStream)
		}()
	}

	for _, page := range pages {
		sched.Schedule(func(ctx context.Context) {
			log.Printf("Running scheduled scrape for page offset: %d", page)
			Work(ctx, browser)(page, googleLinkStream)
		}).Every(time.Hour * 24)
	}

	go func() {
		for googleLink := range googleLinkStream {
			log.Printf("Received job: %s ", googleLink.CompanyName)

			exists, err := jobRepo.JobExistsByLink(googleLink.Link)

			if err != nil {
				log.Printf("Error checking if job exists for link %s: %v", googleLink.Link, err)
				continue
			}

			if exists {
				log.Printf("Job already exists, skipping: %s", googleLink.Link)
				continue
			}

			htmlRaw, url, err := utils.GetHTMLRaw(ctx, browser, googleLink)

			if err != nil {
				log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
				continue
			}

			result, err := analyzer.GetJobDescription(htmlRaw)
			if err != nil {
				log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err)
				continue
			}

			now := time.Now()

			// Save the main job
			job := domain.Job{
				Title:       googleLink.Title,
				Company:     googleLink.CompanyName,
				CompanyLink: url,
				Location:    result.Criteria["location"],
				JobLink:     googleLink.Link,
				Provider:    domain.Google,
				JobPostTime: &now,
				Status:      domain.JobStatusCreated,
			}

			jobID, err := jobRepo.SaveJob(job)
			if err != nil {
				log.Printf("Failed to save job: %v", err)
				continue
			}

			// Save job description
			jobDescription := domain.JobDescription{
				JobID:       jobID,
				Description: result.Description,
				Criteria:    result.Criteria,
			}

			if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil {
				log.Printf("Failed to save job description for job %d: %v", jobID, err)
				continue
			}

			log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, jobID)
		}
	}()

	if err := sched.Run(); err != nil {
		log.Fatalf("Failed to start scheduler: %v", err)
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	<-c
	cancel()
	log.Println("Shutting down scraper...")
}