package main import ( "context" "log" "os" "os/signal" "syscall" "time" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/scraper-google/analyzer" si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure" "github.com/jobs-scraper/services/scraper-google/utils" "github.com/levmv/sked" ) type SearchResult struct { Title string Description string Link string CompanyName string } func main() { db, err := si.SetupInfrastructure() if err != nil { log.Fatalf("Failed to set up infrastructure: %v", err) } jobRepo := repo.NewJobRepository(db) jobDescRepo := repo.NewJobDescriptionRepository(db) ctx, cancel := context.WithCancel(context.Background()) sched := sked.New(ctx) googleLinkStream := make(chan domain.GoogleLink, 100) pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} log.Println("Starting scraper with scheduler...") for _, page := range pages { go func() { log.Printf("Running initial scrape for page offset: %d", page) Work(ctx)(page, googleLinkStream) }() sched.Schedule(func(ctx context.Context) { log.Printf("Running scheduled scrape for page offset: %d", page) Work(ctx)(page, googleLinkStream) }).Every(time.Hour * 24) } go func() { for googleLink := range googleLinkStream { htmlRaw, url, err := utils.GetHTMLRaw(ctx, googleLink) if err != nil { log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err) continue } result, err := analyzer.GetJobDescription(htmlRaw) if err != nil { log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err) continue } now := time.Now() job := domain.Job{ ID: googleLink.ID, Title: googleLink.Title, Company: googleLink.CompanyName, CompanyLink: googleLink.Link, Location: result.Criteria["location"], JobLink: url, Provider: domain.Google, JobPostTime: &now, Status: domain.JobStatusCreated, } jobDescription := domain.JobDescription{ JobID: googleLink.ID, Description: result.Description, Criteria: result.Criteria, } if err := jobRepo.SaveJobs([]domain.Job{job}); err != nil { log.Printf("Failed to save job %d: %v", googleLink.ID, err) continue } if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil { log.Printf("Failed to save job description for job %d: %v", googleLink.ID, err) continue } log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, job.ID) } }() if err := sched.Run(); err != nil { log.Fatalf("Failed to start scheduler: %v", err) } c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, syscall.SIGTERM) <-c cancel() log.Println("Shutting down scraper...") }