package main import ( "context" "log" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/chromedp" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/services/scraper-google/utils" ) func Work(mainCtx context.Context) func(page int, stream chan<- domain.GoogleLink) { return func(page int, stream chan<- domain.GoogleLink) { query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer" opts := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.Flag("headless", false), chromedp.Flag("disable-blink-features", "AutomationControlled"), chromedp.Flag("excludeSwitches", "enable-automation"), chromedp.Flag("disable-web-security", false), chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), chromedp.WindowSize(1920, 1080), ) allocCtx, cancel := chromedp.NewExecAllocator(mainCtx, opts...) defer cancel() ctx, cancel := chromedp.NewContext(allocCtx) defer cancel() var nodes []*cdp.Node // start = 0, start = 10, start = 20 and so on for offset pagination err := chromedp.Run(ctx, chromedp.Navigate(utils.BuildUrl(query, page)), ) if err != nil { log.Fatal(err) } err = chromedp.Run(ctx, chromedp.WaitVisible("#search", chromedp.ByID), ) if err != nil { log.Fatal(err) } err = chromedp.Run(ctx, chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), ) if err != nil { log.Fatal(err) } for i, node := range nodes { result := domain.GoogleLink{} err = chromedp.Run(ctx, chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)), ) if err != nil { log.Printf("Failed to get title for node %d: %v", i, err) } err = chromedp.Run(ctx, chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)), ) if err != nil { log.Printf("Failed to get link for node %d: %v", i, err) } result.ID, err = utils.ExtractJobId(result.Link) if err != nil { log.Printf("Failed to extract job ID for node %d: %v (link: %s)", i, err, result.Link) continue } result.CompanyName, err = utils.ExtractCompanyName(result.Link) if err != nil { log.Printf("Failed to extract company name for node %d: %v (link: %s)", i, err, result.Link) continue } err = chromedp.Run(ctx, chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)), ) if err != nil { log.Printf("Failed to get description for node %d: %v", i, err) } stream <- result } } }