package main import ( "context" "log" "github.com/chromedp/cdproto/cdp" "github.com/chromedp/chromedp" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/services/scraper-google/utils" ) func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- domain.GoogleLink) { return func(page int, stream chan<- domain.GoogleLink) { // query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid" query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` // Connect to the shared Chrome instance allocCtx, allocCancel := chromedp.NewRemoteAllocator(mainCtx, wsURL) defer allocCancel() ctx, cancel := chromedp.NewContext(allocCtx) defer cancel() var nodes []*cdp.Node err := chromedp.Run(ctx, chromedp.Navigate(utils.BuildUrl(query, page)), ) log.Printf("Error in run? %v", err) if err != nil { log.Fatal(err) } err = chromedp.Run(ctx, chromedp.WaitVisible("#search", chromedp.ByID), ) if err != nil { log.Fatal(err) } err = chromedp.Run(ctx, chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), ) if err != nil { log.Fatal(err) } for i, node := range nodes { result := domain.GoogleLink{} err = chromedp.Run(ctx, chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)), ) if err != nil { log.Printf("Failed to get title for node %d: %v", i, err) } err = chromedp.Run(ctx, chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)), ) if err != nil { log.Printf("Failed to get link for node %d: %v", i, err) } result.CompanyName, err = utils.ExtractCompanyName(result.Link) if err != nil { log.Printf("Failed to extract company name for node %d: %v (link: %s)", i, err, result.Link) continue } err = chromedp.Run(ctx, chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)), ) if err != nil { log.Printf("Failed to get description for node %d: %v", i, err) } stream <- result } } }