91 lines
2.1 KiB
Go
91 lines
2.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
|
|
"github.com/chromedp/cdproto/cdp"
|
|
"github.com/chromedp/chromedp"
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
"github.com/jobs-scraper/services/scraper-google/utils"
|
|
)
|
|
|
|
func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- domain.GoogleLink) {
|
|
return func(page int, stream chan<- domain.GoogleLink) {
|
|
// query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid"
|
|
query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en`
|
|
|
|
// Connect to the shared Chrome instance
|
|
allocCtx, allocCancel := chromedp.NewRemoteAllocator(mainCtx, wsURL)
|
|
defer allocCancel()
|
|
|
|
ctx, cancel := chromedp.NewContext(allocCtx)
|
|
defer cancel()
|
|
|
|
var nodes []*cdp.Node
|
|
|
|
err := chromedp.Run(ctx,
|
|
chromedp.Navigate(utils.BuildUrl(query, page)),
|
|
)
|
|
|
|
log.Printf("Error in run? %v", err)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.WaitVisible("#search", chromedp.ByID),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
for i, node := range nodes {
|
|
result := domain.GoogleLink{}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get title for node %d: %v", i, err)
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get link for node %d: %v", i, err)
|
|
}
|
|
|
|
result.CompanyName, err = utils.ExtractCompanyName(result.Link)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to extract company name for node %d: %v (link: %s)", i, err, result.Link)
|
|
continue
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get description for node %d: %v", i, err)
|
|
}
|
|
|
|
stream <- result
|
|
}
|
|
}
|
|
|
|
}
|