104 lines
2.6 KiB
Go
104 lines
2.6 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
|
|
"github.com/chromedp/cdproto/cdp"
|
|
"github.com/chromedp/chromedp"
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
"github.com/jobs-scraper/services/scraper-google/utils"
|
|
)
|
|
|
|
func Work(mainCtx context.Context) func(page int, stream chan<- domain.GoogleLink) {
|
|
|
|
return func(page int, stream chan<- domain.GoogleLink) {
|
|
query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer"
|
|
|
|
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
|
chromedp.Flag("headless", true),
|
|
chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
|
chromedp.Flag("excludeSwitches", "enable-automation"),
|
|
chromedp.Flag("disable-web-security", false),
|
|
chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
chromedp.WindowSize(1920, 1080),
|
|
)
|
|
allocCtx, cancel := chromedp.NewExecAllocator(mainCtx, opts...)
|
|
defer cancel()
|
|
|
|
ctx, cancel := chromedp.NewContext(allocCtx)
|
|
defer cancel()
|
|
|
|
var nodes []*cdp.Node
|
|
|
|
// start = 0, start = 10, start = 20 and so on for offset pagination
|
|
err := chromedp.Run(ctx,
|
|
chromedp.Navigate(utils.BuildUrl(query, page)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.WaitVisible("#search", chromedp.ByID),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
for i, node := range nodes {
|
|
result := domain.GoogleLink{}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get title for node %d: %v", i, err)
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get link for node %d: %v", i, err)
|
|
}
|
|
|
|
result.ID, err = utils.ExtractJobId(result.Link)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to extract job ID for node %d: %v (link: %s)", i, err, result.Link)
|
|
continue
|
|
}
|
|
|
|
result.CompanyName, err = utils.ExtractCompanyName(result.Link)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to extract company name for node %d: %v (link: %s)", i, err, result.Link)
|
|
continue
|
|
}
|
|
|
|
err = chromedp.Run(ctx,
|
|
chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get description for node %d: %v", i, err)
|
|
}
|
|
|
|
stream <- result
|
|
}
|
|
}
|
|
|
|
}
|