88 lines
2 KiB
Go
88 lines
2 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
|
|
"github.com/jobs-scraper/internal/pkg/browser"
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
"github.com/jobs-scraper/services/scraper-google/utils"
|
|
)
|
|
|
|
func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream chan<- domain.GoogleLink) {
|
|
return func(page int, stream chan<- domain.GoogleLink) {
|
|
// Create a new tab for this goroutine to avoid conflicts
|
|
tab, err := b.NewTab()
|
|
if err != nil {
|
|
log.Printf("Failed to create new tab: %v", err)
|
|
return
|
|
}
|
|
defer tab.Close()
|
|
|
|
// Random delay before starting to look more human
|
|
browser.RandomDelay(1000, 3000)
|
|
|
|
// query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid"
|
|
query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en`
|
|
// url := utils.BuildUrl(query, page)
|
|
err = tab.Navigate("https://linkedin.com")
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
browser.RandomDelay(2000, 2500)
|
|
|
|
err = tab.Navigate(utils.BuildUrl(query, page))
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
err = tab.WaitVisible(`#search`)
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
nodes, err := tab.GetNodes("[data-snc]")
|
|
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
for i, node := range nodes {
|
|
result := domain.GoogleLink{}
|
|
|
|
result.Title, err = tab.GetTextFromNode(node, "h3")
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get title for node %d: %v", i, err)
|
|
}
|
|
|
|
result.Link, err = tab.GetAttributeFromNode(node, "a", "href")
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get link for node %d: %v", i, err)
|
|
}
|
|
|
|
result.CompanyName, err = utils.ExtractCompanyName(result.Link)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to extract company name for node %d: %v (link: %s)", i, err, result.Link)
|
|
continue
|
|
}
|
|
|
|
result.Description, err = tab.GetTextFromNode(node, "div.VwiC3b")
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get description for node %d: %v", i, err)
|
|
}
|
|
|
|
stream <- result
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
// AIzaSyCDJhGum5VKTy1t-WMx6ccbTZV2r1S2fuE
|