bugblitz77/jobs-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor

Browse source
This commit is contained in:
Elshimy Ziad Magdy Taha 2025-12-17 17:42:42 +05:00
parent 18842091a4
commit cbcf9bdcc7
18 changed files with 2053 additions and 628 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.work
View file

@ -14,5 +14,6 @@ use (
./libs/server ./libs/server
./services/api ./services/api
./services/scraper-google ./services/scraper-google
./services/scraper-hiringcafe
./services/scraper-linkedin ./services/scraper-linkedin
) )

241
internal/pkg/browser/browser.go Normal file
View file

@ -0,0 +1,241 @@
package browser
import (
"context"
"log"
"math/rand"
"os/exec"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
)
type ChromeInstance struct {
WSURL string
Cmd *exec.Cmd
}
type Browser struct {
Instance *ChromeInstance
allocCtx context.Context
allocCancel context.CancelFunc
ctx context.Context
cancel context.CancelFunc
}
// NewBrowser launches Chrome and creates a shared allocator.
// Use NewTab() to create separate tabs for concurrent use.
func NewBrowser(ctx context.Context) (*Browser, error) {
wsURL, cmd, err := launchChrome()
if err != nil {
return nil, err
}
allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
// Create initial browser context (first tab)
browserCtx, cancel := chromedp.NewContext(allocCtx)
return &Browser{
Instance: &ChromeInstance{
WSURL: wsURL,
Cmd: cmd,
},
allocCtx: allocCtx,
allocCancel: allocCancel,
ctx: browserCtx,
cancel: cancel,
}, nil
}
// NewTab creates a new browser tab that can be used concurrently.
// Returns a new Browser instance sharing the same Chrome process.
func (b *Browser) NewTab() (*Browser, error) {
tabCtx, cancel := chromedp.NewContext(b.allocCtx)
return &Browser{
Instance: b.Instance,
allocCtx: b.allocCtx,
allocCancel: nil, // Don't close allocator when closing a tab
ctx: tabCtx,
cancel: cancel,
}, nil
}
func (b *Browser) Close() error {
if b.cancel != nil {
b.cancel()
}
if b.allocCancel != nil {
b.allocCancel()
}
// Only kill Chrome process if this is the main browser (has allocCancel),
// not a tab created via NewTab()
if b.allocCancel != nil && b.Instance != nil && b.Instance.Cmd != nil && b.Instance.Cmd.Process != nil {
return b.Instance.Cmd.Process.Kill()
}
return nil
}
func (b *Browser) Navigate(url string) error {
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36")
err := chromedp.Run(b.ctx,
chromedp.Navigate(url),
)
if err != nil {
return err
}
return nil
}
func (b *Browser) WaitVisible(selector string) error {
ctx, cancel := context.WithTimeout(b.ctx, 30*time.Second)
defer cancel()
err := chromedp.Run(ctx,
chromedp.WaitVisible(selector, chromedp.ByQuery),
)
if err != nil {
return err
}
return nil
}
func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
var nodes []*cdp.Node
err := chromedp.Run(b.ctx,
chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll),
)
if err != nil {
return nodes, err
}
return nodes, nil
}
func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) {
var text string
err := chromedp.Run(b.ctx,
chromedp.Text(selector, &text, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil {
return "", err
}
return text, nil
}
func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr string) (string, error) {
var value string
err := chromedp.Run(b.ctx,
chromedp.AttributeValue(selector, attr, &value, nil, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil {
return "", err
}
return value, nil
}
func Evaluate[T any](b *Browser, expression string) (T, error) {
var result T
err := chromedp.Run(b.ctx,
chromedp.Evaluate(expression, &result),
)
if err != nil {
return result, err
}
return result, nil
}
func (b *Browser) GetCurrentLocation() (string, error) {
var finalURL string
err := chromedp.Run(b.ctx,
chromedp.Location(&finalURL),
)
if err != nil {
log.Printf("Failed to get current location: %v", err)
return "", err
}
return finalURL, nil
}
func (b *Browser) GetHTML() (string, error) {
var htmlContent string
err := chromedp.Run(b.ctx,
chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
)
if err != nil {
log.Printf("Failed to get HTML content: %v", err)
return "", err
}
return htmlContent, nil
}
func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error {
err := chromedp.Run(b.ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
return page.SetLifecycleEventsEnabled(true).Do(ctx)
}),
)
if err != nil {
return err
}
ch := make(chan struct{}, 1)
chromedp.ListenTarget(b.ctx, func(ev interface{}) {
if e, ok := ev.(*page.EventLifecycleEvent); ok {
if e.Name == "networkIdle" {
select {
case ch <- struct{}{}:
default:
}
}
}
})
select {
case <-ch:
return nil
case <-time.After(timeout):
// Timeout reached, but don't error - page may still be usable
return nil
case <-b.ctx.Done():
return b.ctx.Err()
}
}
// RandomDelay waits for a random duration between min and max milliseconds
func RandomDelay(minMs, maxMs int) {
delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond
time.Sleep(delay)
}
// Sleep waits for the specified duration
func (b *Browser) Sleep(d time.Duration) {
time.Sleep(d)
}

103
internal/pkg/browser/launch-chrome.go Normal file
View file

@ -0,0 +1,103 @@
package browser
import (
"encoding/json"
"fmt"
"net/http"
"os/exec"
"runtime"
"time"
)
func launchChrome() (string, *exec.Cmd, error) {
var chromePath string
switch runtime.GOOS {
case "darwin":
chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
case "linux":
chromePath = "google-chrome"
case "windows":
chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
default:
return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
}
// Kill any existing Chrome debug instances first
exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
time.Sleep(500 * time.Millisecond)
cmd := exec.Command(chromePath,
"--remote-debugging-port=9222",
"--no-first-run",
"--no-default-browser-check",
"--user-data-dir=/tmp/chrome-debug-profile",
// Anti-detection flags
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--disable-dev-shm-usage",
"--disable-browser-side-navigation",
"--disable-gpu",
"--no-sandbox",
"--disable-setuid-sandbox",
// Hide automation indicators
"--disable-extensions",
"--disable-plugins-discovery",
"--disable-bundled-ppapi-flash",
// Reduce fingerprinting
"--disable-background-networking",
"--disable-sync",
"--disable-translate",
"--metrics-recording-only",
"--safebrowsing-disable-auto-update",
"--disable-client-side-phishing-detection",
"--disable-default-apps",
"--disable-hang-monitor",
"--disable-popup-blocking",
"--disable-prompt-on-repost",
// Window size to look like a real browser
"--window-size=1920,1080",
"--start-maximized",
)
if err := cmd.Start(); err != nil {
return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
}
// Poll the debug endpoint to get the WebSocket URL
var wsURL string
for range 50 {
time.Sleep(200 * time.Millisecond)
resp, err := http.Get("http://127.0.0.1:9222/json/version")
if err != nil {
continue
}
var result struct {
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
resp.Body.Close()
continue
}
resp.Body.Close()
if result.WebSocketDebuggerURL != "" {
wsURL = result.WebSocketDebuggerURL
break
}
}
if wsURL == "" {
cmd.Process.Kill()
return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
}
// Wait for Chrome to be fully ready to accept connections
time.Sleep(1 * time.Second)
return wsURL, cmd, nil
}

1
internal/pkg/domain/job.go
View file

@ -15,6 +15,7 @@ const (
TokyoDev TokyoDev
JapanDev JapanDev
Google Google
HiringCafe
) )
const ( const (

317
internal/pkg/utils/retryable-http-request.go
View file

@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
// "net/url" // "net/url"
"time" "time"
) )
@ -52,7 +53,6 @@ func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *Retr
} }
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) { func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
var lastErr error var lastErr error
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ { for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
@ -108,318 +108,3 @@ func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr) return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
} }
// 5.10.245.81:80
// 172.67.70.206:80
// 190.93.245.33:80
// 185.221.160.176:80
// 141.101.123.156:80
// 103.21.244.129:80
// 103.169.142.59:80
// 172.67.81.185:80
// 37.18.73.60:5566
// 209.38.83.56:1088
// 89.169.36.109:1080
// 103.21.244.248:80
// 172.67.70.92:80
// 172.67.88.34:80
// 141.101.120.38:80
// 172.67.75.255:80
// 172.67.75.171:80
// 45.131.4.157:80
// 141.101.121.208:80
// 141.101.120.70:80
// 103.21.244.70:80
// 103.21.244.140:80
// 103.21.244.138:80
// 154.16.146.43:80
// 128.199.202.122:3128
// 160.153.0.28:80
// 185.170.166.31:80
// 172.67.70.95:80
// 141.101.123.91:80
// 141.101.121.81:80
// 141.101.121.71:80
// 141.101.120.224:80
// 192.111.137.35:4145
// 45.12.31.20:80
// 103.21.244.150:80
// 103.21.244.214:80
// 103.21.244.29:80
// 195.85.23.88:80
// 23.227.39.134:80
// 66.42.224.229:41679
// 172.67.73.123:80
// 79.110.200.27:8000
// 5.182.34.33:80
// 103.21.244.174:80
// 172.67.75.203:80
// 141.101.122.150:80
// 141.101.120.35:80
// 175.47.237.95:6128
// 103.21.244.185:80
// 141.101.122.86:80
// 141.101.121.89:80
// 103.21.244.105:80
// 103.21.244.234:80
// 172.67.70.20:80
// 163.223.172.27:1080
// 172.67.84.128:80
// 103.21.244.57:80
// 31.43.179.191:80
// 173.245.59.61:80
// 141.101.120.48:80
// 185.162.230.19:80
// 172.67.172.150:80
// 159.112.235.63:80
// 172.64.40.184:80
// 103.21.244.55:80
// 103.21.244.49:80
// 172.67.70.228:80
// 103.21.244.8:80
// 66.235.200.77:80
// 45.85.119.147:80
// 172.67.177.231:80
// 173.245.49.28:80
// 110.232.92.49:8080
// 212.183.88.212:80
// 108.162.193.73:80
// 185.238.228.29:80
// 172.67.127.8:80
// 164.38.155.86:80
// 160.153.0.18:80
// 141.101.122.119:80
// 103.21.244.186:80
// 172.67.172.162:80
// 8.218.39.40:10800
// 47.237.132.101:60031
// 222.59.173.105:44193
// 143.110.190.60:1080
// 152.53.194.55:32059
// 115.187.50.40:5678
// 171.254.219.180:1080
// 103.165.157.155:8080
// 45.233.169.57:999
// 222.59.173.105:45035
// 202.40.179.18:4145
// 170.79.181.188:60606
// 37.186.66.36:3629
// 104.200.152.30:4145
// 106.13.58.110:8888
// 202.40.181.220:31247
// 43.135.36.240:80
// 103.82.27.107:10001
// 192.252.209.158:4145
// 142.54.236.97:4145
// 142.54.239.1:4145
// 184.170.245.148:4145
// 68.71.247.130:4145
// 72.195.34.58:4145
// 172.65.90.2:80
// 172.65.90.0:80
// 36.138.53.26:10019
// 183.215.23.242:9091
// 45.131.7.34:80
// 141.101.120.230:80
// 172.64.78.193:80
// 141.101.122.9:80
// 103.21.244.80:80
// 103.21.244.147:80
// 103.21.244.106:80
// 103.21.244.16:80
// 199.34.229.210:80
// 141.101.90.98:80
// 172.67.70.127:80
// 103.21.244.221:80
// 172.67.171.203:80
// 172.67.181.188:80
// 72.195.101.99:4145
// 141.101.120.106:80
// 103.21.244.180:80
// 103.21.244.102:80
// 172.67.70.233:80
// 108.162.193.107:80
// 45.131.208.112:80
// 154.194.12.195:80
// 69.84.182.23:80
// 141.101.120.128:80
// 141.101.120.149:80
// 141.101.120.232:80
// 141.101.120.6:80
// 172.67.98.18:80
// 141.101.113.20:80
// 208.65.90.21:4145
// 198.177.254.131:4145
// 142.54.237.38:4145
// 192.252.220.89:4145
// 199.187.210.54:4145
// 199.102.105.242:4145
// 199.102.107.145:4145
// 198.8.84.3:4145
// 98.182.171.161:4145
// 192.111.129.150:4145
// 184.170.248.5:4145
// 98.188.47.150:4145
// 125.228.94.232:4145
// 125.228.94.153:4145
// 185.162.229.219:80
// 103.21.244.22:80
// 172.64.89.97:80
// 103.21.244.54:80
// 103.21.244.37:80
// 141.101.123.225:80
// 103.21.244.168:80
// 103.21.244.149:80
// 103.21.244.100:80
// 103.21.244.24:80
// 23.227.39.209:80
// 172.67.83.15:80
// 45.131.4.241:80
// 141.101.120.201:80
// 172.67.191.237:80
// 172.67.127.188:80
// 103.21.244.92:80
// 103.160.204.22:80
// 220.197.44.36:3128
// 39.185.41.193:5911
// 23.227.39.121:80
// 139.162.78.109:80
// 188.114.99.144:80
// 45.131.5.37:80
// 45.131.4.250:80
// 23.227.39.65:80
// 141.101.121.191:80
// 172.67.188.16:80
// 172.67.254.148:80
// 221.1.104.177:7302
// 222.59.173.105:44008
// 103.21.244.83:80
// 172.64.149.1:80
// 45.12.30.22:80
// 222.59.173.105:44027
// 172.64.149.26:80
// 103.21.244.46:80
// 45.12.31.242:80
// 23.227.38.195:80
// 172.67.177.162:80
// 103.160.204.200:80
// 172.67.70.129:80
// 141.101.123.245:80
// 185.162.230.117:80
// 185.162.230.183:80
// 69.61.200.104:36181
// 208.65.90.3:4145
// 72.223.188.92:4145
// 192.252.214.17:4145
// 68.71.242.118:4145
// 192.252.210.233:4145
// 107.181.161.81:4145
// 107.181.168.145:4145
// 206.220.175.2:4145
// 72.37.217.3:4145
// 192.252.208.70:14282
// 70.166.167.55:57745
// 192.111.137.37:18762
// 98.188.47.132:4145
// 62.99.138.162:80
// 172.64.90.186:80
// 172.67.74.57:80
// 45.131.6.67:80
// 141.101.122.37:80
// 103.21.244.134:80
// 172.64.155.71:80
// 5.182.34.139:80
// 45.131.6.31:80
// 141.101.121.21:80
// 141.101.120.190:80
// 103.21.244.161:80
// 103.21.244.156:80
// 103.21.244.151:80
// 172.64.89.0:80
// 58.216.109.17:800
// 58.241.88.18:800
// 36.147.78.166:80
// 170.244.26.36:8888
// 170.244.25.52:8888
// 170.244.26.206:8888
// 36.138.53.26:10017
// 201.148.32.162:80
// 203.19.38.114:1080
// 31.43.179.204:80
// 45.12.31.50:80
// 170.244.27.142:8888
// 170.244.26.195:8888
// 170.244.27.58:8888
// 170.244.27.61:8888
// 170.244.27.150:8888
// 47.243.94.125:1080
// 40.177.65.8:80
// 47.57.13.107:80
// 194.158.203.14:80
// 194.219.134.234:80
// 103.21.244.51:80
// 103.21.244.160:80
// 103.21.244.144:80
// 213.33.126.130:80
// 103.21.244.69:80
// 103.21.244.192:80
// 103.21.244.133:80
// 141.193.213.189:80
// 68.71.254.6:4145
// 185.221.160.21:80
// 188.114.99.97:80
// 63.141.128.94:80
// 141.193.213.213:80
// 185.162.228.234:80
// 213.143.113.82:80
// 154.194.12.207:80
// 189.203.181.34:1080
// 202.144.134.150:5678
// 190.242.157.215:8080
// 72.49.49.11:31034
// 142.54.237.34:4145
// 68.71.249.153:48606
// 192.252.209.155:14455
// 192.252.216.86:4145
// 142.54.229.249:4145
// 209.97.150.167:8080
// 192.252.220.92:17328
// 98.178.72.21:10919
// 144.124.228.87:1080
// 183.240.46.42:80
// 32.223.6.94:80
// 192.252.214.20:15864
// 198.177.252.24:4145
// 192.252.208.67:14287
// 198.8.94.170:4145
// 68.71.241.33:4145
// 98.181.137.83:4145
// 45.12.30.139:80
// 103.21.244.97:80
// 65.1.148.157:80
// 199.58.184.97:4145
// 141.101.120.148:80
// 185.238.228.77:80
// 172.67.202.134:80
// 172.67.182.49:80
// 45.131.210.112:80
// 141.101.120.100:80
// 103.21.244.59:80
// 103.21.244.189:80
// 103.21.244.154:80
// 103.21.244.23:80
// 173.245.49.40:80
// 82.200.235.134:38191
// 43.224.118.89:2626
// 119.148.47.226:16464
// 211.230.49.122:3128
// 62.171.159.232:8888
// 115.127.112.34:1080
// 185.191.236.162:3128
// 41.223.119.156:3128
// 103.138.123.242:8082
// 121.169.46.116:1090
// 185.145.185.218:8080
// 45.166.93.113:999
// 163.53.204.178:9813
// 181.224.226.154:8080

29
services/scraper-glassdoor/index.ts
View file

@ -107,21 +107,24 @@ async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise<void> {
}); });
await randomDelay(1500, 2500); await randomDelay(1500, 2500);
await page.goto("https://www.google.com/search?q=site:boards.greenhouse.io+(Software+OR+Backend+OR+Full-Stack)+Engineer", {
waitUntil: ["networkidle2", "domcontentloaded"],
timeout: 60000,
});
// await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
// const jobs = await browserService.extractJobListings(searchQuery.location);
await browserService.searchJobs(searchQuery.keywords, searchQuery.location); // if (jobs.length > 0) {
const jobs = await browserService.extractJobListings(searchQuery.location); // await saveJobs(jobs);
// await saveJobDescriptions(jobs);
if (jobs.length > 0) { // console.log(
await saveJobs(jobs); // `saved ${jobs.length} jobs from Glassdoor for query:`,
await saveJobDescriptions(jobs); // searchQuery
// );
console.log( // } else {
`saved ${jobs.length} jobs from Glassdoor for query:`, // console.log("No jobs found for this search query");
searchQuery // }
);
} else {
console.log("No jobs found for this search query");
}
await randomDelay(1500, 2500); await randomDelay(1500, 2500);
} catch (error) { } catch (error) {

98
services/scraper-google/main.go
View file

@ -2,17 +2,13 @@ package main
import ( import (
"context" "context"
"encoding/json"
"fmt"
"log" "log"
"net/http"
"os" "os"
"os/exec"
"os/signal" "os/signal"
"runtime"
"syscall" "syscall"
"time" "time"
"github.com/jobs-scraper/internal/pkg/browser"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper-google/analyzer" "github.com/jobs-scraper/services/scraper-google/analyzer"
@ -40,31 +36,35 @@ func main() {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
// Launch Chrome once and share across all goroutines // Launch Chrome once and share across all goroutines
wsURL, chromeCmd, err := launchChrome() browser, err := browser.NewBrowser(ctx)
if err != nil { if err != nil {
log.Fatalf("Failed to launch Chrome: %v", err) log.Fatalf("Failed to launch Chrome: %v", err)
} }
defer chromeCmd.Process.Kill()
log.Printf("Chrome launched with WebSocket URL: %s", wsURL) defer browser.Close()
log.Printf("Chrome launched with WebSocket URL: %s", browser.Instance.WSURL)
sched := sked.New(ctx) sched := sked.New(ctx)
googleLinkStream := make(chan domain.GoogleLink, 100) googleLinkStream := make(chan domain.GoogleLink, 100)
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} pages := []int{0, 10, 20, 30, 40}
// pages := []int{100, 110, 120, 130} // pages := []int{100}
log.Println("Starting scraper with scheduler...") log.Println("Starting scraper with scheduler...")
for _, page := range pages { for _, page := range pages {
go func() { go func() {
log.Printf("Running initial scrape for page offset: %d, inital run", page) log.Printf("Running initial scrape for page offset: %d, inital run", page)
Work(ctx, wsURL)(page, googleLinkStream) Work(ctx, browser)(page, googleLinkStream)
}() }()
} }
for _, page := range pages { for _, page := range pages {
sched.Schedule(func(ctx context.Context) { sched.Schedule(func(ctx context.Context) {
log.Printf("Running scheduled scrape for page offset: %d", page) log.Printf("Running scheduled scrape for page offset: %d", page)
Work(ctx, wsURL)(page, googleLinkStream) Work(ctx, browser)(page, googleLinkStream)
}).Every(time.Hour * 24) }).Every(time.Hour * 24)
} }
@ -84,7 +84,7 @@ func main() {
continue continue
} }
htmlRaw, url, err := utils.GetHTMLRaw(ctx, wsURL, googleLink) htmlRaw, url, err := utils.GetHTMLRaw(ctx, browser, googleLink)
if err != nil { if err != nil {
log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err) log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
@ -143,75 +143,3 @@ func main() {
cancel() cancel()
log.Println("Shutting down scraper...") log.Println("Shutting down scraper...")
} }
// launchChrome starts Chrome with remote debugging and returns the WebSocket URL
func launchChrome() (string, *exec.Cmd, error) {
var chromePath string
switch runtime.GOOS {
case "darwin":
chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
case "linux":
chromePath = "google-chrome"
case "windows":
chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
default:
return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
}
// Kill any existing Chrome debug instances first
exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
time.Sleep(500 * time.Millisecond)
cmd := exec.Command(chromePath,
"--remote-debugging-port=9222",
"--no-first-run",
"--no-default-browser-check",
"--disable-background-networking",
"--disable-extensions",
"--disable-sync",
"--disable-translate",
"--metrics-recording-only",
"--safebrowsing-disable-auto-update",
"--user-data-dir=/tmp/chrome-debug-profile",
// "--incognito",
"--headless",
)
if err := cmd.Start(); err != nil {
return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
}
// Poll the debug endpoint to get the WebSocket URL
var wsURL string
for i := 0; i < 50; i++ {
time.Sleep(200 * time.Millisecond)
resp, err := http.Get("http://127.0.0.1:9222/json/version")
if err != nil {
continue
}
var result struct {
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
resp.Body.Close()
continue
}
resp.Body.Close()
if result.WebSocketDebuggerURL != "" {
wsURL = result.WebSocketDebuggerURL
break
}
}
if wsURL == "" {
cmd.Process.Kill()
return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
}
// Wait for Chrome to be fully ready to accept connections
time.Sleep(1 * time.Second)
return wsURL, cmd, nil
}

176
services/scraper-google/utils/get-html-raw.go
View file

@ -9,56 +9,72 @@ import (
"time" "time"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/chromedp/chromedp" "github.com/jobs-scraper/internal/pkg/browser"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/domain"
) )
func GetHTMLRaw(ctx context.Context, wsURL string, googleLink domain.GoogleLink) (string, string, error) { func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) {
// Connect to the shared Chrome instance tab, err := b.NewTab()
allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
defer allocCancel()
browserCtx, browserCancel := chromedp.NewContext(allocCtx)
defer browserCancel()
var htmlContent string
var finalURL string
err := chromedp.Run(browserCtx,
chromedp.Navigate(googleLink.Link),
chromedp.WaitVisible("body", chromedp.ByQuery),
WaitForNetworkIdle(10*time.Second), // Wait for all network requests to complete
chromedp.Location(&finalURL),
chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
)
if err != nil { if err != nil {
log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err) return "", "", fmt.Errorf("failed to create new tab: %w", err)
}
defer tab.Close()
err = tab.Navigate(googleLink.Link)
if err != nil {
log.Printf("Failed to navigate for link %s: %v", googleLink.Link, err)
return "", "", err
}
err = tab.WaitVisible("body")
if err != nil {
log.Printf("Failed to wait for visible body for link %s: %v", googleLink.Link, err)
return "", "", err
}
err = tab.WaitForNetworkIdle(10 * time.Second)
if err != nil {
log.Printf("Failed to wait network idle for link %s: %v", googleLink.Link, err)
return "", "", err
}
htmlContent, err := tab.GetHTML()
if err != nil {
log.Printf("Failed to get HTML content for link %s: %v", googleLink.Link, err)
return "", "", err return "", "", err
} }
// Parse the HTML with goquery
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
if err != nil { if err != nil {
log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err) log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err)
return "", "", err return "", "", err
} }
// Remove unwanted elements
doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove()
log.Printf("Received: %s from %s", googleLink.Title, googleLink.CompanyName) log.Printf("Received: %s from %s", googleLink.Title, googleLink.CompanyName)
// Extract text content
var content strings.Builder var content strings.Builder
doc.Find("body").Each(func(i int, s *goquery.Selection) { doc.Find("body").Each(func(i int, s *goquery.Selection) {
content.WriteString(s.Text()) content.WriteString(s.Text())
}) })
// Extract hostname from the final URL finalUrl, err := tab.GetCurrentLocation()
hostname, err := extractHostname(finalURL)
if err != nil { if err != nil {
log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err) log.Printf("Failed to get current location for link %s: %v", googleLink.Link, err)
return cleanText(content.String()), googleLink.Link, nil
}
hostname, err := extractHostname(finalUrl)
if err != nil {
log.Printf("Failed to extract hostname from final URL %s: %v", finalUrl, err)
// Fall back to the original link if extraction fails // Fall back to the original link if extraction fails
return cleanText(content.String()), googleLink.Link, nil return cleanText(content.String()), googleLink.Link, nil
} }
@ -78,111 +94,3 @@ func extractHostname(urlStr string) (string, error) {
} }
return parsedURL.Hostname(), nil return parsedURL.Hostname(), nil
} }
// ClickSelector clicks on an element using the provided selector string
// Selector format: "css:selector" or "xpath:selector"
func ClickSelector(ctx context.Context, selector string) error {
if selector == "" {
return fmt.Errorf("empty selector")
}
parts := strings.SplitN(selector, ":", 2)
if len(parts) != 2 {
return fmt.Errorf("invalid selector format: %s", selector)
}
selectorType := parts[0]
selectorValue := parts[1]
var err error
switch selectorType {
case "css":
err = chromedp.Run(ctx,
chromedp.WaitVisible(selectorValue, chromedp.ByQuery),
chromedp.Click(selectorValue, chromedp.ByQuery),
)
case "xpath":
err = chromedp.Run(ctx,
chromedp.WaitVisible(selectorValue, chromedp.BySearch),
chromedp.Click(selectorValue, chromedp.BySearch),
)
default:
return fmt.Errorf("unknown selector type: %s", selectorType)
}
if err != nil {
return fmt.Errorf("failed to click selector %s: %w", selector, err)
}
log.Printf("Successfully clicked on selector: %s", selector)
return nil
}
// GetHTMLRawWithRetry fetches HTML and can retry with a selector click if needed
func GetHTMLRawWithRetry(ctx context.Context, wsURL string, googleLink domain.GoogleLink, selector string) (string, string, error) {
// Connect to the shared Chrome instance
allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
defer allocCancel()
browserCtx, browserCancel := chromedp.NewContext(allocCtx)
defer browserCancel()
timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 45*time.Second)
defer timeoutCancel()
var htmlContent string
var finalURL string
// Navigate to the page
err := chromedp.Run(timeoutCtx,
chromedp.Navigate(googleLink.Link),
chromedp.WaitVisible("body", chromedp.ByQuery),
WaitForNetworkIdle(5*time.Second),
)
if err != nil {
return "", "", fmt.Errorf("failed to navigate: %w", err)
}
// If a selector is provided, click on it
if selector != "" {
if err := ClickSelector(timeoutCtx, selector); err != nil {
log.Printf("Failed to click selector %s: %v", selector, err)
// Continue anyway, we'll get whatever HTML we can
} else {
// Wait for the new page to load after clicking
_ = chromedp.Run(timeoutCtx,
chromedp.WaitVisible("body", chromedp.ByQuery),
WaitForNetworkIdle(5*time.Second),
)
}
}
// Get the final HTML
err = chromedp.Run(timeoutCtx,
chromedp.Location(&finalURL),
chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
)
if err != nil {
return "", "", fmt.Errorf("failed to get HTML: %w", err)
}
// Parse and clean HTML
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
if err != nil {
return "", "", fmt.Errorf("failed to parse HTML: %w", err)
}
doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove()
var content strings.Builder
doc.Find("body").Each(func(i int, s *goquery.Selection) {
content.WriteString(s.Text())
})
hostname, err := extractHostname(finalURL)
if err != nil {
return cleanText(content.String()), googleLink.Link, nil
}
return cleanText(content.String()), hostname, nil
}

45
services/scraper-google/utils/network.go
View file

@ -1,45 +0,0 @@
package utils
import (
"context"
"time"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
)
// WaitForNetworkIdle waits for the network to be idle (no pending requests)
// for the specified duration. This is useful for pages that load content dynamically.
func WaitForNetworkIdle(timeout time.Duration) chromedp.ActionFunc {
return func(ctx context.Context) error {
// Enable lifecycle events
if err := page.SetLifecycleEventsEnabled(true).Do(ctx); err != nil {
return err
}
ch := make(chan struct{}, 1)
cctx, cancel := context.WithCancel(ctx)
defer cancel()
chromedp.ListenTarget(cctx, func(ev interface{}) {
if e, ok := ev.(*page.EventLifecycleEvent); ok {
if e.Name == "networkIdle" {
select {
case ch <- struct{}{}:
default:
}
}
}
})
select {
case <-ch:
return nil
case <-time.After(timeout):
// Timeout reached, but don't error - page may still be usable
return nil
case <-ctx.Done():
return ctx.Err()
}
}
}

63
services/scraper-google/work.go
View file

@ -4,47 +4,48 @@ import (
"context" "context"
"log" "log"
"github.com/chromedp/cdproto/cdp" "github.com/jobs-scraper/internal/pkg/browser"
"github.com/chromedp/chromedp"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/services/scraper-google/utils" "github.com/jobs-scraper/services/scraper-google/utils"
) )
func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- domain.GoogleLink) { func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream chan<- domain.GoogleLink) {
return func(page int, stream chan<- domain.GoogleLink) { return func(page int, stream chan<- domain.GoogleLink) {
// Create a new tab for this goroutine to avoid conflicts
tab, err := b.NewTab()
if err != nil {
log.Printf("Failed to create new tab: %v", err)
return
}
defer tab.Close()
// Random delay before starting to look more human
browser.RandomDelay(1000, 3000)
// query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid" // query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid"
query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en`
// url := utils.BuildUrl(query, page)
// Connect to the shared Chrome instance err = tab.Navigate("https://linkedin.com")
allocCtx, allocCancel := chromedp.NewRemoteAllocator(mainCtx, wsURL)
defer allocCancel()
ctx, cancel := chromedp.NewContext(allocCtx)
defer cancel()
var nodes []*cdp.Node
err := chromedp.Run(ctx,
chromedp.Navigate(utils.BuildUrl(query, page)),
)
log.Printf("Error in run? %v", err)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
err = chromedp.Run(ctx, browser.RandomDelay(2000, 2500)
chromedp.WaitVisible("#search", chromedp.ByID),
) err = tab.Navigate(utils.BuildUrl(query, page))
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
err = chromedp.Run(ctx, err = tab.WaitVisible(`#search`)
chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll),
) if err != nil {
log.Fatal(err)
}
nodes, err := tab.GetNodes("[data-snc]")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
@ -53,17 +54,13 @@ func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- do
for i, node := range nodes { for i, node := range nodes {
result := domain.GoogleLink{} result := domain.GoogleLink{}
err = chromedp.Run(ctx, result.Title, err = tab.GetTextFromNode(node, "h3")
chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil { if err != nil {
log.Printf("Failed to get title for node %d: %v", i, err) log.Printf("Failed to get title for node %d: %v", i, err)
} }
err = chromedp.Run(ctx, result.Link, err = tab.GetAttributeFromNode(node, "a", "href")
chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil { if err != nil {
log.Printf("Failed to get link for node %d: %v", i, err) log.Printf("Failed to get link for node %d: %v", i, err)
@ -76,9 +73,7 @@ func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- do
continue continue
} }
err = chromedp.Run(ctx, result.Description, err = tab.GetTextFromNode(node, "div.VwiC3b")
chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil { if err != nil {
log.Printf("Failed to get description for node %d: %v", i, err) log.Printf("Failed to get description for node %d: %v", i, err)
@ -89,3 +84,5 @@ func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- do
} }
} }
// AIzaSyCDJhGum5VKTy1t-WMx6ccbTZV2r1S2fuE

245
services/scraper-hiringcafe/api/client.go Normal file
View file

@ -0,0 +1,245 @@
package api
import (
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"time"
)
const (
BaseURL = "https://hiring.cafe"
SearchURL = BaseURL + "/api/search-jobs"
UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
PageSize = 20
)
// Client handles API requests to hiring.cafe
type Client struct {
httpClient *http.Client
}
// NewClient creates a new API client
func NewClient() *Client {
return &Client{
httpClient: &http.Client{
Timeout: 30 * time.Second,
},
}
}
// SearchRequest represents the search request payload (based on RSSHub implementation)
type SearchRequest struct {
Size int `json:"size"`
Page int `json:"page"`
SearchState SearchState `json:"searchState"`
}
// SearchState contains search parameters
type SearchState struct {
SearchQuery string `json:"searchQuery"`
SortBy string `json:"sortBy"` // date, default, compensation_desc, experience_asc
}
// SearchResponse represents the API response
type SearchResponse struct {
Results []JobResult `json:"results"`
Total int `json:"total"`
}
// JobResult represents a job result from the API
type JobResult struct {
ID string `json:"id"`
ApplyURL string `json:"apply_url"`
JobInformation JobInformation `json:"job_information"`
ProcessedJobData ProcessedJobData `json:"v5_processed_job_data"`
}
// JobInformation contains basic job info
type JobInformation struct {
Title string `json:"title"`
Description string `json:"description"`
}
// ProcessedJobData contains processed job details
type ProcessedJobData struct {
CompanyName string `json:"company_name"`
IsCompensationTransparent bool `json:"is_compensation_transparent"`
YearlyMinCompensation *float64 `json:"yearly_min_compensation"`
YearlyMaxCompensation *float64 `json:"yearly_max_compensation"`
WorkplaceType string `json:"workplace_type"`
RequirementsSummary string `json:"requirements_summary"`
JobCategory string `json:"job_category"`
RoleActivities []string `json:"role_activities"`
FormattedWorkplaceLocation string `json:"formatted_workplace_location"`
EstimatedPublishDateMillis string `json:"estimated_publish_date_millis"`
}
// Job is a simplified job structure for external use
type Job struct {
ID string
ApplyURL string
Title string
Description string
Company string
CompanyURL string
Location string
Remote bool
Salary string
PostedAt time.Time
DescriptionClean string
}
// SearchFilters contains user-configurable search filters
type SearchFilters struct {
Query string
Country string
Remote bool
Hybrid bool
Onsite bool
FullTime bool
PartTime bool
Contract bool
Internship bool
EntryLevel bool
MidLevel bool
SeniorLevel bool
DateRangeDays int
MaxPages int
}
// DefaultFilters returns default search filters
func DefaultFilters() SearchFilters {
return SearchFilters{
Query: "",
Country: "United States",
Remote: true,
Hybrid: true,
Onsite: true,
FullTime: true,
PartTime: false,
Contract: false,
Internship: false,
EntryLevel: true,
MidLevel: true,
SeniorLevel: false,
DateRangeDays: 30,
MaxPages: 10,
}
}
// Search performs a job search with the given filters
func (c *Client) Search(filters SearchFilters) ([]Job, error) {
var allJobs []Job
page := 1
for {
jobs, totalCount, err := c.searchPage(filters, page)
if err != nil {
return nil, fmt.Errorf("failed to search page %d: %w", page, err)
}
allJobs = append(allJobs, jobs...)
// Check if we've fetched all jobs or reached max pages
if len(allJobs) >= totalCount || page >= filters.MaxPages || len(jobs) == 0 {
break
}
page++
// Rate limiting - be nice to the API
time.Sleep(500 * time.Millisecond)
}
return allJobs, nil
}
// searchPage performs a single page search using the correct API format
func (c *Client) searchPage(filters SearchFilters, page int) ([]Job, int, error) {
reqBody := SearchRequest{
Size: PageSize,
Page: page - 1, // API uses 0-based indexing
SearchState: SearchState{
SearchQuery: filters.Query,
SortBy: "date",
},
}
jsonBody, err := json.Marshal(reqBody)
if err != nil {
return nil, 0, fmt.Errorf("failed to marshal request: %w", err)
}
log.Printf("Sending request to %s with body: %s", SearchURL, string(jsonBody))
req, err := http.NewRequest("POST", SearchURL, bytes.NewBuffer(jsonBody))
if err != nil {
return nil, 0, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", UserAgent)
req.Header.Set("Accept", "application/json")
req.Header.Set("Origin", BaseURL)
req.Header.Set("Referer", BaseURL+"/")
resp, err := c.httpClient.Do(req)
if err != nil {
return nil, 0, fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, 0, fmt.Errorf("failed to read response: %w", err)
}
if resp.StatusCode != http.StatusOK {
return nil, 0, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body))
}
var searchResp SearchResponse
if err := json.Unmarshal(body, &searchResp); err != nil {
return nil, 0, fmt.Errorf("failed to unmarshal response: %w", err)
}
// Convert JobResult to Job
var jobs []Job
for _, result := range searchResp.Results {
job := Job{
ID: result.ID,
ApplyURL: result.ApplyURL,
Title: result.JobInformation.Title,
Description: result.JobInformation.Description,
Company: result.ProcessedJobData.CompanyName,
Location: result.ProcessedJobData.FormattedWorkplaceLocation,
Remote: result.ProcessedJobData.WorkplaceType == "remote",
DescriptionClean: result.JobInformation.Description,
}
// Parse salary if available
if result.ProcessedJobData.IsCompensationTransparent &&
result.ProcessedJobData.YearlyMinCompensation != nil &&
result.ProcessedJobData.YearlyMaxCompensation != nil {
job.Salary = fmt.Sprintf("$%.0f - $%.0f",
*result.ProcessedJobData.YearlyMinCompensation,
*result.ProcessedJobData.YearlyMaxCompensation)
}
// Parse posted date
if result.ProcessedJobData.EstimatedPublishDateMillis != "" {
// Try to parse as milliseconds timestamp
var millis int64
if _, err := fmt.Sscanf(result.ProcessedJobData.EstimatedPublishDateMillis, "%d", &millis); err == nil {
job.PostedAt = time.UnixMilli(millis)
}
}
jobs = append(jobs, job)
}
return jobs, searchResp.Total, nil
}

583
services/scraper-hiringcafe/browser/browser.go Normal file
View file

@ -0,0 +1,583 @@
package browser
import (
"context"
"encoding/json"
"fmt"
"log"
"math/rand"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
)
// Config holds browser configuration options
type Config struct {
Headless bool
Timeout time.Duration
UserDataDir string
DebugPort int
UseRealChrome bool // Connect to real Chrome instead of launching automated one
ConnectToExisting bool // Connect to already running Chrome (user must start it manually)
}
// DefaultConfig returns default browser configuration
func DefaultConfig() Config {
return Config{
Headless: false,
Timeout: 10 * time.Minute,
UserDataDir: filepath.Join(os.TempDir(), "hiringcafe-chrome-profile"),
DebugPort: 9224, // Use different port to avoid conflicts
UseRealChrome: true,
ConnectToExisting: false,
}
}
// Browser wraps chromedp browser context with anti-detection features
type Browser struct {
allocCtx context.Context
allocCancel context.CancelFunc
ctx context.Context
cancel context.CancelFunc
config Config
chromeCmd *exec.Cmd
}
// chromeResponse holds the Chrome DevTools Protocol response
type chromeResponse struct {
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
}
// ErrCaptchaDetected is returned when bot detection/CAPTCHA is detected
var ErrCaptchaDetected = fmt.Errorf("captcha or bot detection detected - terminating")
// New creates a new browser instance with anti-detection measures
func New(parentCtx context.Context, config Config) (*Browser, error) {
// Always use the automated real Chrome approach now
return newRealChromeAutomated(parentCtx, config)
}
// newRealChromeAutomated launches Chrome like a normal user would and connects to it
func newRealChromeAutomated(parentCtx context.Context, config Config) (*Browser, error) {
b := &Browser{config: config}
// Check if Chrome is already running on the debug port
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", config.DebugPort), time.Second)
if err == nil {
conn.Close()
log.Printf("Chrome already running on port %d, connecting to it...", config.DebugPort)
} else {
// Launch Chrome with remote debugging
if err := b.launchChromeClean(); err != nil {
return nil, fmt.Errorf("failed to launch Chrome: %w", err)
}
// Wait for Chrome to be ready
if err := b.waitForChrome(); err != nil {
b.Close()
return nil, fmt.Errorf("chrome not ready: %w", err)
}
}
// Get WebSocket URL
wsURL, err := b.getWebSocketURL()
if err != nil {
b.Close()
return nil, fmt.Errorf("failed to get WebSocket URL: %w", err)
}
log.Printf("Connected to Chrome at %s", wsURL)
// Connect to Chrome via WebSocket
allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL)
ctx, cancel := chromedp.NewContext(allocCtx)
// Set timeout
ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)
b.allocCtx = allocCtx
b.allocCancel = allocCancel
b.ctx = ctx
b.cancel = func() {
timeoutCancel()
cancel()
}
return b, nil
}
// launchChromeClean launches Chrome as cleanly as possible (like a normal user)
func (b *Browser) launchChromeClean() error {
chromePath := findChromePath()
if chromePath == "" {
return fmt.Errorf("chrome not found")
}
// Ensure user data directory exists
if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil {
return fmt.Errorf("failed to create user data dir: %w", err)
}
// Minimal args - just what's needed for remote debugging
// Avoid any automation-related flags that might be detected
args := []string{
fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort),
fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir),
"--remote-allow-origins=*",
"--no-first-run",
"--no-default-browser-check",
"--start-maximized",
}
log.Printf("Launching Chrome: %s", chromePath)
b.chromeCmd = exec.Command(chromePath, args...)
b.chromeCmd.Stdout = nil
b.chromeCmd.Stderr = nil
if err := b.chromeCmd.Start(); err != nil {
return fmt.Errorf("failed to start Chrome: %w", err)
}
log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort)
return nil
}
// newRealChrome launches a real Chrome browser and connects via DevTools Protocol
func newRealChrome(parentCtx context.Context, config Config) (*Browser, error) {
b := &Browser{config: config}
// Launch Chrome with remote debugging
if err := b.launchChrome(); err != nil {
return nil, fmt.Errorf("failed to launch Chrome: %w", err)
}
// Wait for Chrome to be ready
if err := b.waitForChrome(); err != nil {
b.Close()
return nil, fmt.Errorf("Chrome not ready: %w", err)
}
// Get WebSocket URL
wsURL, err := b.getWebSocketURL()
if err != nil {
b.Close()
return nil, fmt.Errorf("failed to get WebSocket URL: %w", err)
}
log.Printf("Connecting to Chrome at %s", wsURL)
// Connect to Chrome via WebSocket
allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL)
ctx, cancel := chromedp.NewContext(allocCtx)
// Set timeout
ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)
b.allocCtx = allocCtx
b.allocCancel = allocCancel
b.ctx = ctx
b.cancel = func() {
timeoutCancel()
cancel()
}
// Inject anti-detection scripts
if err := b.injectAntiDetection(); err != nil {
log.Printf("Warning: could not inject anti-detection: %v", err)
}
return b, nil
}
// launchChrome launches Chrome with remote debugging enabled
func (b *Browser) launchChrome() error {
chromePath := findChromePath()
if chromePath == "" {
return fmt.Errorf("Chrome not found")
}
// Ensure user data directory exists
if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil {
return fmt.Errorf("failed to create user data dir: %w", err)
}
args := []string{
fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort),
fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir),
"--remote-allow-origins=*",
"--no-first-run",
"--no-default-browser-check",
"--disable-background-networking",
"--disable-client-side-phishing-detection",
"--disable-default-apps",
"--disable-hang-monitor",
"--disable-popup-blocking",
"--disable-prompt-on-repost",
"--disable-sync",
"--disable-translate",
"--metrics-recording-only",
"--safebrowsing-disable-auto-update",
"--password-store=basic",
"--use-mock-keychain",
}
log.Printf("Launching Chrome: %s", chromePath)
b.chromeCmd = exec.Command(chromePath, args...)
b.chromeCmd.Stdout = nil
b.chromeCmd.Stderr = nil
if err := b.chromeCmd.Start(); err != nil {
return fmt.Errorf("failed to start Chrome: %w", err)
}
log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort)
return nil
}
// waitForChrome waits for Chrome to be ready to accept connections
func (b *Browser) waitForChrome() error {
timeout := time.After(30 * time.Second)
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-timeout:
return fmt.Errorf("timeout waiting for Chrome to start")
case <-ticker.C:
conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", b.config.DebugPort), time.Second)
if err == nil {
conn.Close()
time.Sleep(500 * time.Millisecond) // Give Chrome a moment to fully initialize
return nil
}
}
}
}
// getWebSocketURL gets the WebSocket debugger URL from Chrome
func (b *Browser) getWebSocketURL() (string, error) {
resp, err := http.Get(fmt.Sprintf("http://localhost:%d/json/version", b.config.DebugPort))
if err != nil {
return "", err
}
defer resp.Body.Close()
var result chromeResponse
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", err
}
return result.WebSocketDebuggerURL, nil
}
// findChromePath finds the Chrome executable path
func findChromePath() string {
var paths []string
switch runtime.GOOS {
case "darwin":
paths = []string{
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
}
case "linux":
paths = []string{
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
}
case "windows":
paths = []string{
os.Getenv("LOCALAPPDATA") + "\\Google\\Chrome\\Application\\chrome.exe",
os.Getenv("PROGRAMFILES") + "\\Google\\Chrome\\Application\\chrome.exe",
os.Getenv("PROGRAMFILES(X86)") + "\\Google\\Chrome\\Application\\chrome.exe",
}
}
for _, p := range paths {
if _, err := os.Stat(p); err == nil {
return p
}
}
return ""
}
// newAutomatedChrome creates a browser using chromedp's built-in launcher (fallback)
func newAutomatedChrome(parentCtx context.Context, config Config) (*Browser, error) {
opts := []chromedp.ExecAllocatorOption{
chromedp.NoFirstRun,
chromedp.NoDefaultBrowserCheck,
chromedp.DisableGPU,
chromedp.NoSandbox,
chromedp.Flag("disable-blink-features", "AutomationControlled"),
chromedp.Flag("disable-infobars", true),
chromedp.Flag("disable-dev-shm-usage", true),
chromedp.Flag("window-size", "1920,1080"),
chromedp.UserAgent(getRandomUserAgent()),
}
if config.Headless {
opts = append(opts, chromedp.Flag("headless", "new"))
}
if config.UserDataDir != "" {
opts = append(opts, chromedp.UserDataDir(config.UserDataDir))
}
allocCtx, allocCancel := chromedp.NewExecAllocator(parentCtx, opts...)
ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)
b := &Browser{
allocCtx: allocCtx,
allocCancel: allocCancel,
ctx: ctx,
cancel: func() {
timeoutCancel()
cancel()
},
config: config,
}
if err := b.injectAntiDetection(); err != nil {
b.Close()
return nil, fmt.Errorf("failed to inject anti-detection: %w", err)
}
return b, nil
}
// Context returns the browser context
func (b *Browser) Context() context.Context {
return b.ctx
}
// Close closes the browser and kills Chrome process if we launched it
func (b *Browser) Close() {
if b.cancel != nil {
b.cancel()
}
if b.allocCancel != nil {
b.allocCancel()
}
// Kill Chrome process if we launched it
if b.chromeCmd != nil && b.chromeCmd.Process != nil {
log.Println("Killing Chrome process...")
b.chromeCmd.Process.Kill()
}
}
// injectAntiDetection injects JavaScript to bypass bot detection
func (b *Browser) injectAntiDetection() error {
// JavaScript to override navigator properties and hide automation
antiDetectionScript := `
// Override webdriver property
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
});
// Override plugins
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5],
});
// Override languages
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en'],
});
// Override platform
Object.defineProperty(navigator, 'platform', {
get: () => 'MacIntel',
});
// Override hardware concurrency
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8,
});
// Override device memory
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8,
});
// Override permissions query
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
// Override chrome runtime
window.chrome = {
runtime: {},
};
// Override console.debug to prevent detection
window.console.debug = () => {};
// Add mouse movement simulation capability
window.simulateMouseMovement = function() {
const event = new MouseEvent('mousemove', {
bubbles: true,
cancelable: true,
clientX: Math.random() * window.innerWidth,
clientY: Math.random() * window.innerHeight
});
document.dispatchEvent(event);
};
`
return chromedp.Run(b.ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
_, err := page.AddScriptToEvaluateOnNewDocument(antiDetectionScript).Do(ctx)
return err
}),
)
}
// Navigate navigates to a URL and checks for CAPTCHA/bot detection
func (b *Browser) Navigate(url string) error {
if err := chromedp.Run(b.ctx,
chromedp.Navigate(url),
chromedp.WaitVisible("body", chromedp.ByQuery),
chromedp.Sleep(2*time.Second),
); err != nil {
return err
}
// Check for CAPTCHA/bot detection
if detected, err := b.CheckForCaptcha(); err != nil {
return err
} else if detected {
return ErrCaptchaDetected
}
return nil
}
// CheckForCaptcha checks if the page shows CAPTCHA or bot detection
func (b *Browser) CheckForCaptcha() (bool, error) {
var pageContent string
var pageTitle string
if err := chromedp.Run(b.ctx,
chromedp.Title(&pageTitle),
chromedp.Evaluate(`document.body.innerText`, &pageContent),
); err != nil {
return false, err
}
// Check for common CAPTCHA/bot detection indicators
captchaIndicators := []string{
"Failed to verify your browser",
"Vercel Security Checkpoint",
"Please verify you are a human",
"Access denied",
"Checking your browser",
"Please complete the security check",
"captcha",
"CAPTCHA",
"robot",
"bot detection",
"security challenge",
"Code 10",
"Code 21",
"Code 22",
}
contentLower := strings.ToLower(pageContent)
titleLower := strings.ToLower(pageTitle)
for _, indicator := range captchaIndicators {
indicatorLower := strings.ToLower(indicator)
if strings.Contains(contentLower, indicatorLower) || strings.Contains(titleLower, indicatorLower) {
log.Printf("CAPTCHA/Bot detection detected: found '%s'", indicator)
return true, nil
}
}
return false, nil
}
// WaitAndClick waits for an element and clicks it with human-like delay
func (b *Browser) WaitAndClick(selector string) error {
RandomDelay(500, 1500)
return chromedp.Run(b.ctx,
chromedp.WaitVisible(selector, chromedp.ByQuery),
chromedp.Click(selector, chromedp.ByQuery),
)
}
// Type types text with human-like delays between keystrokes
func (b *Browser) Type(selector, text string) error {
RandomDelay(300, 800)
return chromedp.Run(b.ctx,
chromedp.WaitVisible(selector, chromedp.ByQuery),
chromedp.Click(selector, chromedp.ByQuery),
chromedp.Sleep(200*time.Millisecond),
chromedp.SendKeys(selector, text, chromedp.ByQuery),
)
}
// Evaluate evaluates JavaScript and returns the result
func (b *Browser) Evaluate(script string, result interface{}) error {
return chromedp.Run(b.ctx,
chromedp.Evaluate(script, result),
)
}
// Sleep pauses execution for the specified duration
func (b *Browser) Sleep(d time.Duration) error {
return chromedp.Run(b.ctx, chromedp.Sleep(d))
}
// Nodes returns nodes matching the selector
func (b *Browser) Nodes(selector string) ([]*cdp.Node, error) {
var nodes []*cdp.Node
err := chromedp.Run(b.ctx,
chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
)
return nodes, err
}
// SimulateHumanBehavior performs random mouse movements and scrolls
func (b *Browser) SimulateHumanBehavior() error {
return chromedp.Run(b.ctx,
chromedp.Evaluate(`window.simulateMouseMovement()`, nil),
chromedp.Sleep(time.Duration(rand.Intn(500)+200)*time.Millisecond),
chromedp.Evaluate(`window.scrollBy(0, Math.random() * 100)`, nil),
)
}
// RandomDelay waits for a random duration between min and max milliseconds
func RandomDelay(minMs, maxMs int) {
delay := time.Duration(rand.Intn(maxMs-minMs)+minMs) * time.Millisecond
time.Sleep(delay)
}
// getRandomUserAgent returns a random modern user agent
func getRandomUserAgent() string {
userAgents := []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
}
return userAgents[rand.Intn(len(userAgents))]
}

30
services/scraper-hiringcafe/go.mod
View file

@ -1,3 +1,31 @@
module scraper-hiringcafe module github.com/jobs-scraper/services/scraper-hiringcafe
go 1.24.0 go 1.24.0
require (
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.2
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0
github.com/joho/godotenv v1.5.1
)
require (
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/lib/pq v1.10.9 // indirect
golang.org/x/sys v0.36.0 // indirect
)
replace (
github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
github.com/jobs-scraper/libs/repo => ../../libs/repo
)

88
services/scraper-hiringcafe/go.sum Normal file
View file

@ -0,0 +1,88 @@
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

69
services/scraper-hiringcafe/main.go
View file

@ -1,3 +1,70 @@
package main package main
func main() {} import (
"context"
"log"
"os"
"os/signal"
"syscall"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper-hiringcafe/models"
"github.com/jobs-scraper/services/scraper-hiringcafe/scraper"
"github.com/joho/godotenv"
)
func main() {
if err := godotenv.Load(".local.env"); err != nil {
log.Println("No .local.env file found, trying .env")
if err := godotenv.Load(); err != nil {
log.Println("No .env file found, using system environment variables")
}
}
// Setup database connection
dbConfig := infrastructure.LoadConfigFromEnv()
db, err := infrastructure.NewConnection(dbConfig)
if err != nil {
log.Fatalf("Error connecting to db: %v", err)
}
defer db.Close()
if err := db.Ping(); err != nil {
log.Fatalf("Error pinging db: %v", err)
}
log.Println("Successfully connected to db")
// Initialize repositories
jobRepo := repo.NewJobRepository(db)
jobDescRepo := repo.NewJobDescriptionRepository(db)
// Example filters - these will be configurable later
filters := models.SearchFilters{
Keywords: "software engineer",
Location: "",
Remote: true,
}
// Create scraper and run
s := scraper.New(jobRepo, jobDescRepo)
// Handle graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
go func() {
<-sigChan
log.Println("Received shutdown signal, cleaning up...")
cancel()
}()
if err := s.Scrape(ctx, filters); err != nil {
log.Printf("Error scraping: %v", err)
}
log.Println("Scraper finished")
}

22
services/scraper-hiringcafe/models/models.go Normal file
View file

@ -0,0 +1,22 @@
package models
// SearchFilters holds the filter configuration for hiring.cafe
type SearchFilters struct {
Keywords string
Location string
Remote bool
JobType string // full-time, part-time, contract, internship
DatePosted string // 24h, 7d, 30d
}
// ScrapedJob holds the raw scraped job data before conversion to domain.Job
type ScrapedJob struct {
Title string
Company string
CompanyLink string
Location string
JobLink string
PostedTime string
Description string
Criteria map[string]string
}

BIN
services/scraper-hiringcafe/scraper-hiringcafe Executable file
View file

Binary file not shown.

570
services/scraper-hiringcafe/scraper/scraper.go Normal file
View file

@ -0,0 +1,570 @@
package scraper
import (
"context"
"fmt"
"log"
"strings"
"time"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper-hiringcafe/api"
"github.com/jobs-scraper/services/scraper-hiringcafe/browser"
"github.com/jobs-scraper/services/scraper-hiringcafe/models"
)
// HiringCafeScraper handles scraping jobs from hiring.cafe
type HiringCafeScraper struct {
jobRepo *repo.JobRepository
jobDescRepo *repo.JobDescriptionRepository
apiClient *api.Client
browser *browser.Browser
}
// New creates a new scraper instance
func New(jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository) *HiringCafeScraper {
return &HiringCafeScraper{
jobRepo: jobRepo,
jobDescRepo: jobDescRepo,
apiClient: api.NewClient(),
}
}
// Scrape performs the main scraping operation using the API
func (s *HiringCafeScraper) Scrape(ctx context.Context, filters models.SearchFilters) error {
log.Println("Starting hiring.cafe scraper using API...")
// Convert models.SearchFilters to api.SearchFilters
apiFilters := api.SearchFilters{
Query: filters.Keywords,
Country: "United States",
Remote: filters.Remote,
Hybrid: true,
Onsite: true,
FullTime: true,
PartTime: false,
Contract: false,
Internship: false,
EntryLevel: true,
MidLevel: true,
SeniorLevel: false,
DateRangeDays: 30,
MaxPages: 10,
}
if filters.Location != "" {
apiFilters.Country = filters.Location
}
log.Printf("Searching for: %s in %s", apiFilters.Query, apiFilters.Country)
// Fetch jobs from API
apiJobs, err := s.apiClient.Search(apiFilters)
if err != nil {
return fmt.Errorf("failed to fetch jobs from API: %w", err)
}
log.Printf("Found %d jobs from API", len(apiJobs))
if len(apiJobs) == 0 {
log.Println("No jobs found")
return nil
}
// Convert API jobs to scraped jobs
var scrapedJobs []models.ScrapedJob
for _, job := range apiJobs {
scrapedJob := models.ScrapedJob{
Title: job.Title,
Company: job.Company,
CompanyLink: job.CompanyURL,
Location: job.Location,
JobLink: job.ApplyURL,
PostedTime: job.PostedAt.Format("2006-01-02"),
Description: job.DescriptionClean,
Criteria: make(map[string]string),
}
if job.Salary != "" {
scrapedJob.Criteria["Salary"] = job.Salary
}
if job.Remote {
scrapedJob.Criteria["Remote"] = "Yes"
}
scrapedJobs = append(scrapedJobs, scrapedJob)
}
// Save jobs to database
if err := s.saveJobs(scrapedJobs); err != nil {
return fmt.Errorf("failed to save jobs: %w", err)
}
log.Printf("Successfully saved %d jobs to database", len(scrapedJobs))
return nil
}
// ScrapeWithBrowser performs scraping using browser automation (fallback if API fails)
func (s *HiringCafeScraper) ScrapeWithBrowser(ctx context.Context, filters models.SearchFilters) error {
// Create browser with anti-detection
config := browser.DefaultConfig()
config.Headless = false
b, err := browser.New(ctx, config)
if err != nil {
return fmt.Errorf("failed to create browser: %w", err)
}
defer b.Close()
s.browser = b
log.Println("Starting browser and navigating to hiring.cafe...")
// Navigate to hiring.cafe
if err := b.Navigate("https://hiring.cafe/"); err != nil {
if err == browser.ErrCaptchaDetected {
log.Println("CAPTCHA/Bot detection encountered - terminating scraper")
return err
}
return fmt.Errorf("failed to navigate to hiring.cafe: %w", err)
}
log.Println("Page loaded successfully, no CAPTCHA detected")
// Apply filters
if err := s.applyFilters(filters); err != nil {
return fmt.Errorf("failed to apply filters: %w", err)
}
log.Println("Filters applied, extracting job listings...")
// Extract job listings
jobs, err := s.extractJobListings()
if err != nil {
return fmt.Errorf("failed to extract job listings: %w", err)
}
log.Printf("Found %d jobs", len(jobs))
// Save jobs to database
if err := s.saveJobs(jobs); err != nil {
return fmt.Errorf("failed to save jobs: %w", err)
}
log.Printf("Successfully saved %d jobs to database", len(jobs))
return nil
}
// applyFilters applies search filters on hiring.cafe
func (s *HiringCafeScraper) applyFilters(filters models.SearchFilters) error {
if err := s.browser.Sleep(2 * time.Second); err != nil {
return err
}
// Apply keyword search if provided
if filters.Keywords != "" {
log.Printf("Searching for: %s", filters.Keywords)
// Try multiple possible search input selectors
searchSelectors := []string{
`input[type="search"]`,
`input[placeholder*="search" i]`,
`input[placeholder*="job" i]`,
`input[type="text"]`,
`input[name="q"]`,
`input[name="search"]`,
}
var searchApplied bool
for _, selector := range searchSelectors {
if err := s.browser.Type(selector, filters.Keywords); err == nil {
searchApplied = true
log.Printf("Applied search using selector: %s", selector)
break
}
}
if searchApplied {
browser.RandomDelay(500, 1000)
// Press Enter to search
var result interface{}
s.browser.Evaluate(`
const input = document.querySelector('input[type="search"], input[placeholder*="search" i], input[type="text"]');
if (input) {
input.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
input.form?.submit();
}
`, &result)
s.browser.Sleep(3 * time.Second)
} else {
log.Println("Warning: could not find search input")
}
}
// Apply remote filter if enabled
if filters.Remote {
log.Println("Applying remote filter...")
remoteSelectors := []string{
`[data-filter="remote"]`,
`button:contains("Remote")`,
`label:contains("Remote")`,
`input[value="remote"]`,
`[class*="remote"]`,
`a[href*="remote"]`,
}
for _, selector := range remoteSelectors {
if err := s.browser.WaitAndClick(selector); err == nil {
log.Printf("Applied remote filter using selector: %s", selector)
s.browser.Sleep(2 * time.Second)
break
}
}
}
// Wait for results to load
return s.browser.Sleep(3 * time.Second)
}
// extractJobListings extracts job listings from the current page
func (s *HiringCafeScraper) extractJobListings() ([]models.ScrapedJob, error) {
if err := s.browser.Sleep(2 * time.Second); err != nil {
return nil, err
}
// Simulate scrolling to load more jobs
s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 2)`, nil)
s.browser.Sleep(1 * time.Second)
s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil)
s.browser.Sleep(2 * time.Second)
// Extract job data using JavaScript
var jobData []map[string]string
err := s.browser.Evaluate(`
(function() {
const jobs = [];
// Try multiple selectors for job cards
const selectors = [
'[class*="job-card"]',
'[class*="job-listing"]',
'[class*="JobCard"]',
'[class*="job-item"]',
'article[class*="job"]',
'div[data-job-id]',
'a[href*="/jobs/"]',
'a[href*="/job/"]',
'[class*="listing-card"]',
'[class*="position-card"]',
'li[class*="job"]',
'tr[class*="job"]'
];
let elements = [];
for (const selector of selectors) {
const found = document.querySelectorAll(selector);
if (found.length > 0) {
elements = Array.from(found);
console.log('Found jobs with selector:', selector, found.length);
break;
}
}
// If no specific job cards found, try to find any links that look like job postings
if (elements.length === 0) {
const allLinks = document.querySelectorAll('a[href*="job"], a[href*="position"], a[href*="career"]');
elements = Array.from(allLinks).filter(el => {
const text = el.textContent.toLowerCase();
return text.length > 10 && text.length < 200;
});
}
elements.forEach((el, index) => {
try {
// Try to extract job information
const titleSelectors = ['h1', 'h2', 'h3', 'h4', '[class*="title"]', '[class*="Title"]', '[class*="name"]'];
const companySelectors = ['[class*="company"]', '[class*="Company"]', '[class*="employer"]', '[class*="org"]'];
const locationSelectors = ['[class*="location"]', '[class*="Location"]', '[class*="place"]', '[class*="city"]'];
const timeSelectors = ['[class*="time"]', '[class*="date"]', '[class*="posted"]', 'time', '[class*="ago"]'];
let title = '';
let company = '';
let location = '';
let postedTime = '';
for (const sel of titleSelectors) {
const titleEl = el.querySelector(sel);
if (titleEl && titleEl.textContent.trim()) {
title = titleEl.textContent.trim();
break;
}
}
for (const sel of companySelectors) {
const companyEl = el.querySelector(sel);
if (companyEl && companyEl.textContent.trim()) {
company = companyEl.textContent.trim();
break;
}
}
for (const sel of locationSelectors) {
const locationEl = el.querySelector(sel);
if (locationEl && locationEl.textContent.trim()) {
location = locationEl.textContent.trim();
break;
}
}
for (const sel of timeSelectors) {
const timeEl = el.querySelector(sel);
if (timeEl && timeEl.textContent.trim()) {
postedTime = timeEl.textContent.trim();
break;
}
}
// Get job link
let jobLink = '';
const linkEl = el.tagName === 'A' ? el : el.querySelector('a[href*="job"], a[href*="position"], a');
if (linkEl && linkEl.href) {
jobLink = linkEl.href;
}
// Get company link
let companyLink = '';
const companyLinkEl = el.querySelector('a[href*="company"], a[href*="employer"]');
if (companyLinkEl && companyLinkEl.href) {
companyLink = companyLinkEl.href;
}
// If no title found, use link text
if (!title && linkEl) {
title = linkEl.textContent.trim();
}
const job = {
title: title.substring(0, 500),
company: company.substring(0, 200),
location: location.substring(0, 200),
jobLink: jobLink,
postedTime: postedTime.substring(0, 100),
companyLink: companyLink
};
// Only add if we have at least a title and link
if (job.title && job.jobLink) {
jobs.push(job);
}
} catch (error) {
console.error('Error extracting job ' + index + ':', error);
}
});
// Remove duplicates based on jobLink
const seen = new Set();
return jobs.filter(job => {
if (seen.has(job.jobLink)) return false;
seen.add(job.jobLink);
return true;
});
})()
`, &jobData)
if err != nil {
return nil, fmt.Errorf("failed to extract job data: %w", err)
}
// Convert to ScrapedJob structs
var jobs []models.ScrapedJob
for _, data := range jobData {
job := models.ScrapedJob{
Title: data["title"],
Company: data["company"],
CompanyLink: data["companyLink"],
Location: data["location"],
JobLink: data["jobLink"],
PostedTime: data["postedTime"],
Criteria: make(map[string]string),
}
jobs = append(jobs, job)
}
return jobs, nil
}
// getJobDetails navigates to a job page and extracts detailed information
func (s *HiringCafeScraper) getJobDetails(job models.ScrapedJob) (models.ScrapedJob, error) {
if job.JobLink == "" {
return job, nil
}
// Navigate to job detail page
if err := s.browser.Navigate(job.JobLink); err != nil {
return job, fmt.Errorf("failed to navigate to job page: %w", err)
}
s.browser.Sleep(2 * time.Second)
s.browser.SimulateHumanBehavior()
// Extract job description
var description string
s.browser.Evaluate(`
(function() {
const descSelectors = [
'[class*="description"]',
'[class*="Description"]',
'[class*="job-content"]',
'[class*="job-details"]',
'[class*="details"]',
'[id*="description"]',
'article',
'main',
'.content'
];
for (const selector of descSelectors) {
const el = document.querySelector(selector);
if (el && el.textContent.trim().length > 100) {
return el.textContent.trim().substring(0, 10000);
}
}
// Fallback: get body text
return document.body.innerText.substring(0, 10000);
})()
`, &description)
// Extract criteria
var criteriaData map[string]string
s.browser.Evaluate(`
(function() {
const criteria = {};
// Look for common job criteria patterns
const patterns = [
{key: 'Experience', selectors: ['[class*="experience"]', '[class*="Experience"]']},
{key: 'Education', selectors: ['[class*="education"]', '[class*="Education"]']},
{key: 'Skills', selectors: ['[class*="skills"]', '[class*="Skills"]']},
{key: 'Salary', selectors: ['[class*="salary"]', '[class*="Salary"]', '[class*="compensation"]']},
{key: 'JobType', selectors: ['[class*="job-type"]', '[class*="employment"]', '[class*="type"]']},
];
patterns.forEach(({key, selectors}) => {
for (const selector of selectors) {
const el = document.querySelector(selector);
if (el && el.textContent.trim()) {
criteria[key] = el.textContent.trim().substring(0, 500);
break;
}
}
});
return criteria;
})()
`, &criteriaData)
job.Description = description
if criteriaData != nil {
job.Criteria = criteriaData
}
return job, nil
}
// saveJobs saves the scraped jobs to the database
func (s *HiringCafeScraper) saveJobs(scrapedJobs []models.ScrapedJob) error {
if len(scrapedJobs) == 0 {
return nil
}
var domainJobs []domain.Job
var jobDescriptions []domain.JobDescription
for _, sj := range scrapedJobs {
postTime := parsePostedTime(sj.PostedTime)
job := domain.Job{
Title: sj.Title,
Company: sj.Company,
CompanyLink: sj.CompanyLink,
Location: sj.Location,
JobLink: sj.JobLink,
Provider: domain.HiringCafe,
JobPostTime: postTime,
Status: domain.JobStatusCreated,
}
domainJobs = append(domainJobs, job)
}
// Save jobs
if err := s.jobRepo.SaveJobs(domainJobs); err != nil {
return fmt.Errorf("failed to save jobs: %w", err)
}
// Collect job descriptions
for _, sj := range scrapedJobs {
if sj.Description == "" {
continue
}
jobDesc := domain.JobDescription{
Description: sj.Description,
Criteria: sj.Criteria,
}
jobDescriptions = append(jobDescriptions, jobDesc)
}
log.Printf("Job descriptions collected: %d (saving requires job IDs)", len(jobDescriptions))
return nil
}
// parsePostedTime converts relative time strings to actual dates
func parsePostedTime(timeStr string) *time.Time {
now := time.Now()
timeStr = strings.ToLower(strings.TrimSpace(timeStr))
if timeStr == "" {
return &now
}
// Parse common patterns
if strings.Contains(timeStr, "hour") || strings.HasSuffix(timeStr, "h") {
var hours int
fmt.Sscanf(timeStr, "%d", &hours)
if hours > 0 {
t := now.Add(-time.Duration(hours) * time.Hour)
return &t
}
}
if strings.Contains(timeStr, "day") || strings.HasSuffix(timeStr, "d") {
var days int
fmt.Sscanf(timeStr, "%d", &days)
if days > 0 {
t := now.AddDate(0, 0, -days)
return &t
}
}
if strings.Contains(timeStr, "week") || strings.HasSuffix(timeStr, "w") {
var weeks int
fmt.Sscanf(timeStr, "%d", &weeks)
if weeks > 0 {
t := now.AddDate(0, 0, -weeks*7)
return &t
}
}
if strings.Contains(timeStr, "month") || strings.HasSuffix(timeStr, "m") {
var months int
fmt.Sscanf(timeStr, "%d", &months)
if months > 0 {
t := now.AddDate(0, -months, 0)
return &t
}
}
return &now
}