From cbcf9bdcc7086c2da7325b6388ef9dc5c61b5a8c Mon Sep 17 00:00:00 2001 From: Elshimy Ziad Magdy Taha Date: Wed, 17 Dec 2025 17:42:42 +0500 Subject: [PATCH] refactor --- go.work | 1 + internal/pkg/browser/browser.go | 241 ++++++++ internal/pkg/browser/launch-chrome.go | 103 ++++ internal/pkg/domain/job.go | 1 + internal/pkg/utils/retryable-http-request.go | 317 +--------- services/scraper-glassdoor/index.ts | 29 +- services/scraper-google/main.go | 98 +-- services/scraper-google/utils/get-html-raw.go | 176 ++---- services/scraper-google/utils/network.go | 45 -- services/scraper-google/work.go | 63 +- services/scraper-hiringcafe/api/client.go | 245 ++++++++ .../scraper-hiringcafe/browser/browser.go | 583 ++++++++++++++++++ services/scraper-hiringcafe/go.mod | 30 +- services/scraper-hiringcafe/go.sum | 88 +++ services/scraper-hiringcafe/main.go | 69 ++- services/scraper-hiringcafe/models/models.go | 22 + .../scraper-hiringcafe/scraper-hiringcafe | Bin 0 -> 9688770 bytes .../scraper-hiringcafe/scraper/scraper.go | 570 +++++++++++++++++ 18 files changed, 2053 insertions(+), 628 deletions(-) create mode 100644 internal/pkg/browser/browser.go create mode 100644 internal/pkg/browser/launch-chrome.go delete mode 100644 services/scraper-google/utils/network.go create mode 100644 services/scraper-hiringcafe/api/client.go create mode 100644 services/scraper-hiringcafe/browser/browser.go create mode 100644 services/scraper-hiringcafe/go.sum create mode 100644 services/scraper-hiringcafe/models/models.go create mode 100755 services/scraper-hiringcafe/scraper-hiringcafe create mode 100644 services/scraper-hiringcafe/scraper/scraper.go diff --git a/go.work b/go.work index 998a0ea..9435af1 100644 --- a/go.work +++ b/go.work @@ -14,5 +14,6 @@ use ( ./libs/server ./services/api ./services/scraper-google + ./services/scraper-hiringcafe ./services/scraper-linkedin ) diff --git a/internal/pkg/browser/browser.go b/internal/pkg/browser/browser.go new file mode 100644 index 0000000..d8f4a62 --- /dev/null +++ b/internal/pkg/browser/browser.go @@ -0,0 +1,241 @@ +package browser + +import ( + "context" + "log" + "math/rand" + "os/exec" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" +) + +type ChromeInstance struct { + WSURL string + Cmd *exec.Cmd +} + +type Browser struct { + Instance *ChromeInstance + allocCtx context.Context + allocCancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc +} + +// NewBrowser launches Chrome and creates a shared allocator. +// Use NewTab() to create separate tabs for concurrent use. +func NewBrowser(ctx context.Context) (*Browser, error) { + wsURL, cmd, err := launchChrome() + + if err != nil { + return nil, err + } + + allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL) + + // Create initial browser context (first tab) + browserCtx, cancel := chromedp.NewContext(allocCtx) + + return &Browser{ + Instance: &ChromeInstance{ + WSURL: wsURL, + Cmd: cmd, + }, + allocCtx: allocCtx, + allocCancel: allocCancel, + ctx: browserCtx, + cancel: cancel, + }, nil +} + +// NewTab creates a new browser tab that can be used concurrently. +// Returns a new Browser instance sharing the same Chrome process. +func (b *Browser) NewTab() (*Browser, error) { + tabCtx, cancel := chromedp.NewContext(b.allocCtx) + + return &Browser{ + Instance: b.Instance, + allocCtx: b.allocCtx, + allocCancel: nil, // Don't close allocator when closing a tab + ctx: tabCtx, + cancel: cancel, + }, nil +} + +func (b *Browser) Close() error { + if b.cancel != nil { + b.cancel() + } + + if b.allocCancel != nil { + b.allocCancel() + } + + // Only kill Chrome process if this is the main browser (has allocCancel), + // not a tab created via NewTab() + if b.allocCancel != nil && b.Instance != nil && b.Instance.Cmd != nil && b.Instance.Cmd.Process != nil { + return b.Instance.Cmd.Process.Kill() + } + + return nil +} + +func (b *Browser) Navigate(url string) error { + chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36") + + err := chromedp.Run(b.ctx, + chromedp.Navigate(url), + ) + + if err != nil { + return err + } + + return nil +} + +func (b *Browser) WaitVisible(selector string) error { + ctx, cancel := context.WithTimeout(b.ctx, 30*time.Second) + defer cancel() + + err := chromedp.Run(ctx, + chromedp.WaitVisible(selector, chromedp.ByQuery), + ) + + if err != nil { + return err + } + + return nil +} + +func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) { + var nodes []*cdp.Node + + err := chromedp.Run(b.ctx, + chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), + ) + + if err != nil { + return nodes, err + } + + return nodes, nil +} + +func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) { + var text string + + err := chromedp.Run(b.ctx, + chromedp.Text(selector, &text, chromedp.ByQuery, chromedp.FromNode(node)), + ) + + if err != nil { + return "", err + } + + return text, nil +} + +func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr string) (string, error) { + var value string + + err := chromedp.Run(b.ctx, + chromedp.AttributeValue(selector, attr, &value, nil, chromedp.ByQuery, chromedp.FromNode(node)), + ) + + if err != nil { + return "", err + } + + return value, nil +} + +func Evaluate[T any](b *Browser, expression string) (T, error) { + var result T + + err := chromedp.Run(b.ctx, + chromedp.Evaluate(expression, &result), + ) + + if err != nil { + return result, err + } + + return result, nil +} + +func (b *Browser) GetCurrentLocation() (string, error) { + var finalURL string + err := chromedp.Run(b.ctx, + chromedp.Location(&finalURL), + ) + + if err != nil { + log.Printf("Failed to get current location: %v", err) + return "", err + } + + return finalURL, nil +} + +func (b *Browser) GetHTML() (string, error) { + var htmlContent string + + err := chromedp.Run(b.ctx, + chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), + ) + if err != nil { + log.Printf("Failed to get HTML content: %v", err) + return "", err + } + + return htmlContent, nil +} + +func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error { + err := chromedp.Run(b.ctx, + chromedp.ActionFunc(func(ctx context.Context) error { + return page.SetLifecycleEventsEnabled(true).Do(ctx) + }), + ) + if err != nil { + return err + } + + ch := make(chan struct{}, 1) + chromedp.ListenTarget(b.ctx, func(ev interface{}) { + if e, ok := ev.(*page.EventLifecycleEvent); ok { + if e.Name == "networkIdle" { + select { + case ch <- struct{}{}: + default: + } + } + } + }) + + select { + case <-ch: + return nil + case <-time.After(timeout): + // Timeout reached, but don't error - page may still be usable + return nil + case <-b.ctx.Done(): + return b.ctx.Err() + } +} + +// RandomDelay waits for a random duration between min and max milliseconds +func RandomDelay(minMs, maxMs int) { + delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond + time.Sleep(delay) +} + +// Sleep waits for the specified duration +func (b *Browser) Sleep(d time.Duration) { + time.Sleep(d) +} diff --git a/internal/pkg/browser/launch-chrome.go b/internal/pkg/browser/launch-chrome.go new file mode 100644 index 0000000..6928548 --- /dev/null +++ b/internal/pkg/browser/launch-chrome.go @@ -0,0 +1,103 @@ +package browser + +import ( + "encoding/json" + "fmt" + "net/http" + "os/exec" + "runtime" + "time" +) + +func launchChrome() (string, *exec.Cmd, error) { + var chromePath string + switch runtime.GOOS { + case "darwin": + chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + case "linux": + chromePath = "google-chrome" + case "windows": + chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" + default: + return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS) + } + + // Kill any existing Chrome debug instances first + exec.Command("pkill", "-f", "remote-debugging-port=9222").Run() + time.Sleep(500 * time.Millisecond) + + cmd := exec.Command(chromePath, + "--remote-debugging-port=9222", + "--no-first-run", + "--no-default-browser-check", + "--user-data-dir=/tmp/chrome-debug-profile", + + // Anti-detection flags + "--disable-blink-features=AutomationControlled", + "--disable-infobars", + "--disable-dev-shm-usage", + "--disable-browser-side-navigation", + "--disable-gpu", + "--no-sandbox", + "--disable-setuid-sandbox", + + // Hide automation indicators + "--disable-extensions", + "--disable-plugins-discovery", + "--disable-bundled-ppapi-flash", + + // Reduce fingerprinting + "--disable-background-networking", + "--disable-sync", + "--disable-translate", + "--metrics-recording-only", + "--safebrowsing-disable-auto-update", + "--disable-client-side-phishing-detection", + "--disable-default-apps", + "--disable-hang-monitor", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + + // Window size to look like a real browser + "--window-size=1920,1080", + "--start-maximized", + ) + + if err := cmd.Start(); err != nil { + return "", nil, fmt.Errorf("failed to start Chrome: %w", err) + } + + // Poll the debug endpoint to get the WebSocket URL + var wsURL string + for range 50 { + time.Sleep(200 * time.Millisecond) + resp, err := http.Get("http://127.0.0.1:9222/json/version") + if err != nil { + continue + } + + var result struct { + WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + resp.Body.Close() + continue + } + resp.Body.Close() + + if result.WebSocketDebuggerURL != "" { + wsURL = result.WebSocketDebuggerURL + break + } + } + + if wsURL == "" { + cmd.Process.Kill() + return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL") + } + + // Wait for Chrome to be fully ready to accept connections + time.Sleep(1 * time.Second) + + return wsURL, cmd, nil +} diff --git a/internal/pkg/domain/job.go b/internal/pkg/domain/job.go index fb84665..43cc2a4 100644 --- a/internal/pkg/domain/job.go +++ b/internal/pkg/domain/job.go @@ -15,6 +15,7 @@ const ( TokyoDev JapanDev Google + HiringCafe ) const ( diff --git a/internal/pkg/utils/retryable-http-request.go b/internal/pkg/utils/retryable-http-request.go index aa0bde1..dfd4098 100644 --- a/internal/pkg/utils/retryable-http-request.go +++ b/internal/pkg/utils/retryable-http-request.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + // "net/url" "time" ) @@ -52,7 +53,6 @@ func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *Retr } func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) { - var lastErr error for attempt := 0; attempt <= s.config.MaxRetries; attempt++ { @@ -108,318 +108,3 @@ func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr) } - -// 5.10.245.81:80 -// 172.67.70.206:80 -// 190.93.245.33:80 -// 185.221.160.176:80 -// 141.101.123.156:80 -// 103.21.244.129:80 -// 103.169.142.59:80 -// 172.67.81.185:80 -// 37.18.73.60:5566 -// 209.38.83.56:1088 -// 89.169.36.109:1080 -// 103.21.244.248:80 -// 172.67.70.92:80 -// 172.67.88.34:80 -// 141.101.120.38:80 -// 172.67.75.255:80 -// 172.67.75.171:80 -// 45.131.4.157:80 -// 141.101.121.208:80 -// 141.101.120.70:80 -// 103.21.244.70:80 -// 103.21.244.140:80 -// 103.21.244.138:80 -// 154.16.146.43:80 -// 128.199.202.122:3128 -// 160.153.0.28:80 -// 185.170.166.31:80 -// 172.67.70.95:80 -// 141.101.123.91:80 -// 141.101.121.81:80 -// 141.101.121.71:80 -// 141.101.120.224:80 -// 192.111.137.35:4145 -// 45.12.31.20:80 -// 103.21.244.150:80 -// 103.21.244.214:80 -// 103.21.244.29:80 -// 195.85.23.88:80 -// 23.227.39.134:80 -// 66.42.224.229:41679 -// 172.67.73.123:80 -// 79.110.200.27:8000 -// 5.182.34.33:80 -// 103.21.244.174:80 -// 172.67.75.203:80 -// 141.101.122.150:80 -// 141.101.120.35:80 -// 175.47.237.95:6128 -// 103.21.244.185:80 -// 141.101.122.86:80 -// 141.101.121.89:80 -// 103.21.244.105:80 -// 103.21.244.234:80 -// 172.67.70.20:80 -// 163.223.172.27:1080 -// 172.67.84.128:80 -// 103.21.244.57:80 -// 31.43.179.191:80 -// 173.245.59.61:80 -// 141.101.120.48:80 -// 185.162.230.19:80 -// 172.67.172.150:80 -// 159.112.235.63:80 -// 172.64.40.184:80 -// 103.21.244.55:80 -// 103.21.244.49:80 -// 172.67.70.228:80 -// 103.21.244.8:80 -// 66.235.200.77:80 -// 45.85.119.147:80 -// 172.67.177.231:80 -// 173.245.49.28:80 -// 110.232.92.49:8080 -// 212.183.88.212:80 -// 108.162.193.73:80 -// 185.238.228.29:80 -// 172.67.127.8:80 -// 164.38.155.86:80 -// 160.153.0.18:80 -// 141.101.122.119:80 -// 103.21.244.186:80 -// 172.67.172.162:80 -// 8.218.39.40:10800 -// 47.237.132.101:60031 -// 222.59.173.105:44193 -// 143.110.190.60:1080 -// 152.53.194.55:32059 -// 115.187.50.40:5678 -// 171.254.219.180:1080 -// 103.165.157.155:8080 -// 45.233.169.57:999 -// 222.59.173.105:45035 -// 202.40.179.18:4145 -// 170.79.181.188:60606 -// 37.186.66.36:3629 -// 104.200.152.30:4145 -// 106.13.58.110:8888 -// 202.40.181.220:31247 -// 43.135.36.240:80 -// 103.82.27.107:10001 -// 192.252.209.158:4145 -// 142.54.236.97:4145 -// 142.54.239.1:4145 -// 184.170.245.148:4145 -// 68.71.247.130:4145 -// 72.195.34.58:4145 -// 172.65.90.2:80 -// 172.65.90.0:80 -// 36.138.53.26:10019 -// 183.215.23.242:9091 -// 45.131.7.34:80 -// 141.101.120.230:80 -// 172.64.78.193:80 -// 141.101.122.9:80 -// 103.21.244.80:80 -// 103.21.244.147:80 -// 103.21.244.106:80 -// 103.21.244.16:80 -// 199.34.229.210:80 -// 141.101.90.98:80 -// 172.67.70.127:80 -// 103.21.244.221:80 -// 172.67.171.203:80 -// 172.67.181.188:80 -// 72.195.101.99:4145 -// 141.101.120.106:80 -// 103.21.244.180:80 -// 103.21.244.102:80 -// 172.67.70.233:80 -// 108.162.193.107:80 -// 45.131.208.112:80 -// 154.194.12.195:80 -// 69.84.182.23:80 -// 141.101.120.128:80 -// 141.101.120.149:80 -// 141.101.120.232:80 -// 141.101.120.6:80 -// 172.67.98.18:80 -// 141.101.113.20:80 -// 208.65.90.21:4145 -// 198.177.254.131:4145 -// 142.54.237.38:4145 -// 192.252.220.89:4145 -// 199.187.210.54:4145 -// 199.102.105.242:4145 -// 199.102.107.145:4145 -// 198.8.84.3:4145 -// 98.182.171.161:4145 -// 192.111.129.150:4145 -// 184.170.248.5:4145 -// 98.188.47.150:4145 -// 125.228.94.232:4145 -// 125.228.94.153:4145 -// 185.162.229.219:80 -// 103.21.244.22:80 -// 172.64.89.97:80 -// 103.21.244.54:80 -// 103.21.244.37:80 -// 141.101.123.225:80 -// 103.21.244.168:80 -// 103.21.244.149:80 -// 103.21.244.100:80 -// 103.21.244.24:80 -// 23.227.39.209:80 -// 172.67.83.15:80 -// 45.131.4.241:80 -// 141.101.120.201:80 -// 172.67.191.237:80 -// 172.67.127.188:80 -// 103.21.244.92:80 -// 103.160.204.22:80 -// 220.197.44.36:3128 -// 39.185.41.193:5911 -// 23.227.39.121:80 -// 139.162.78.109:80 -// 188.114.99.144:80 -// 45.131.5.37:80 -// 45.131.4.250:80 -// 23.227.39.65:80 -// 141.101.121.191:80 -// 172.67.188.16:80 -// 172.67.254.148:80 -// 221.1.104.177:7302 -// 222.59.173.105:44008 -// 103.21.244.83:80 -// 172.64.149.1:80 -// 45.12.30.22:80 -// 222.59.173.105:44027 -// 172.64.149.26:80 -// 103.21.244.46:80 -// 45.12.31.242:80 -// 23.227.38.195:80 -// 172.67.177.162:80 -// 103.160.204.200:80 -// 172.67.70.129:80 -// 141.101.123.245:80 -// 185.162.230.117:80 -// 185.162.230.183:80 -// 69.61.200.104:36181 -// 208.65.90.3:4145 -// 72.223.188.92:4145 -// 192.252.214.17:4145 -// 68.71.242.118:4145 -// 192.252.210.233:4145 -// 107.181.161.81:4145 -// 107.181.168.145:4145 -// 206.220.175.2:4145 -// 72.37.217.3:4145 -// 192.252.208.70:14282 -// 70.166.167.55:57745 -// 192.111.137.37:18762 -// 98.188.47.132:4145 -// 62.99.138.162:80 -// 172.64.90.186:80 -// 172.67.74.57:80 -// 45.131.6.67:80 -// 141.101.122.37:80 -// 103.21.244.134:80 -// 172.64.155.71:80 -// 5.182.34.139:80 -// 45.131.6.31:80 -// 141.101.121.21:80 -// 141.101.120.190:80 -// 103.21.244.161:80 -// 103.21.244.156:80 -// 103.21.244.151:80 -// 172.64.89.0:80 -// 58.216.109.17:800 -// 58.241.88.18:800 -// 36.147.78.166:80 -// 170.244.26.36:8888 -// 170.244.25.52:8888 -// 170.244.26.206:8888 -// 36.138.53.26:10017 -// 201.148.32.162:80 -// 203.19.38.114:1080 -// 31.43.179.204:80 -// 45.12.31.50:80 -// 170.244.27.142:8888 -// 170.244.26.195:8888 -// 170.244.27.58:8888 -// 170.244.27.61:8888 -// 170.244.27.150:8888 -// 47.243.94.125:1080 -// 40.177.65.8:80 -// 47.57.13.107:80 -// 194.158.203.14:80 -// 194.219.134.234:80 -// 103.21.244.51:80 -// 103.21.244.160:80 -// 103.21.244.144:80 -// 213.33.126.130:80 -// 103.21.244.69:80 -// 103.21.244.192:80 -// 103.21.244.133:80 -// 141.193.213.189:80 -// 68.71.254.6:4145 -// 185.221.160.21:80 -// 188.114.99.97:80 -// 63.141.128.94:80 -// 141.193.213.213:80 -// 185.162.228.234:80 -// 213.143.113.82:80 -// 154.194.12.207:80 -// 189.203.181.34:1080 -// 202.144.134.150:5678 -// 190.242.157.215:8080 -// 72.49.49.11:31034 -// 142.54.237.34:4145 -// 68.71.249.153:48606 -// 192.252.209.155:14455 -// 192.252.216.86:4145 -// 142.54.229.249:4145 -// 209.97.150.167:8080 -// 192.252.220.92:17328 -// 98.178.72.21:10919 -// 144.124.228.87:1080 -// 183.240.46.42:80 -// 32.223.6.94:80 -// 192.252.214.20:15864 -// 198.177.252.24:4145 -// 192.252.208.67:14287 -// 198.8.94.170:4145 -// 68.71.241.33:4145 -// 98.181.137.83:4145 -// 45.12.30.139:80 -// 103.21.244.97:80 -// 65.1.148.157:80 -// 199.58.184.97:4145 -// 141.101.120.148:80 -// 185.238.228.77:80 -// 172.67.202.134:80 -// 172.67.182.49:80 -// 45.131.210.112:80 -// 141.101.120.100:80 -// 103.21.244.59:80 -// 103.21.244.189:80 -// 103.21.244.154:80 -// 103.21.244.23:80 -// 173.245.49.40:80 -// 82.200.235.134:38191 -// 43.224.118.89:2626 -// 119.148.47.226:16464 -// 211.230.49.122:3128 -// 62.171.159.232:8888 -// 115.127.112.34:1080 -// 185.191.236.162:3128 -// 41.223.119.156:3128 -// 103.138.123.242:8082 -// 121.169.46.116:1090 -// 185.145.185.218:8080 -// 45.166.93.113:999 -// 163.53.204.178:9813 -// 181.224.226.154:8080 diff --git a/services/scraper-glassdoor/index.ts b/services/scraper-glassdoor/index.ts index 29e87e8..e39b704 100644 --- a/services/scraper-glassdoor/index.ts +++ b/services/scraper-glassdoor/index.ts @@ -107,21 +107,24 @@ async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise { }); await randomDelay(1500, 2500); + await page.goto("https://www.google.com/search?q=site:boards.greenhouse.io+(Software+OR+Backend+OR+Full-Stack)+Engineer", { + waitUntil: ["networkidle2", "domcontentloaded"], + timeout: 60000, + }); + // await browserService.searchJobs(searchQuery.keywords, searchQuery.location); + // const jobs = await browserService.extractJobListings(searchQuery.location); - await browserService.searchJobs(searchQuery.keywords, searchQuery.location); - const jobs = await browserService.extractJobListings(searchQuery.location); + // if (jobs.length > 0) { + // await saveJobs(jobs); + // await saveJobDescriptions(jobs); - if (jobs.length > 0) { - await saveJobs(jobs); - await saveJobDescriptions(jobs); - - console.log( - `saved ${jobs.length} jobs from Glassdoor for query:`, - searchQuery - ); - } else { - console.log("No jobs found for this search query"); - } + // console.log( + // `saved ${jobs.length} jobs from Glassdoor for query:`, + // searchQuery + // ); + // } else { + // console.log("No jobs found for this search query"); + // } await randomDelay(1500, 2500); } catch (error) { diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go index 107f50b..1ded428 100644 --- a/services/scraper-google/main.go +++ b/services/scraper-google/main.go @@ -2,17 +2,13 @@ package main import ( "context" - "encoding/json" - "fmt" "log" - "net/http" "os" - "os/exec" "os/signal" - "runtime" "syscall" "time" + "github.com/jobs-scraper/internal/pkg/browser" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/scraper-google/analyzer" @@ -40,31 +36,35 @@ func main() { ctx, cancel := context.WithCancel(context.Background()) // Launch Chrome once and share across all goroutines - wsURL, chromeCmd, err := launchChrome() + browser, err := browser.NewBrowser(ctx) + if err != nil { log.Fatalf("Failed to launch Chrome: %v", err) } - defer chromeCmd.Process.Kill() - log.Printf("Chrome launched with WebSocket URL: %s", wsURL) + + defer browser.Close() + + log.Printf("Chrome launched with WebSocket URL: %s", browser.Instance.WSURL) sched := sked.New(ctx) googleLinkStream := make(chan domain.GoogleLink, 100) - pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} - // pages := []int{100, 110, 120, 130} + pages := []int{0, 10, 20, 30, 40} + // pages := []int{100} log.Println("Starting scraper with scheduler...") for _, page := range pages { go func() { log.Printf("Running initial scrape for page offset: %d, inital run", page) - Work(ctx, wsURL)(page, googleLinkStream) + Work(ctx, browser)(page, googleLinkStream) }() + } for _, page := range pages { sched.Schedule(func(ctx context.Context) { log.Printf("Running scheduled scrape for page offset: %d", page) - Work(ctx, wsURL)(page, googleLinkStream) + Work(ctx, browser)(page, googleLinkStream) }).Every(time.Hour * 24) } @@ -84,7 +84,7 @@ func main() { continue } - htmlRaw, url, err := utils.GetHTMLRaw(ctx, wsURL, googleLink) + htmlRaw, url, err := utils.GetHTMLRaw(ctx, browser, googleLink) if err != nil { log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err) @@ -143,75 +143,3 @@ func main() { cancel() log.Println("Shutting down scraper...") } - -// launchChrome starts Chrome with remote debugging and returns the WebSocket URL -func launchChrome() (string, *exec.Cmd, error) { - var chromePath string - switch runtime.GOOS { - case "darwin": - chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" - case "linux": - chromePath = "google-chrome" - case "windows": - chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe" - default: - return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS) - } - - // Kill any existing Chrome debug instances first - exec.Command("pkill", "-f", "remote-debugging-port=9222").Run() - time.Sleep(500 * time.Millisecond) - - cmd := exec.Command(chromePath, - "--remote-debugging-port=9222", - "--no-first-run", - "--no-default-browser-check", - "--disable-background-networking", - "--disable-extensions", - "--disable-sync", - "--disable-translate", - "--metrics-recording-only", - "--safebrowsing-disable-auto-update", - "--user-data-dir=/tmp/chrome-debug-profile", - // "--incognito", - "--headless", - ) - - if err := cmd.Start(); err != nil { - return "", nil, fmt.Errorf("failed to start Chrome: %w", err) - } - - // Poll the debug endpoint to get the WebSocket URL - var wsURL string - for i := 0; i < 50; i++ { - time.Sleep(200 * time.Millisecond) - resp, err := http.Get("http://127.0.0.1:9222/json/version") - if err != nil { - continue - } - - var result struct { - WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - resp.Body.Close() - continue - } - resp.Body.Close() - - if result.WebSocketDebuggerURL != "" { - wsURL = result.WebSocketDebuggerURL - break - } - } - - if wsURL == "" { - cmd.Process.Kill() - return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL") - } - - // Wait for Chrome to be fully ready to accept connections - time.Sleep(1 * time.Second) - - return wsURL, cmd, nil -} diff --git a/services/scraper-google/utils/get-html-raw.go b/services/scraper-google/utils/get-html-raw.go index 3993fcc..d05fe67 100644 --- a/services/scraper-google/utils/get-html-raw.go +++ b/services/scraper-google/utils/get-html-raw.go @@ -9,56 +9,72 @@ import ( "time" "github.com/PuerkitoBio/goquery" - "github.com/chromedp/chromedp" + "github.com/jobs-scraper/internal/pkg/browser" "github.com/jobs-scraper/internal/pkg/domain" ) -func GetHTMLRaw(ctx context.Context, wsURL string, googleLink domain.GoogleLink) (string, string, error) { - // Connect to the shared Chrome instance - allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL) - defer allocCancel() - - browserCtx, browserCancel := chromedp.NewContext(allocCtx) - defer browserCancel() - - var htmlContent string - var finalURL string - - err := chromedp.Run(browserCtx, - chromedp.Navigate(googleLink.Link), - chromedp.WaitVisible("body", chromedp.ByQuery), - WaitForNetworkIdle(10*time.Second), // Wait for all network requests to complete - chromedp.Location(&finalURL), - chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), - ) +func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) { + tab, err := b.NewTab() if err != nil { - log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err) + return "", "", fmt.Errorf("failed to create new tab: %w", err) + } + defer tab.Close() + + err = tab.Navigate(googleLink.Link) + + if err != nil { + log.Printf("Failed to navigate for link %s: %v", googleLink.Link, err) + return "", "", err + } + + err = tab.WaitVisible("body") + + if err != nil { + log.Printf("Failed to wait for visible body for link %s: %v", googleLink.Link, err) + return "", "", err + } + + err = tab.WaitForNetworkIdle(10 * time.Second) + + if err != nil { + log.Printf("Failed to wait network idle for link %s: %v", googleLink.Link, err) + return "", "", err + } + + htmlContent, err := tab.GetHTML() + + if err != nil { + log.Printf("Failed to get HTML content for link %s: %v", googleLink.Link, err) return "", "", err } - // Parse the HTML with goquery doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) if err != nil { log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err) return "", "", err } - // Remove unwanted elements doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() log.Printf("Received: %s from %s", googleLink.Title, googleLink.CompanyName) - // Extract text content var content strings.Builder doc.Find("body").Each(func(i int, s *goquery.Selection) { content.WriteString(s.Text()) }) - // Extract hostname from the final URL - hostname, err := extractHostname(finalURL) + finalUrl, err := tab.GetCurrentLocation() + if err != nil { - log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err) + log.Printf("Failed to get current location for link %s: %v", googleLink.Link, err) + return cleanText(content.String()), googleLink.Link, nil + } + + hostname, err := extractHostname(finalUrl) + + if err != nil { + log.Printf("Failed to extract hostname from final URL %s: %v", finalUrl, err) // Fall back to the original link if extraction fails return cleanText(content.String()), googleLink.Link, nil } @@ -78,111 +94,3 @@ func extractHostname(urlStr string) (string, error) { } return parsedURL.Hostname(), nil } - -// ClickSelector clicks on an element using the provided selector string -// Selector format: "css:selector" or "xpath:selector" -func ClickSelector(ctx context.Context, selector string) error { - if selector == "" { - return fmt.Errorf("empty selector") - } - - parts := strings.SplitN(selector, ":", 2) - if len(parts) != 2 { - return fmt.Errorf("invalid selector format: %s", selector) - } - - selectorType := parts[0] - selectorValue := parts[1] - - var err error - switch selectorType { - case "css": - err = chromedp.Run(ctx, - chromedp.WaitVisible(selectorValue, chromedp.ByQuery), - chromedp.Click(selectorValue, chromedp.ByQuery), - ) - case "xpath": - err = chromedp.Run(ctx, - chromedp.WaitVisible(selectorValue, chromedp.BySearch), - chromedp.Click(selectorValue, chromedp.BySearch), - ) - default: - return fmt.Errorf("unknown selector type: %s", selectorType) - } - - if err != nil { - return fmt.Errorf("failed to click selector %s: %w", selector, err) - } - - log.Printf("Successfully clicked on selector: %s", selector) - return nil -} - -// GetHTMLRawWithRetry fetches HTML and can retry with a selector click if needed -func GetHTMLRawWithRetry(ctx context.Context, wsURL string, googleLink domain.GoogleLink, selector string) (string, string, error) { - // Connect to the shared Chrome instance - allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL) - defer allocCancel() - - browserCtx, browserCancel := chromedp.NewContext(allocCtx) - defer browserCancel() - - timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 45*time.Second) - defer timeoutCancel() - - var htmlContent string - var finalURL string - - // Navigate to the page - err := chromedp.Run(timeoutCtx, - chromedp.Navigate(googleLink.Link), - chromedp.WaitVisible("body", chromedp.ByQuery), - WaitForNetworkIdle(5*time.Second), - ) - if err != nil { - return "", "", fmt.Errorf("failed to navigate: %w", err) - } - - // If a selector is provided, click on it - if selector != "" { - if err := ClickSelector(timeoutCtx, selector); err != nil { - log.Printf("Failed to click selector %s: %v", selector, err) - // Continue anyway, we'll get whatever HTML we can - } else { - // Wait for the new page to load after clicking - _ = chromedp.Run(timeoutCtx, - chromedp.WaitVisible("body", chromedp.ByQuery), - WaitForNetworkIdle(5*time.Second), - ) - } - } - - // Get the final HTML - err = chromedp.Run(timeoutCtx, - chromedp.Location(&finalURL), - chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), - ) - if err != nil { - return "", "", fmt.Errorf("failed to get HTML: %w", err) - } - - // Parse and clean HTML - doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) - if err != nil { - return "", "", fmt.Errorf("failed to parse HTML: %w", err) - } - - doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() - - var content strings.Builder - doc.Find("body").Each(func(i int, s *goquery.Selection) { - content.WriteString(s.Text()) - }) - - hostname, err := extractHostname(finalURL) - if err != nil { - return cleanText(content.String()), googleLink.Link, nil - } - - return cleanText(content.String()), hostname, nil -} diff --git a/services/scraper-google/utils/network.go b/services/scraper-google/utils/network.go deleted file mode 100644 index e02d1cd..0000000 --- a/services/scraper-google/utils/network.go +++ /dev/null @@ -1,45 +0,0 @@ -package utils - -import ( - "context" - "time" - - "github.com/chromedp/cdproto/page" - "github.com/chromedp/chromedp" -) - -// WaitForNetworkIdle waits for the network to be idle (no pending requests) -// for the specified duration. This is useful for pages that load content dynamically. -func WaitForNetworkIdle(timeout time.Duration) chromedp.ActionFunc { - return func(ctx context.Context) error { - // Enable lifecycle events - if err := page.SetLifecycleEventsEnabled(true).Do(ctx); err != nil { - return err - } - - ch := make(chan struct{}, 1) - cctx, cancel := context.WithCancel(ctx) - defer cancel() - - chromedp.ListenTarget(cctx, func(ev interface{}) { - if e, ok := ev.(*page.EventLifecycleEvent); ok { - if e.Name == "networkIdle" { - select { - case ch <- struct{}{}: - default: - } - } - } - }) - - select { - case <-ch: - return nil - case <-time.After(timeout): - // Timeout reached, but don't error - page may still be usable - return nil - case <-ctx.Done(): - return ctx.Err() - } - } -} diff --git a/services/scraper-google/work.go b/services/scraper-google/work.go index 4d35a7b..d768655 100644 --- a/services/scraper-google/work.go +++ b/services/scraper-google/work.go @@ -4,47 +4,48 @@ import ( "context" "log" - "github.com/chromedp/cdproto/cdp" - "github.com/chromedp/chromedp" + "github.com/jobs-scraper/internal/pkg/browser" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/services/scraper-google/utils" ) -func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- domain.GoogleLink) { +func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream chan<- domain.GoogleLink) { return func(page int, stream chan<- domain.GoogleLink) { + // Create a new tab for this goroutine to avoid conflicts + tab, err := b.NewTab() + if err != nil { + log.Printf("Failed to create new tab: %v", err) + return + } + defer tab.Close() + + // Random delay before starting to look more human + browser.RandomDelay(1000, 3000) + // query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid" query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` - - // Connect to the shared Chrome instance - allocCtx, allocCancel := chromedp.NewRemoteAllocator(mainCtx, wsURL) - defer allocCancel() - - ctx, cancel := chromedp.NewContext(allocCtx) - defer cancel() - - var nodes []*cdp.Node - - err := chromedp.Run(ctx, - chromedp.Navigate(utils.BuildUrl(query, page)), - ) - - log.Printf("Error in run? %v", err) + // url := utils.BuildUrl(query, page) + err = tab.Navigate("https://linkedin.com") if err != nil { log.Fatal(err) } - err = chromedp.Run(ctx, - chromedp.WaitVisible("#search", chromedp.ByID), - ) + browser.RandomDelay(2000, 2500) + + err = tab.Navigate(utils.BuildUrl(query, page)) if err != nil { log.Fatal(err) } - err = chromedp.Run(ctx, - chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), - ) + err = tab.WaitVisible(`#search`) + + if err != nil { + log.Fatal(err) + } + + nodes, err := tab.GetNodes("[data-snc]") if err != nil { log.Fatal(err) @@ -53,17 +54,13 @@ func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- do for i, node := range nodes { result := domain.GoogleLink{} - err = chromedp.Run(ctx, - chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)), - ) + result.Title, err = tab.GetTextFromNode(node, "h3") if err != nil { log.Printf("Failed to get title for node %d: %v", i, err) } - err = chromedp.Run(ctx, - chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)), - ) + result.Link, err = tab.GetAttributeFromNode(node, "a", "href") if err != nil { log.Printf("Failed to get link for node %d: %v", i, err) @@ -76,9 +73,7 @@ func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- do continue } - err = chromedp.Run(ctx, - chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)), - ) + result.Description, err = tab.GetTextFromNode(node, "div.VwiC3b") if err != nil { log.Printf("Failed to get description for node %d: %v", i, err) @@ -89,3 +84,5 @@ func Work(mainCtx context.Context, wsURL string) func(page int, stream chan<- do } } + +// AIzaSyCDJhGum5VKTy1t-WMx6ccbTZV2r1S2fuE diff --git a/services/scraper-hiringcafe/api/client.go b/services/scraper-hiringcafe/api/client.go new file mode 100644 index 0000000..3bf62da --- /dev/null +++ b/services/scraper-hiringcafe/api/client.go @@ -0,0 +1,245 @@ +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "time" +) + +const ( + BaseURL = "https://hiring.cafe" + SearchURL = BaseURL + "/api/search-jobs" + UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + PageSize = 20 +) + +// Client handles API requests to hiring.cafe +type Client struct { + httpClient *http.Client +} + +// NewClient creates a new API client +func NewClient() *Client { + return &Client{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +// SearchRequest represents the search request payload (based on RSSHub implementation) +type SearchRequest struct { + Size int `json:"size"` + Page int `json:"page"` + SearchState SearchState `json:"searchState"` +} + +// SearchState contains search parameters +type SearchState struct { + SearchQuery string `json:"searchQuery"` + SortBy string `json:"sortBy"` // date, default, compensation_desc, experience_asc +} + +// SearchResponse represents the API response +type SearchResponse struct { + Results []JobResult `json:"results"` + Total int `json:"total"` +} + +// JobResult represents a job result from the API +type JobResult struct { + ID string `json:"id"` + ApplyURL string `json:"apply_url"` + JobInformation JobInformation `json:"job_information"` + ProcessedJobData ProcessedJobData `json:"v5_processed_job_data"` +} + +// JobInformation contains basic job info +type JobInformation struct { + Title string `json:"title"` + Description string `json:"description"` +} + +// ProcessedJobData contains processed job details +type ProcessedJobData struct { + CompanyName string `json:"company_name"` + IsCompensationTransparent bool `json:"is_compensation_transparent"` + YearlyMinCompensation *float64 `json:"yearly_min_compensation"` + YearlyMaxCompensation *float64 `json:"yearly_max_compensation"` + WorkplaceType string `json:"workplace_type"` + RequirementsSummary string `json:"requirements_summary"` + JobCategory string `json:"job_category"` + RoleActivities []string `json:"role_activities"` + FormattedWorkplaceLocation string `json:"formatted_workplace_location"` + EstimatedPublishDateMillis string `json:"estimated_publish_date_millis"` +} + +// Job is a simplified job structure for external use +type Job struct { + ID string + ApplyURL string + Title string + Description string + Company string + CompanyURL string + Location string + Remote bool + Salary string + PostedAt time.Time + DescriptionClean string +} + +// SearchFilters contains user-configurable search filters +type SearchFilters struct { + Query string + Country string + Remote bool + Hybrid bool + Onsite bool + FullTime bool + PartTime bool + Contract bool + Internship bool + EntryLevel bool + MidLevel bool + SeniorLevel bool + DateRangeDays int + MaxPages int +} + +// DefaultFilters returns default search filters +func DefaultFilters() SearchFilters { + return SearchFilters{ + Query: "", + Country: "United States", + Remote: true, + Hybrid: true, + Onsite: true, + FullTime: true, + PartTime: false, + Contract: false, + Internship: false, + EntryLevel: true, + MidLevel: true, + SeniorLevel: false, + DateRangeDays: 30, + MaxPages: 10, + } +} + +// Search performs a job search with the given filters +func (c *Client) Search(filters SearchFilters) ([]Job, error) { + var allJobs []Job + page := 1 + + for { + jobs, totalCount, err := c.searchPage(filters, page) + if err != nil { + return nil, fmt.Errorf("failed to search page %d: %w", page, err) + } + + allJobs = append(allJobs, jobs...) + + // Check if we've fetched all jobs or reached max pages + if len(allJobs) >= totalCount || page >= filters.MaxPages || len(jobs) == 0 { + break + } + + page++ + // Rate limiting - be nice to the API + time.Sleep(500 * time.Millisecond) + } + + return allJobs, nil +} + +// searchPage performs a single page search using the correct API format +func (c *Client) searchPage(filters SearchFilters, page int) ([]Job, int, error) { + reqBody := SearchRequest{ + Size: PageSize, + Page: page - 1, // API uses 0-based indexing + SearchState: SearchState{ + SearchQuery: filters.Query, + SortBy: "date", + }, + } + + jsonBody, err := json.Marshal(reqBody) + if err != nil { + return nil, 0, fmt.Errorf("failed to marshal request: %w", err) + } + + log.Printf("Sending request to %s with body: %s", SearchURL, string(jsonBody)) + + req, err := http.NewRequest("POST", SearchURL, bytes.NewBuffer(jsonBody)) + if err != nil { + return nil, 0, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("Accept", "application/json") + req.Header.Set("Origin", BaseURL) + req.Header.Set("Referer", BaseURL+"/") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, 0, fmt.Errorf("failed to read response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body)) + } + + var searchResp SearchResponse + if err := json.Unmarshal(body, &searchResp); err != nil { + return nil, 0, fmt.Errorf("failed to unmarshal response: %w", err) + } + + // Convert JobResult to Job + var jobs []Job + for _, result := range searchResp.Results { + job := Job{ + ID: result.ID, + ApplyURL: result.ApplyURL, + Title: result.JobInformation.Title, + Description: result.JobInformation.Description, + Company: result.ProcessedJobData.CompanyName, + Location: result.ProcessedJobData.FormattedWorkplaceLocation, + Remote: result.ProcessedJobData.WorkplaceType == "remote", + DescriptionClean: result.JobInformation.Description, + } + + // Parse salary if available + if result.ProcessedJobData.IsCompensationTransparent && + result.ProcessedJobData.YearlyMinCompensation != nil && + result.ProcessedJobData.YearlyMaxCompensation != nil { + job.Salary = fmt.Sprintf("$%.0f - $%.0f", + *result.ProcessedJobData.YearlyMinCompensation, + *result.ProcessedJobData.YearlyMaxCompensation) + } + + // Parse posted date + if result.ProcessedJobData.EstimatedPublishDateMillis != "" { + // Try to parse as milliseconds timestamp + var millis int64 + if _, err := fmt.Sscanf(result.ProcessedJobData.EstimatedPublishDateMillis, "%d", &millis); err == nil { + job.PostedAt = time.UnixMilli(millis) + } + } + + jobs = append(jobs, job) + } + + return jobs, searchResp.Total, nil +} diff --git a/services/scraper-hiringcafe/browser/browser.go b/services/scraper-hiringcafe/browser/browser.go new file mode 100644 index 0000000..4891581 --- /dev/null +++ b/services/scraper-hiringcafe/browser/browser.go @@ -0,0 +1,583 @@ +package browser + +import ( + "context" + "encoding/json" + "fmt" + "log" + "math/rand" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/cdproto/page" + "github.com/chromedp/chromedp" +) + +// Config holds browser configuration options +type Config struct { + Headless bool + Timeout time.Duration + UserDataDir string + DebugPort int + UseRealChrome bool // Connect to real Chrome instead of launching automated one + ConnectToExisting bool // Connect to already running Chrome (user must start it manually) +} + +// DefaultConfig returns default browser configuration +func DefaultConfig() Config { + return Config{ + Headless: false, + Timeout: 10 * time.Minute, + UserDataDir: filepath.Join(os.TempDir(), "hiringcafe-chrome-profile"), + DebugPort: 9224, // Use different port to avoid conflicts + UseRealChrome: true, + ConnectToExisting: false, + } +} + +// Browser wraps chromedp browser context with anti-detection features +type Browser struct { + allocCtx context.Context + allocCancel context.CancelFunc + ctx context.Context + cancel context.CancelFunc + config Config + chromeCmd *exec.Cmd +} + +// chromeResponse holds the Chrome DevTools Protocol response +type chromeResponse struct { + WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` +} + +// ErrCaptchaDetected is returned when bot detection/CAPTCHA is detected +var ErrCaptchaDetected = fmt.Errorf("captcha or bot detection detected - terminating") + +// New creates a new browser instance with anti-detection measures +func New(parentCtx context.Context, config Config) (*Browser, error) { + // Always use the automated real Chrome approach now + return newRealChromeAutomated(parentCtx, config) +} + +// newRealChromeAutomated launches Chrome like a normal user would and connects to it +func newRealChromeAutomated(parentCtx context.Context, config Config) (*Browser, error) { + b := &Browser{config: config} + + // Check if Chrome is already running on the debug port + conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", config.DebugPort), time.Second) + if err == nil { + conn.Close() + log.Printf("Chrome already running on port %d, connecting to it...", config.DebugPort) + } else { + // Launch Chrome with remote debugging + if err := b.launchChromeClean(); err != nil { + return nil, fmt.Errorf("failed to launch Chrome: %w", err) + } + + // Wait for Chrome to be ready + if err := b.waitForChrome(); err != nil { + b.Close() + return nil, fmt.Errorf("chrome not ready: %w", err) + } + } + + // Get WebSocket URL + wsURL, err := b.getWebSocketURL() + if err != nil { + b.Close() + return nil, fmt.Errorf("failed to get WebSocket URL: %w", err) + } + + log.Printf("Connected to Chrome at %s", wsURL) + + // Connect to Chrome via WebSocket + allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL) + ctx, cancel := chromedp.NewContext(allocCtx) + + // Set timeout + ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout) + + b.allocCtx = allocCtx + b.allocCancel = allocCancel + b.ctx = ctx + b.cancel = func() { + timeoutCancel() + cancel() + } + + return b, nil +} + +// launchChromeClean launches Chrome as cleanly as possible (like a normal user) +func (b *Browser) launchChromeClean() error { + chromePath := findChromePath() + if chromePath == "" { + return fmt.Errorf("chrome not found") + } + + // Ensure user data directory exists + if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil { + return fmt.Errorf("failed to create user data dir: %w", err) + } + + // Minimal args - just what's needed for remote debugging + // Avoid any automation-related flags that might be detected + args := []string{ + fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort), + fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir), + "--remote-allow-origins=*", + "--no-first-run", + "--no-default-browser-check", + "--start-maximized", + } + + log.Printf("Launching Chrome: %s", chromePath) + b.chromeCmd = exec.Command(chromePath, args...) + b.chromeCmd.Stdout = nil + b.chromeCmd.Stderr = nil + + if err := b.chromeCmd.Start(); err != nil { + return fmt.Errorf("failed to start Chrome: %w", err) + } + + log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort) + return nil +} + +// newRealChrome launches a real Chrome browser and connects via DevTools Protocol +func newRealChrome(parentCtx context.Context, config Config) (*Browser, error) { + b := &Browser{config: config} + + // Launch Chrome with remote debugging + if err := b.launchChrome(); err != nil { + return nil, fmt.Errorf("failed to launch Chrome: %w", err) + } + + // Wait for Chrome to be ready + if err := b.waitForChrome(); err != nil { + b.Close() + return nil, fmt.Errorf("Chrome not ready: %w", err) + } + + // Get WebSocket URL + wsURL, err := b.getWebSocketURL() + if err != nil { + b.Close() + return nil, fmt.Errorf("failed to get WebSocket URL: %w", err) + } + + log.Printf("Connecting to Chrome at %s", wsURL) + + // Connect to Chrome via WebSocket + allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL) + ctx, cancel := chromedp.NewContext(allocCtx) + + // Set timeout + ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout) + + b.allocCtx = allocCtx + b.allocCancel = allocCancel + b.ctx = ctx + b.cancel = func() { + timeoutCancel() + cancel() + } + + // Inject anti-detection scripts + if err := b.injectAntiDetection(); err != nil { + log.Printf("Warning: could not inject anti-detection: %v", err) + } + + return b, nil +} + +// launchChrome launches Chrome with remote debugging enabled +func (b *Browser) launchChrome() error { + chromePath := findChromePath() + if chromePath == "" { + return fmt.Errorf("Chrome not found") + } + + // Ensure user data directory exists + if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil { + return fmt.Errorf("failed to create user data dir: %w", err) + } + + args := []string{ + fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort), + fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir), + "--remote-allow-origins=*", + "--no-first-run", + "--no-default-browser-check", + "--disable-background-networking", + "--disable-client-side-phishing-detection", + "--disable-default-apps", + "--disable-hang-monitor", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + "--disable-sync", + "--disable-translate", + "--metrics-recording-only", + "--safebrowsing-disable-auto-update", + "--password-store=basic", + "--use-mock-keychain", + } + + log.Printf("Launching Chrome: %s", chromePath) + b.chromeCmd = exec.Command(chromePath, args...) + b.chromeCmd.Stdout = nil + b.chromeCmd.Stderr = nil + + if err := b.chromeCmd.Start(); err != nil { + return fmt.Errorf("failed to start Chrome: %w", err) + } + + log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort) + return nil +} + +// waitForChrome waits for Chrome to be ready to accept connections +func (b *Browser) waitForChrome() error { + timeout := time.After(30 * time.Second) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-timeout: + return fmt.Errorf("timeout waiting for Chrome to start") + case <-ticker.C: + conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", b.config.DebugPort), time.Second) + if err == nil { + conn.Close() + time.Sleep(500 * time.Millisecond) // Give Chrome a moment to fully initialize + return nil + } + } + } +} + +// getWebSocketURL gets the WebSocket debugger URL from Chrome +func (b *Browser) getWebSocketURL() (string, error) { + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/json/version", b.config.DebugPort)) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result chromeResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + + return result.WebSocketDebuggerURL, nil +} + +// findChromePath finds the Chrome executable path +func findChromePath() string { + var paths []string + + switch runtime.GOOS { + case "darwin": + paths = []string{ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + } + case "linux": + paths = []string{ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + } + case "windows": + paths = []string{ + os.Getenv("LOCALAPPDATA") + "\\Google\\Chrome\\Application\\chrome.exe", + os.Getenv("PROGRAMFILES") + "\\Google\\Chrome\\Application\\chrome.exe", + os.Getenv("PROGRAMFILES(X86)") + "\\Google\\Chrome\\Application\\chrome.exe", + } + } + + for _, p := range paths { + if _, err := os.Stat(p); err == nil { + return p + } + } + return "" +} + +// newAutomatedChrome creates a browser using chromedp's built-in launcher (fallback) +func newAutomatedChrome(parentCtx context.Context, config Config) (*Browser, error) { + opts := []chromedp.ExecAllocatorOption{ + chromedp.NoFirstRun, + chromedp.NoDefaultBrowserCheck, + chromedp.DisableGPU, + chromedp.NoSandbox, + chromedp.Flag("disable-blink-features", "AutomationControlled"), + chromedp.Flag("disable-infobars", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.Flag("window-size", "1920,1080"), + chromedp.UserAgent(getRandomUserAgent()), + } + + if config.Headless { + opts = append(opts, chromedp.Flag("headless", "new")) + } + + if config.UserDataDir != "" { + opts = append(opts, chromedp.UserDataDir(config.UserDataDir)) + } + + allocCtx, allocCancel := chromedp.NewExecAllocator(parentCtx, opts...) + ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf)) + ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout) + + b := &Browser{ + allocCtx: allocCtx, + allocCancel: allocCancel, + ctx: ctx, + cancel: func() { + timeoutCancel() + cancel() + }, + config: config, + } + + if err := b.injectAntiDetection(); err != nil { + b.Close() + return nil, fmt.Errorf("failed to inject anti-detection: %w", err) + } + + return b, nil +} + +// Context returns the browser context +func (b *Browser) Context() context.Context { + return b.ctx +} + +// Close closes the browser and kills Chrome process if we launched it +func (b *Browser) Close() { + if b.cancel != nil { + b.cancel() + } + if b.allocCancel != nil { + b.allocCancel() + } + // Kill Chrome process if we launched it + if b.chromeCmd != nil && b.chromeCmd.Process != nil { + log.Println("Killing Chrome process...") + b.chromeCmd.Process.Kill() + } +} + +// injectAntiDetection injects JavaScript to bypass bot detection +func (b *Browser) injectAntiDetection() error { + // JavaScript to override navigator properties and hide automation + antiDetectionScript := ` + // Override webdriver property + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + + // Override plugins + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + + // Override languages + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + }); + + // Override platform + Object.defineProperty(navigator, 'platform', { + get: () => 'MacIntel', + }); + + // Override hardware concurrency + Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8, + }); + + // Override device memory + Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8, + }); + + // Override permissions query + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters) + ); + + // Override chrome runtime + window.chrome = { + runtime: {}, + }; + + // Override console.debug to prevent detection + window.console.debug = () => {}; + + // Add mouse movement simulation capability + window.simulateMouseMovement = function() { + const event = new MouseEvent('mousemove', { + bubbles: true, + cancelable: true, + clientX: Math.random() * window.innerWidth, + clientY: Math.random() * window.innerHeight + }); + document.dispatchEvent(event); + }; + ` + + return chromedp.Run(b.ctx, + chromedp.ActionFunc(func(ctx context.Context) error { + _, err := page.AddScriptToEvaluateOnNewDocument(antiDetectionScript).Do(ctx) + return err + }), + ) +} + +// Navigate navigates to a URL and checks for CAPTCHA/bot detection +func (b *Browser) Navigate(url string) error { + if err := chromedp.Run(b.ctx, + chromedp.Navigate(url), + chromedp.WaitVisible("body", chromedp.ByQuery), + chromedp.Sleep(2*time.Second), + ); err != nil { + return err + } + + // Check for CAPTCHA/bot detection + if detected, err := b.CheckForCaptcha(); err != nil { + return err + } else if detected { + return ErrCaptchaDetected + } + + return nil +} + +// CheckForCaptcha checks if the page shows CAPTCHA or bot detection +func (b *Browser) CheckForCaptcha() (bool, error) { + var pageContent string + var pageTitle string + + if err := chromedp.Run(b.ctx, + chromedp.Title(&pageTitle), + chromedp.Evaluate(`document.body.innerText`, &pageContent), + ); err != nil { + return false, err + } + + // Check for common CAPTCHA/bot detection indicators + captchaIndicators := []string{ + "Failed to verify your browser", + "Vercel Security Checkpoint", + "Please verify you are a human", + "Access denied", + "Checking your browser", + "Please complete the security check", + "captcha", + "CAPTCHA", + "robot", + "bot detection", + "security challenge", + "Code 10", + "Code 21", + "Code 22", + } + + contentLower := strings.ToLower(pageContent) + titleLower := strings.ToLower(pageTitle) + + for _, indicator := range captchaIndicators { + indicatorLower := strings.ToLower(indicator) + if strings.Contains(contentLower, indicatorLower) || strings.Contains(titleLower, indicatorLower) { + log.Printf("CAPTCHA/Bot detection detected: found '%s'", indicator) + return true, nil + } + } + + return false, nil +} + +// WaitAndClick waits for an element and clicks it with human-like delay +func (b *Browser) WaitAndClick(selector string) error { + RandomDelay(500, 1500) + return chromedp.Run(b.ctx, + chromedp.WaitVisible(selector, chromedp.ByQuery), + chromedp.Click(selector, chromedp.ByQuery), + ) +} + +// Type types text with human-like delays between keystrokes +func (b *Browser) Type(selector, text string) error { + RandomDelay(300, 800) + return chromedp.Run(b.ctx, + chromedp.WaitVisible(selector, chromedp.ByQuery), + chromedp.Click(selector, chromedp.ByQuery), + chromedp.Sleep(200*time.Millisecond), + chromedp.SendKeys(selector, text, chromedp.ByQuery), + ) +} + +// Evaluate evaluates JavaScript and returns the result +func (b *Browser) Evaluate(script string, result interface{}) error { + return chromedp.Run(b.ctx, + chromedp.Evaluate(script, result), + ) +} + +// Sleep pauses execution for the specified duration +func (b *Browser) Sleep(d time.Duration) error { + return chromedp.Run(b.ctx, chromedp.Sleep(d)) +} + +// Nodes returns nodes matching the selector +func (b *Browser) Nodes(selector string) ([]*cdp.Node, error) { + var nodes []*cdp.Node + err := chromedp.Run(b.ctx, + chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll), + ) + return nodes, err +} + +// SimulateHumanBehavior performs random mouse movements and scrolls +func (b *Browser) SimulateHumanBehavior() error { + return chromedp.Run(b.ctx, + chromedp.Evaluate(`window.simulateMouseMovement()`, nil), + chromedp.Sleep(time.Duration(rand.Intn(500)+200)*time.Millisecond), + chromedp.Evaluate(`window.scrollBy(0, Math.random() * 100)`, nil), + ) +} + +// RandomDelay waits for a random duration between min and max milliseconds +func RandomDelay(minMs, maxMs int) { + delay := time.Duration(rand.Intn(maxMs-minMs)+minMs) * time.Millisecond + time.Sleep(delay) +} + +// getRandomUserAgent returns a random modern user agent +func getRandomUserAgent() string { + userAgents := []string{ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", + } + return userAgents[rand.Intn(len(userAgents))] +} diff --git a/services/scraper-hiringcafe/go.mod b/services/scraper-hiringcafe/go.mod index 627339c..71f7bf6 100644 --- a/services/scraper-hiringcafe/go.mod +++ b/services/scraper-hiringcafe/go.mod @@ -1,3 +1,31 @@ -module scraper-hiringcafe +module github.com/jobs-scraper/services/scraper-hiringcafe go 1.24.0 + +require ( + github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d + github.com/chromedp/chromedp v0.14.2 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/joho/godotenv v1.5.1 +) + +require ( + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/lib/pq v1.10.9 // indirect + golang.org/x/sys v0.36.0 // indirect +) + +replace ( + github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure + github.com/jobs-scraper/libs/repo => ../../libs/repo +) diff --git a/services/scraper-hiringcafe/go.sum b/services/scraper-hiringcafe/go.sum new file mode 100644 index 0000000..ef02ac7 --- /dev/null +++ b/services/scraper-hiringcafe/go.sum @@ -0,0 +1,88 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU= +github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/scraper-hiringcafe/main.go b/services/scraper-hiringcafe/main.go index 38dd16d..850cff6 100644 --- a/services/scraper-hiringcafe/main.go +++ b/services/scraper-hiringcafe/main.go @@ -1,3 +1,70 @@ package main -func main() {} +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/services/scraper-hiringcafe/models" + "github.com/jobs-scraper/services/scraper-hiringcafe/scraper" + "github.com/joho/godotenv" +) + +func main() { + if err := godotenv.Load(".local.env"); err != nil { + log.Println("No .local.env file found, trying .env") + if err := godotenv.Load(); err != nil { + log.Println("No .env file found, using system environment variables") + } + } + + // Setup database connection + dbConfig := infrastructure.LoadConfigFromEnv() + db, err := infrastructure.NewConnection(dbConfig) + if err != nil { + log.Fatalf("Error connecting to db: %v", err) + } + defer db.Close() + + if err := db.Ping(); err != nil { + log.Fatalf("Error pinging db: %v", err) + } + log.Println("Successfully connected to db") + + // Initialize repositories + jobRepo := repo.NewJobRepository(db) + jobDescRepo := repo.NewJobDescriptionRepository(db) + + // Example filters - these will be configurable later + filters := models.SearchFilters{ + Keywords: "software engineer", + Location: "", + Remote: true, + } + + // Create scraper and run + s := scraper.New(jobRepo, jobDescRepo) + + // Handle graceful shutdown + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) + + go func() { + <-sigChan + log.Println("Received shutdown signal, cleaning up...") + cancel() + }() + + if err := s.Scrape(ctx, filters); err != nil { + log.Printf("Error scraping: %v", err) + } + + log.Println("Scraper finished") +} diff --git a/services/scraper-hiringcafe/models/models.go b/services/scraper-hiringcafe/models/models.go new file mode 100644 index 0000000..ed5eb1c --- /dev/null +++ b/services/scraper-hiringcafe/models/models.go @@ -0,0 +1,22 @@ +package models + +// SearchFilters holds the filter configuration for hiring.cafe +type SearchFilters struct { + Keywords string + Location string + Remote bool + JobType string // full-time, part-time, contract, internship + DatePosted string // 24h, 7d, 30d +} + +// ScrapedJob holds the raw scraped job data before conversion to domain.Job +type ScrapedJob struct { + Title string + Company string + CompanyLink string + Location string + JobLink string + PostedTime string + Description string + Criteria map[string]string +} diff --git a/services/scraper-hiringcafe/scraper-hiringcafe b/services/scraper-hiringcafe/scraper-hiringcafe new file mode 100755 index 0000000000000000000000000000000000000000..6ec530ff18e1851b293a15b7db20627e318f2625 GIT binary patch literal 9688770 zcmeFa3w%`9b??7NGouIKfMZgVfGr6L1Ti*Fi~+@MHF^LBJ2;KGaT2%D6A;D*zmh_X zBLM;}+WTGT10_LkX@!lN8|xU`QMuX_7i^nb8xF9UGF{Xku*L?{A-TW{w7d zoi_LO|M~o@&u27q&e><}wby&Cz4tjk`pui~4KXHQ_{-q)VLm?_WK789@ynQx^0}E$ zW#wl|XH5Izw9nqAv;F_<&HYbZa3nZWX;@h~XWHlI^uKKHk4)-+(ml@NY?x2by;@oM z;M^q-rkY*=ZGhv(hreT!~CvvjK{ol{y_ ze%tKZ9qK_J(XIF3IH##gFYj3N(nqi%0d!BtYhZY9j~PsB{(yHWyx_yqd*|N0V9`RS zd1Yl9ym=)SzwhNa*A2&)hPU9s>Y9tgYhCTYyBXQkpN=mLZ~kI93rwedI)2$v2cC1) z7#&~gud;If;(KfFUgQBxhnMHV``K5nvS4(4Dg4COB`BN&$JcY@Pb}Sropj{GaIC-9 z1r9WiuhI+s39rhUFE6;eZo#5?i!ToEdhZ6{_0tCjO!B{lH*dkc4>;g1z%Mex!EcFs zhmJ2b&dSO~=$4~m_da<41@OM(!t0gs;ZMizPkSuAsrI)}1vDe>bE?>4)d^l4|dc zHwiQyzgYQP3-9`$r|!8FJfGa%QTo~GshcA6ES%(t)*Acs$<4}91Lddckp{(c=PVB zz6XHm_=R7+$Bwu1+YW-^nRod>*`I0nIm4Y_S5x^w?fnbV;ngi$Y~guO>dw1{F$FaZ zo=d-!p7o>OYc9MOs)t$#uYFK>cQ3Bh4e9u;`uDF{{60S2F`%oy^iT>j=hwja-CwVXIxuE5=daff;7+|I!%RK0*6ixvL|2o48jawbI=4f{2Y)kvwmp<^|+%7)Hn@3$bI+EePHq3U%9S0 z{l0oYT+pA>$;4Y1)ZAZv|I&%mrA~jk`2K|tOe|k~|J><|zq|+ox?u65>#U;b%EZsj zz5fBuJTS2|tqt&nU)68d&+T(-zkL6K2bT`qgLf5}`WwkdW0>)`>#DcD{0H-X^tu1I zvG40Y{#$uv#X(1RJz>s`@kdA*d$_Ek%Zs{{y0*cH!Sjmhqvz2 zxqQx*RG9m!jhSQeoVJ4Rwe!Z