jobs-monorepo/services/scraper-hiringcafe/browser/browser.go

package browser

import (
	"context"
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/chromedp/cdproto/cdp"
	"github.com/chromedp/cdproto/page"
	"github.com/chromedp/chromedp"
)

// Config holds browser configuration options
type Config struct {
	Headless          bool
	Timeout           time.Duration
	UserDataDir       string
	DebugPort         int
	UseRealChrome     bool // Connect to real Chrome instead of launching automated one
	ConnectToExisting bool // Connect to already running Chrome (user must start it manually)
}

// DefaultConfig returns default browser configuration
func DefaultConfig() Config {
	return Config{
		Headless:          false,
		Timeout:           10 * time.Minute,
		UserDataDir:       filepath.Join(os.TempDir(), "hiringcafe-chrome-profile"),
		DebugPort:         9224, // Use different port to avoid conflicts
		UseRealChrome:     true,
		ConnectToExisting: false,
	}
}

// Browser wraps chromedp browser context with anti-detection features
type Browser struct {
	allocCtx    context.Context
	allocCancel context.CancelFunc
	ctx         context.Context
	cancel      context.CancelFunc
	config      Config
	chromeCmd   *exec.Cmd
}

// chromeResponse holds the Chrome DevTools Protocol response
type chromeResponse struct {
	WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
}

// ErrCaptchaDetected is returned when bot detection/CAPTCHA is detected
var ErrCaptchaDetected = fmt.Errorf("captcha or bot detection detected - terminating")

// New creates a new browser instance with anti-detection measures
func New(parentCtx context.Context, config Config) (*Browser, error) {
	// Always use the automated real Chrome approach now
	return newRealChromeAutomated(parentCtx, config)
}

// newRealChromeAutomated launches Chrome like a normal user would and connects to it
func newRealChromeAutomated(parentCtx context.Context, config Config) (*Browser, error) {
	b := &Browser{config: config}

	// Check if Chrome is already running on the debug port
	conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", config.DebugPort), time.Second)
	if err == nil {
		conn.Close()
		log.Printf("Chrome already running on port %d, connecting to it...", config.DebugPort)
	} else {
		// Launch Chrome with remote debugging
		if err := b.launchChromeClean(); err != nil {
			return nil, fmt.Errorf("failed to launch Chrome: %w", err)
		}

		// Wait for Chrome to be ready
		if err := b.waitForChrome(); err != nil {
			b.Close()
			return nil, fmt.Errorf("chrome not ready: %w", err)
		}
	}

	// Get WebSocket URL
	wsURL, err := b.getWebSocketURL()
	if err != nil {
		b.Close()
		return nil, fmt.Errorf("failed to get WebSocket URL: %w", err)
	}

	log.Printf("Connected to Chrome at %s", wsURL)

	// Connect to Chrome via WebSocket
	allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL)
	ctx, cancel := chromedp.NewContext(allocCtx)

	// Set timeout
	ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)

	b.allocCtx = allocCtx
	b.allocCancel = allocCancel
	b.ctx = ctx
	b.cancel = func() {
		timeoutCancel()
		cancel()
	}

	return b, nil
}

// launchChromeClean launches Chrome as cleanly as possible (like a normal user)
func (b *Browser) launchChromeClean() error {
	chromePath := findChromePath()
	if chromePath == "" {
		return fmt.Errorf("chrome not found")
	}

	// Ensure user data directory exists
	if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil {
		return fmt.Errorf("failed to create user data dir: %w", err)
	}

	// Minimal args - just what's needed for remote debugging
	// Avoid any automation-related flags that might be detected
	args := []string{
		fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort),
		fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir),
		"--remote-allow-origins=*",
		"--no-first-run",
		"--no-default-browser-check",
		"--start-maximized",
	}

	log.Printf("Launching Chrome: %s", chromePath)
	b.chromeCmd = exec.Command(chromePath, args...)
	b.chromeCmd.Stdout = nil
	b.chromeCmd.Stderr = nil

	if err := b.chromeCmd.Start(); err != nil {
		return fmt.Errorf("failed to start Chrome: %w", err)
	}

	log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort)
	return nil
}

// newRealChrome launches a real Chrome browser and connects via DevTools Protocol
func newRealChrome(parentCtx context.Context, config Config) (*Browser, error) {
	b := &Browser{config: config}

	// Launch Chrome with remote debugging
	if err := b.launchChrome(); err != nil {
		return nil, fmt.Errorf("failed to launch Chrome: %w", err)
	}

	// Wait for Chrome to be ready
	if err := b.waitForChrome(); err != nil {
		b.Close()
		return nil, fmt.Errorf("Chrome not ready: %w", err)
	}

	// Get WebSocket URL
	wsURL, err := b.getWebSocketURL()
	if err != nil {
		b.Close()
		return nil, fmt.Errorf("failed to get WebSocket URL: %w", err)
	}

	log.Printf("Connecting to Chrome at %s", wsURL)

	// Connect to Chrome via WebSocket
	allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL)
	ctx, cancel := chromedp.NewContext(allocCtx)

	// Set timeout
	ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)

	b.allocCtx = allocCtx
	b.allocCancel = allocCancel
	b.ctx = ctx
	b.cancel = func() {
		timeoutCancel()
		cancel()
	}

	// Inject anti-detection scripts
	if err := b.injectAntiDetection(); err != nil {
		log.Printf("Warning: could not inject anti-detection: %v", err)
	}

	return b, nil
}

// launchChrome launches Chrome with remote debugging enabled
func (b *Browser) launchChrome() error {
	chromePath := findChromePath()
	if chromePath == "" {
		return fmt.Errorf("Chrome not found")
	}

	// Ensure user data directory exists
	if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil {
		return fmt.Errorf("failed to create user data dir: %w", err)
	}

	args := []string{
		fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort),
		fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir),
		"--remote-allow-origins=*",
		"--no-first-run",
		"--no-default-browser-check",
		"--disable-background-networking",
		"--disable-client-side-phishing-detection",
		"--disable-default-apps",
		"--disable-hang-monitor",
		"--disable-popup-blocking",
		"--disable-prompt-on-repost",
		"--disable-sync",
		"--disable-translate",
		"--metrics-recording-only",
		"--safebrowsing-disable-auto-update",
		"--password-store=basic",
		"--use-mock-keychain",
	}

	log.Printf("Launching Chrome: %s", chromePath)
	b.chromeCmd = exec.Command(chromePath, args...)
	b.chromeCmd.Stdout = nil
	b.chromeCmd.Stderr = nil

	if err := b.chromeCmd.Start(); err != nil {
		return fmt.Errorf("failed to start Chrome: %w", err)
	}

	log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort)
	return nil
}

// waitForChrome waits for Chrome to be ready to accept connections
func (b *Browser) waitForChrome() error {
	timeout := time.After(30 * time.Second)
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()

	for {
		select {
		case <-timeout:
			return fmt.Errorf("timeout waiting for Chrome to start")
		case <-ticker.C:
			conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", b.config.DebugPort), time.Second)
			if err == nil {
				conn.Close()
				time.Sleep(500 * time.Millisecond) // Give Chrome a moment to fully initialize
				return nil
			}
		}
	}
}

// getWebSocketURL gets the WebSocket debugger URL from Chrome
func (b *Browser) getWebSocketURL() (string, error) {
	resp, err := http.Get(fmt.Sprintf("http://localhost:%d/json/version", b.config.DebugPort))
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	var result chromeResponse
	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
		return "", err
	}

	return result.WebSocketDebuggerURL, nil
}

// findChromePath finds the Chrome executable path
func findChromePath() string {
	var paths []string

	switch runtime.GOOS {
	case "darwin":
		paths = []string{
			"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
			"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
			"/Applications/Chromium.app/Contents/MacOS/Chromium",
		}
	case "linux":
		paths = []string{
			"/usr/bin/google-chrome",
			"/usr/bin/google-chrome-stable",
			"/usr/bin/chromium",
			"/usr/bin/chromium-browser",
		}
	case "windows":
		paths = []string{
			os.Getenv("LOCALAPPDATA") + "\\Google\\Chrome\\Application\\chrome.exe",
			os.Getenv("PROGRAMFILES") + "\\Google\\Chrome\\Application\\chrome.exe",
			os.Getenv("PROGRAMFILES(X86)") + "\\Google\\Chrome\\Application\\chrome.exe",
		}
	}

	for _, p := range paths {
		if _, err := os.Stat(p); err == nil {
			return p
		}
	}
	return ""
}

// newAutomatedChrome creates a browser using chromedp's built-in launcher (fallback)
func newAutomatedChrome(parentCtx context.Context, config Config) (*Browser, error) {
	opts := []chromedp.ExecAllocatorOption{
		chromedp.NoFirstRun,
		chromedp.NoDefaultBrowserCheck,
		chromedp.DisableGPU,
		chromedp.NoSandbox,
		chromedp.Flag("disable-blink-features", "AutomationControlled"),
		chromedp.Flag("disable-infobars", true),
		chromedp.Flag("disable-dev-shm-usage", true),
		chromedp.Flag("window-size", "1920,1080"),
		chromedp.UserAgent(getRandomUserAgent()),
	}

	if config.Headless {
		opts = append(opts, chromedp.Flag("headless", "new"))
	}

	if config.UserDataDir != "" {
		opts = append(opts, chromedp.UserDataDir(config.UserDataDir))
	}

	allocCtx, allocCancel := chromedp.NewExecAllocator(parentCtx, opts...)
	ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
	ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout)

	b := &Browser{
		allocCtx:    allocCtx,
		allocCancel: allocCancel,
		ctx:         ctx,
		cancel: func() {
			timeoutCancel()
			cancel()
		},
		config: config,
	}

	if err := b.injectAntiDetection(); err != nil {
		b.Close()
		return nil, fmt.Errorf("failed to inject anti-detection: %w", err)
	}

	return b, nil
}

// Context returns the browser context
func (b *Browser) Context() context.Context {
	return b.ctx
}

// Close closes the browser and kills Chrome process if we launched it
func (b *Browser) Close() {
	if b.cancel != nil {
		b.cancel()
	}
	if b.allocCancel != nil {
		b.allocCancel()
	}
	// Kill Chrome process if we launched it
	if b.chromeCmd != nil && b.chromeCmd.Process != nil {
		log.Println("Killing Chrome process...")
		b.chromeCmd.Process.Kill()
	}
}

// injectAntiDetection injects JavaScript to bypass bot detection
func (b *Browser) injectAntiDetection() error {
	// JavaScript to override navigator properties and hide automation
	antiDetectionScript := `
		// Override webdriver property
		Object.defineProperty(navigator, 'webdriver', {
			get: () => undefined,
		});

		// Override plugins
		Object.defineProperty(navigator, 'plugins', {
			get: () => [1, 2, 3, 4, 5],
		});

		// Override languages
		Object.defineProperty(navigator, 'languages', {
			get: () => ['en-US', 'en'],
		});

		// Override platform
		Object.defineProperty(navigator, 'platform', {
			get: () => 'MacIntel',
		});

		// Override hardware concurrency
		Object.defineProperty(navigator, 'hardwareConcurrency', {
			get: () => 8,
		});

		// Override device memory
		Object.defineProperty(navigator, 'deviceMemory', {
			get: () => 8,
		});

		// Override permissions query
		const originalQuery = window.navigator.permissions.query;
		window.navigator.permissions.query = (parameters) => (
			parameters.name === 'notifications' ?
				Promise.resolve({ state: Notification.permission }) :
				originalQuery(parameters)
		);

		// Override chrome runtime
		window.chrome = {
			runtime: {},
		};

		// Override console.debug to prevent detection
		window.console.debug = () => {};

		// Add mouse movement simulation capability
		window.simulateMouseMovement = function() {
			const event = new MouseEvent('mousemove', {
				bubbles: true,
				cancelable: true,
				clientX: Math.random() * window.innerWidth,
				clientY: Math.random() * window.innerHeight
			});
			document.dispatchEvent(event);
		};
	`

	return chromedp.Run(b.ctx,
		chromedp.ActionFunc(func(ctx context.Context) error {
			_, err := page.AddScriptToEvaluateOnNewDocument(antiDetectionScript).Do(ctx)
			return err
		}),
	)
}

// Navigate navigates to a URL and checks for CAPTCHA/bot detection
func (b *Browser) Navigate(url string) error {
	if err := chromedp.Run(b.ctx,
		chromedp.Navigate(url),
		chromedp.WaitVisible("body", chromedp.ByQuery),
		chromedp.Sleep(2*time.Second),
	); err != nil {
		return err
	}

	// Check for CAPTCHA/bot detection
	if detected, err := b.CheckForCaptcha(); err != nil {
		return err
	} else if detected {
		return ErrCaptchaDetected
	}

	return nil
}

// CheckForCaptcha checks if the page shows CAPTCHA or bot detection
func (b *Browser) CheckForCaptcha() (bool, error) {
	var pageContent string
	var pageTitle string

	if err := chromedp.Run(b.ctx,
		chromedp.Title(&pageTitle),
		chromedp.Evaluate(`document.body.innerText`, &pageContent),
	); err != nil {
		return false, err
	}

	// Check for common CAPTCHA/bot detection indicators
	captchaIndicators := []string{
		"Failed to verify your browser",
		"Vercel Security Checkpoint",
		"Please verify you are a human",
		"Access denied",
		"Checking your browser",
		"Please complete the security check",
		"captcha",
		"CAPTCHA",
		"robot",
		"bot detection",
		"security challenge",
		"Code 10",
		"Code 21",
		"Code 22",
	}

	contentLower := strings.ToLower(pageContent)
	titleLower := strings.ToLower(pageTitle)

	for _, indicator := range captchaIndicators {
		indicatorLower := strings.ToLower(indicator)
		if strings.Contains(contentLower, indicatorLower) || strings.Contains(titleLower, indicatorLower) {
			log.Printf("CAPTCHA/Bot detection detected: found '%s'", indicator)
			return true, nil
		}
	}

	return false, nil
}

// WaitAndClick waits for an element and clicks it with human-like delay
func (b *Browser) WaitAndClick(selector string) error {
	RandomDelay(500, 1500)
	return chromedp.Run(b.ctx,
		chromedp.WaitVisible(selector, chromedp.ByQuery),
		chromedp.Click(selector, chromedp.ByQuery),
	)
}

// Type types text with human-like delays between keystrokes
func (b *Browser) Type(selector, text string) error {
	RandomDelay(300, 800)
	return chromedp.Run(b.ctx,
		chromedp.WaitVisible(selector, chromedp.ByQuery),
		chromedp.Click(selector, chromedp.ByQuery),
		chromedp.Sleep(200*time.Millisecond),
		chromedp.SendKeys(selector, text, chromedp.ByQuery),
	)
}

// Evaluate evaluates JavaScript and returns the result
func (b *Browser) Evaluate(script string, result interface{}) error {
	return chromedp.Run(b.ctx,
		chromedp.Evaluate(script, result),
	)
}

// Sleep pauses execution for the specified duration
func (b *Browser) Sleep(d time.Duration) error {
	return chromedp.Run(b.ctx, chromedp.Sleep(d))
}

// Nodes returns nodes matching the selector
func (b *Browser) Nodes(selector string) ([]*cdp.Node, error) {
	var nodes []*cdp.Node
	err := chromedp.Run(b.ctx,
		chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
	)
	return nodes, err
}

// SimulateHumanBehavior performs random mouse movements and scrolls
func (b *Browser) SimulateHumanBehavior() error {
	return chromedp.Run(b.ctx,
		chromedp.Evaluate(`window.simulateMouseMovement()`, nil),
		chromedp.Sleep(time.Duration(rand.Intn(500)+200)*time.Millisecond),
		chromedp.Evaluate(`window.scrollBy(0, Math.random() * 100)`, nil),
	)
}

// RandomDelay waits for a random duration between min and max milliseconds
func RandomDelay(minMs, maxMs int) {
	delay := time.Duration(rand.Intn(maxMs-minMs)+minMs) * time.Millisecond
	time.Sleep(delay)
}

// getRandomUserAgent returns a random modern user agent
func getRandomUserAgent() string {
	userAgents := []string{
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
		"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
	}
	return userAgents[rand.Intn(len(userAgents))]
}