jobs-monorepo/services/scraper-google/utils/get-html-raw.go

package utils

import (
	"context"
	"fmt"
	"log"
	"net/url"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/chromedp/chromedp"
	"github.com/jobs-scraper/internal/pkg/domain"
)

func GetHTMLRaw(ctx context.Context, wsURL string, googleLink domain.GoogleLink) (string, string, error) {
	// Connect to the shared Chrome instance
	allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
	defer allocCancel()

	browserCtx, browserCancel := chromedp.NewContext(allocCtx)
	defer browserCancel()

	var htmlContent string
	var finalURL string

	err := chromedp.Run(browserCtx,
		chromedp.Navigate(googleLink.Link),
		chromedp.WaitVisible("body", chromedp.ByQuery),
		WaitForNetworkIdle(10*time.Second), // Wait for all network requests to complete
		chromedp.Location(&finalURL),
		chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
	)

	if err != nil {
		log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err)
		return "", "", err
	}

	// Parse the HTML with goquery
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
	if err != nil {
		log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err)
		return "", "", err
	}

	// Remove unwanted elements
	doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove()

	log.Printf("Received: %s from %s", googleLink.Title, googleLink.CompanyName)

	// Extract text content
	var content strings.Builder
	doc.Find("body").Each(func(i int, s *goquery.Selection) {
		content.WriteString(s.Text())
	})

	// Extract hostname from the final URL
	hostname, err := extractHostname(finalURL)
	if err != nil {
		log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err)
		// Fall back to the original link if extraction fails
		return cleanText(content.String()), googleLink.Link, nil
	}

	return cleanText(content.String()), hostname, nil
}

func cleanText(text string) string {
	text = strings.Join(strings.Fields(text), " ")
	return text
}

func extractHostname(urlStr string) (string, error) {
	parsedURL, err := url.Parse(urlStr)
	if err != nil {
		return "", fmt.Errorf("failed to parse URL: %w", err)
	}
	return parsedURL.Hostname(), nil
}

// ClickSelector clicks on an element using the provided selector string
// Selector format: "css:selector" or "xpath:selector"
func ClickSelector(ctx context.Context, selector string) error {
	if selector == "" {
		return fmt.Errorf("empty selector")
	}

	parts := strings.SplitN(selector, ":", 2)
	if len(parts) != 2 {
		return fmt.Errorf("invalid selector format: %s", selector)
	}

	selectorType := parts[0]
	selectorValue := parts[1]

	var err error
	switch selectorType {
	case "css":
		err = chromedp.Run(ctx,
			chromedp.WaitVisible(selectorValue, chromedp.ByQuery),
			chromedp.Click(selectorValue, chromedp.ByQuery),
		)
	case "xpath":
		err = chromedp.Run(ctx,
			chromedp.WaitVisible(selectorValue, chromedp.BySearch),
			chromedp.Click(selectorValue, chromedp.BySearch),
		)
	default:
		return fmt.Errorf("unknown selector type: %s", selectorType)
	}

	if err != nil {
		return fmt.Errorf("failed to click selector %s: %w", selector, err)
	}

	log.Printf("Successfully clicked on selector: %s", selector)
	return nil
}

// GetHTMLRawWithRetry fetches HTML and can retry with a selector click if needed
func GetHTMLRawWithRetry(ctx context.Context, wsURL string, googleLink domain.GoogleLink, selector string) (string, string, error) {
	// Connect to the shared Chrome instance
	allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
	defer allocCancel()

	browserCtx, browserCancel := chromedp.NewContext(allocCtx)
	defer browserCancel()

	timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 45*time.Second)
	defer timeoutCancel()

	var htmlContent string
	var finalURL string

	// Navigate to the page
	err := chromedp.Run(timeoutCtx,
		chromedp.Navigate(googleLink.Link),
		chromedp.WaitVisible("body", chromedp.ByQuery),
		WaitForNetworkIdle(5*time.Second),
	)
	if err != nil {
		return "", "", fmt.Errorf("failed to navigate: %w", err)
	}

	// If a selector is provided, click on it
	if selector != "" {
		if err := ClickSelector(timeoutCtx, selector); err != nil {
			log.Printf("Failed to click selector %s: %v", selector, err)
			// Continue anyway, we'll get whatever HTML we can
		} else {
			// Wait for the new page to load after clicking
			_ = chromedp.Run(timeoutCtx,
				chromedp.WaitVisible("body", chromedp.ByQuery),
				WaitForNetworkIdle(5*time.Second),
			)
		}
	}

	// Get the final HTML
	err = chromedp.Run(timeoutCtx,
		chromedp.Location(&finalURL),
		chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
	)
	if err != nil {
		return "", "", fmt.Errorf("failed to get HTML: %w", err)
	}

	// Parse and clean HTML
	doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
	if err != nil {
		return "", "", fmt.Errorf("failed to parse HTML: %w", err)
	}

	doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove()

	var content strings.Builder
	doc.Find("body").Each(func(i int, s *goquery.Selection) {
		content.WriteString(s.Text())
	})

	hostname, err := extractHostname(finalURL)
	if err != nil {
		return cleanText(content.String()), googleLink.Link, nil
	}

	return cleanText(content.String()), hostname, nil
}