package utils import ( "context" "fmt" "log" "net/url" "strings" "time" "github.com/PuerkitoBio/goquery" "github.com/chromedp/chromedp" "github.com/jobs-scraper/internal/pkg/domain" ) func GetHTMLRaw(ctx context.Context, wsURL string, googleLink domain.GoogleLink) (string, string, error) { // Connect to the shared Chrome instance allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL) defer allocCancel() browserCtx, browserCancel := chromedp.NewContext(allocCtx) defer browserCancel() var htmlContent string var finalURL string err := chromedp.Run(browserCtx, chromedp.Navigate(googleLink.Link), chromedp.WaitVisible("body", chromedp.ByQuery), WaitForNetworkIdle(10*time.Second), // Wait for all network requests to complete chromedp.Location(&finalURL), chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), ) if err != nil { log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err) return "", "", err } // Parse the HTML with goquery doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) if err != nil { log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err) return "", "", err } // Remove unwanted elements doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() log.Printf("Received: %s from %s", googleLink.Title, googleLink.CompanyName) // Extract text content var content strings.Builder doc.Find("body").Each(func(i int, s *goquery.Selection) { content.WriteString(s.Text()) }) // Extract hostname from the final URL hostname, err := extractHostname(finalURL) if err != nil { log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err) // Fall back to the original link if extraction fails return cleanText(content.String()), googleLink.Link, nil } return cleanText(content.String()), hostname, nil } func cleanText(text string) string { text = strings.Join(strings.Fields(text), " ") return text } func extractHostname(urlStr string) (string, error) { parsedURL, err := url.Parse(urlStr) if err != nil { return "", fmt.Errorf("failed to parse URL: %w", err) } return parsedURL.Hostname(), nil } // ClickSelector clicks on an element using the provided selector string // Selector format: "css:selector" or "xpath:selector" func ClickSelector(ctx context.Context, selector string) error { if selector == "" { return fmt.Errorf("empty selector") } parts := strings.SplitN(selector, ":", 2) if len(parts) != 2 { return fmt.Errorf("invalid selector format: %s", selector) } selectorType := parts[0] selectorValue := parts[1] var err error switch selectorType { case "css": err = chromedp.Run(ctx, chromedp.WaitVisible(selectorValue, chromedp.ByQuery), chromedp.Click(selectorValue, chromedp.ByQuery), ) case "xpath": err = chromedp.Run(ctx, chromedp.WaitVisible(selectorValue, chromedp.BySearch), chromedp.Click(selectorValue, chromedp.BySearch), ) default: return fmt.Errorf("unknown selector type: %s", selectorType) } if err != nil { return fmt.Errorf("failed to click selector %s: %w", selector, err) } log.Printf("Successfully clicked on selector: %s", selector) return nil } // GetHTMLRawWithRetry fetches HTML and can retry with a selector click if needed func GetHTMLRawWithRetry(ctx context.Context, wsURL string, googleLink domain.GoogleLink, selector string) (string, string, error) { // Connect to the shared Chrome instance allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL) defer allocCancel() browserCtx, browserCancel := chromedp.NewContext(allocCtx) defer browserCancel() timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 45*time.Second) defer timeoutCancel() var htmlContent string var finalURL string // Navigate to the page err := chromedp.Run(timeoutCtx, chromedp.Navigate(googleLink.Link), chromedp.WaitVisible("body", chromedp.ByQuery), WaitForNetworkIdle(5*time.Second), ) if err != nil { return "", "", fmt.Errorf("failed to navigate: %w", err) } // If a selector is provided, click on it if selector != "" { if err := ClickSelector(timeoutCtx, selector); err != nil { log.Printf("Failed to click selector %s: %v", selector, err) // Continue anyway, we'll get whatever HTML we can } else { // Wait for the new page to load after clicking _ = chromedp.Run(timeoutCtx, chromedp.WaitVisible("body", chromedp.ByQuery), WaitForNetworkIdle(5*time.Second), ) } } // Get the final HTML err = chromedp.Run(timeoutCtx, chromedp.Location(&finalURL), chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), ) if err != nil { return "", "", fmt.Errorf("failed to get HTML: %w", err) } // Parse and clean HTML doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) if err != nil { return "", "", fmt.Errorf("failed to parse HTML: %w", err) } doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() var content strings.Builder doc.Find("body").Each(func(i int, s *goquery.Selection) { content.WriteString(s.Text()) }) hostname, err := extractHostname(finalURL) if err != nil { return cleanText(content.String()), googleLink.Link, nil } return cleanText(content.String()), hostname, nil }