package utils import ( "context" "fmt" "log" "net/url" "strings" "time" "github.com/PuerkitoBio/goquery" "github.com/chromedp/chromedp" "github.com/jobs-scraper/internal/pkg/domain" ) func GetHTMLRaw(ctx context.Context, googleLink domain.GoogleLink) (string, string, error) { // Set up chromedp options opts := append(chromedp.DefaultExecAllocatorOptions[:], chromedp.Flag("headless", true), chromedp.Flag("disable-blink-features", "AutomationControlled"), chromedp.Flag("excludeSwitches", "enable-automation"), chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), chromedp.WindowSize(1920, 1080), ) allocCtx, allocCancel := chromedp.NewExecAllocator(ctx, opts...) defer allocCancel() browserCtx, browserCancel := chromedp.NewContext(allocCtx) defer browserCancel() // Add timeout for the entire operation timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 30*time.Second) defer timeoutCancel() var htmlContent string var finalURL string err := chromedp.Run(timeoutCtx, chromedp.Navigate(googleLink.Link), chromedp.WaitVisible("body", chromedp.ByQuery), chromedp.Sleep(2*time.Second), // Wait for JS to render chromedp.Location(&finalURL), chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), ) if err != nil { log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err) return "", "", err } // Parse the HTML with goquery doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) if err != nil { log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err) return "", "", err } // Remove unwanted elements doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() log.Printf(" Received: %s from %s\n, ID: %d", googleLink.Title, googleLink.CompanyName, googleLink.ID) // Extract text content var content strings.Builder doc.Find("body").Each(func(i int, s *goquery.Selection) { content.WriteString(s.Text()) }) // Extract hostname from the final URL hostname, err := extractHostname(finalURL) if err != nil { log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err) // Fall back to the original link if extraction fails return cleanText(content.String()), googleLink.Link, nil } return cleanText(content.String()), hostname, nil } func cleanText(text string) string { text = strings.Join(strings.Fields(text), " ") return text } func extractHostname(urlStr string) (string, error) { parsedURL, err := url.Parse(urlStr) if err != nil { return "", fmt.Errorf("failed to parse URL: %w", err) } return parsedURL.Hostname(), nil }