92 lines
2.7 KiB
Go
92 lines
2.7 KiB
Go
package utils
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/chromedp/chromedp"
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
)
|
|
|
|
func GetHTMLRaw(ctx context.Context, googleLink domain.GoogleLink) (string, string, error) {
|
|
// Set up chromedp options
|
|
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
|
chromedp.Flag("headless", true),
|
|
chromedp.Flag("disable-blink-features", "AutomationControlled"),
|
|
chromedp.Flag("excludeSwitches", "enable-automation"),
|
|
chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"),
|
|
chromedp.WindowSize(1920, 1080),
|
|
)
|
|
|
|
allocCtx, allocCancel := chromedp.NewExecAllocator(ctx, opts...)
|
|
defer allocCancel()
|
|
|
|
browserCtx, browserCancel := chromedp.NewContext(allocCtx)
|
|
defer browserCancel()
|
|
|
|
// Add timeout for the entire operation
|
|
timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 30*time.Second)
|
|
defer timeoutCancel()
|
|
|
|
var htmlContent string
|
|
var finalURL string
|
|
|
|
err := chromedp.Run(timeoutCtx,
|
|
chromedp.Navigate(googleLink.Link),
|
|
chromedp.WaitVisible("body", chromedp.ByQuery),
|
|
chromedp.Sleep(2*time.Second), // Wait for JS to render
|
|
chromedp.Location(&finalURL),
|
|
chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
|
|
)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
// Parse the HTML with goquery
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
|
|
if err != nil {
|
|
log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
// Remove unwanted elements
|
|
doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove()
|
|
|
|
log.Printf(" Received: %s from %s\n, ID: %d", googleLink.Title, googleLink.CompanyName, googleLink.ID)
|
|
|
|
// Extract text content
|
|
var content strings.Builder
|
|
doc.Find("body").Each(func(i int, s *goquery.Selection) {
|
|
content.WriteString(s.Text())
|
|
})
|
|
|
|
// Extract hostname from the final URL
|
|
hostname, err := extractHostname(finalURL)
|
|
if err != nil {
|
|
log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err)
|
|
// Fall back to the original link if extraction fails
|
|
return cleanText(content.String()), googleLink.Link, nil
|
|
}
|
|
|
|
return cleanText(content.String()), hostname, nil
|
|
}
|
|
|
|
func cleanText(text string) string {
|
|
text = strings.Join(strings.Fields(text), " ")
|
|
return text
|
|
}
|
|
|
|
func extractHostname(urlStr string) (string, error) {
|
|
parsedURL, err := url.Parse(urlStr)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to parse URL: %w", err)
|
|
}
|
|
return parsedURL.Hostname(), nil
|
|
}
|