96 lines
2.3 KiB
Go
96 lines
2.3 KiB
Go
package utils
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/jobs-scraper/internal/browser"
|
|
"github.com/jobs-scraper/internal/domain"
|
|
)
|
|
|
|
func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) {
|
|
tab, err := b.NewTab()
|
|
|
|
if err != nil {
|
|
return "", "", fmt.Errorf("failed to create new tab: %w", err)
|
|
}
|
|
defer tab.Close()
|
|
|
|
err = tab.Navigate(googleLink.Link)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to navigate for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
err = tab.WaitVisible("body")
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to wait for visible body for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
err = tab.WaitForNetworkIdle(10 * time.Second)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to wait network idle for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
htmlContent, err := tab.GetHTML()
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get HTML content for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
|
|
if err != nil {
|
|
log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err)
|
|
return "", "", err
|
|
}
|
|
|
|
doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove()
|
|
|
|
log.Printf("Received: %s from %s", googleLink.Title, googleLink.CompanyName)
|
|
|
|
var content strings.Builder
|
|
doc.Find("body").Each(func(i int, s *goquery.Selection) {
|
|
content.WriteString(s.Text())
|
|
})
|
|
|
|
finalUrl, err := tab.GetCurrentLocation()
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to get current location for link %s: %v", googleLink.Link, err)
|
|
return cleanText(content.String()), googleLink.Link, nil
|
|
}
|
|
|
|
hostname, err := extractHostname(finalUrl)
|
|
|
|
if err != nil {
|
|
log.Printf("Failed to extract hostname from final URL %s: %v", finalUrl, err)
|
|
// Fall back to the original link if extraction fails
|
|
return cleanText(content.String()), googleLink.Link, nil
|
|
}
|
|
|
|
return cleanText(content.String()), hostname, nil
|
|
}
|
|
|
|
func cleanText(text string) string {
|
|
text = strings.Join(strings.Fields(text), " ")
|
|
return text
|
|
}
|
|
|
|
func extractHostname(urlStr string) (string, error) {
|
|
parsedURL, err := url.Parse(urlStr)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to parse URL: %w", err)
|
|
}
|
|
return parsedURL.Hostname(), nil
|
|
}
|