248 lines
7 KiB
Go
248 lines
7 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"net/url"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/jobs-scraper/internal/domain"
|
|
"github.com/jobs-scraper/internal/utils"
|
|
)
|
|
|
|
type Config struct {
|
|
Timespan string // Time filter for job postings (e.g., "r86400" for last 24 hours)
|
|
SortBy string // Sort results by (R for relevance, DD for date posted)
|
|
MaxRetries int // Maximum number of retries for failed requests
|
|
BaseDelay time.Duration // Base delay for exponential backoff
|
|
MaxDelay time.Duration // Maximum delay between retries
|
|
RequestTimeout time.Duration // Timeout for individual HTTP requests
|
|
}
|
|
|
|
type Scraper struct {
|
|
config Config
|
|
}
|
|
|
|
func NewScraper(config Config) *Scraper {
|
|
// r86400 last 24 hours
|
|
// r604800 last week
|
|
// r2592000 last month
|
|
// r7776000 last 3 months
|
|
// r31536000 last year
|
|
if config.Timespan == "" {
|
|
config.Timespan = "r604800" // Default to last week
|
|
}
|
|
|
|
// Set default retry configuration
|
|
if config.MaxRetries == 0 {
|
|
config.MaxRetries = 3
|
|
}
|
|
if config.BaseDelay == 0 {
|
|
config.BaseDelay = 1 * time.Second
|
|
}
|
|
if config.MaxDelay == 0 {
|
|
config.MaxDelay = 30 * time.Second
|
|
}
|
|
if config.RequestTimeout == 0 {
|
|
config.RequestTimeout = 30 * time.Second
|
|
}
|
|
|
|
return &Scraper{
|
|
config: config,
|
|
}
|
|
}
|
|
|
|
// ScrapeLinkedInJobsStreaming scrapes jobs page by page and sends them to channel immediately
|
|
func (s *Scraper) ScrapeLinkedInJobsStreaming(ctx context.Context, numPages int, jobChan chan<- domain.Job, params domain.SearchQuery) error {
|
|
// Process pages sequentially to send jobs immediately
|
|
for i := range numPages {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
default:
|
|
jobs, err := s.ScrapeJobsWithContext(ctx, i, params)
|
|
if err != nil {
|
|
return fmt.Errorf("error scraping page %d: %w", i, err)
|
|
}
|
|
|
|
// Send jobs to channel immediately
|
|
for _, job := range jobs {
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case jobChan <- job:
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Scraper) ScrapeJobsWithContext(ctx context.Context, page int, params domain.SearchQuery) ([]domain.Job, error) {
|
|
jobs := make([]domain.Job, 0, 10)
|
|
url := s.buildSearchURL(params, page)
|
|
log.Println("Parsing page: ", page+1)
|
|
retryableRequest := utils.NewRetryableHTTPRequest(utils.RetryConfig{
|
|
BaseDelay: s.config.BaseDelay,
|
|
MaxDelay: s.config.MaxDelay,
|
|
MaxRetries: s.config.MaxRetries,
|
|
})
|
|
|
|
res, err := retryableRequest.RetryableHTTPRequest(ctx, url, "GET", nil, nil)
|
|
if err != nil {
|
|
fmt.Printf("Error fetching URL after retries: %v\n", err)
|
|
return jobs, err
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
|
if err != nil {
|
|
return jobs, err
|
|
}
|
|
|
|
doc.Find("li > div.base-card").Each(func(i int, s *goquery.Selection) {
|
|
job := domain.Job{}
|
|
title := s.Find("[class*=_title]").Text()
|
|
job.Title = strings.TrimSpace(title)
|
|
job.Company = strings.TrimSpace(s.Find(".hidden-nested-link").Text())
|
|
job.CompanyLink = strings.TrimSpace(s.Find(".hidden-nested-link").AttrOr("href", ""))
|
|
job.Location = strings.TrimSpace(params.Location)
|
|
job.JobLink = strings.TrimSpace(s.Find("a.base-card__full-link").AttrOr("href", ""))
|
|
postTimeText := s.Find("[class*=listdate]").AttrOr("datetime", time.DateTime)
|
|
postTime, err := time.Parse(time.DateOnly, postTimeText)
|
|
// Handle any parsing errors by logging them
|
|
if err != nil {
|
|
fmt.Printf("Couldn't parse the job post time: %v", err)
|
|
}
|
|
|
|
job.JobPostTime = &postTime
|
|
jobId, err := extractJobIDFromURL(job.JobLink)
|
|
if err != nil {
|
|
fmt.Printf("Error extracting job ID from URL %s: %v\n", job.JobLink, err)
|
|
}
|
|
jobIdInt, err := strconv.ParseInt(jobId, 10, 64)
|
|
|
|
if err != nil {
|
|
fmt.Printf("Error extracting job ID from URL %s: %v\n", job.JobLink, err)
|
|
}
|
|
job.ID = jobIdInt
|
|
job.Provider = 0
|
|
|
|
if job.Title != "" && job.Company != "" && job.Location != "" && job.JobLink != "" {
|
|
jobs = append(jobs, job)
|
|
}
|
|
})
|
|
|
|
return jobs, nil
|
|
}
|
|
|
|
func (s *Scraper) ScrapeJobDescriptionWithContext(ctx context.Context, job domain.Job) (string, map[string]string, error) {
|
|
var jobDescription string
|
|
jobLink := fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/%d", job.ID)
|
|
|
|
url := s.buildJobDescriptionSearchURL(jobLink)
|
|
|
|
retryableRequest := utils.NewRetryableHTTPRequest(utils.RetryConfig{
|
|
MaxRetries: s.config.MaxRetries,
|
|
BaseDelay: s.config.BaseDelay,
|
|
MaxDelay: s.config.MaxDelay,
|
|
})
|
|
|
|
res, err := retryableRequest.RetryableHTTPRequest(ctx, url, "GET", nil, nil)
|
|
if err != nil {
|
|
fmt.Printf("Error fetching job description URL after retries: %v\n", err)
|
|
return "", map[string]string{}, err
|
|
}
|
|
|
|
defer res.Body.Close()
|
|
|
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
|
if err != nil {
|
|
return "", map[string]string{}, err
|
|
}
|
|
|
|
// Find the job details section directly by ID and extract its text content
|
|
jobDescription, jobCriteria, err := s.parseJobDescription(doc)
|
|
if err != nil {
|
|
return "", map[string]string{}, err
|
|
}
|
|
|
|
return strings.TrimSpace(jobDescription), jobCriteria, nil
|
|
}
|
|
|
|
func (s *Scraper) buildSearchURL(query domain.SearchQuery, page int) string {
|
|
baseURL := "https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search"
|
|
params := url.Values{}
|
|
|
|
params.Set("keywords", query.Keywords)
|
|
params.Set("location", query.Location)
|
|
|
|
if query.FWT != "" {
|
|
params.Set("f_WT", query.FWT)
|
|
}
|
|
|
|
params.Set("start", strconv.Itoa(25*page))
|
|
|
|
return fmt.Sprintf("%s?%s", baseURL, params.Encode())
|
|
}
|
|
|
|
func (s *Scraper) buildJobDescriptionSearchURL(jobLink string) string {
|
|
return strings.ReplaceAll(jobLink, "jp.linkedin.com", "linkedin.com")
|
|
}
|
|
|
|
func (s *Scraper) parseJobDescription(doc *goquery.Document) (string, map[string]string, error) {
|
|
var jobDescription string
|
|
jobCriteria := make(map[string]string)
|
|
|
|
jobDescription = strings.TrimSpace(doc.Find("[class*=description] > section > div").Text())
|
|
|
|
criteriaList := doc.Find("[class*=_job-criteria-list]")
|
|
|
|
if criteriaList.Length() > 0 {
|
|
criteriaList.Find("li.description__job-criteria-item").Each(func(i int, li *goquery.Selection) {
|
|
header := strings.TrimSpace(li.Find("h3.description__job-criteria-subheader").Text())
|
|
|
|
value := strings.TrimSpace(li.Find("span.description__job-criteria-text").Text())
|
|
|
|
if header != "" && value != "" {
|
|
jobCriteria[header] = value
|
|
}
|
|
})
|
|
}
|
|
|
|
jobDescription = strings.TrimSpace(jobDescription)
|
|
|
|
if jobDescription == "" {
|
|
return "", jobCriteria, fmt.Errorf("job description not found")
|
|
}
|
|
|
|
return jobDescription, jobCriteria, nil
|
|
}
|
|
|
|
func extractJobIDFromURL(jobURL string) (string, error) {
|
|
parsed, err := url.Parse(jobURL)
|
|
if err != nil {
|
|
return "", fmt.Errorf("invalid URL: %w", err)
|
|
}
|
|
|
|
// Get the last segment of the path
|
|
lastSegment := path.Base(parsed.Path)
|
|
|
|
// Extract job ID from the end of the segment using regex
|
|
// Look for a sequence of digits at the end of the string
|
|
re := regexp.MustCompile(`(\d+)$`)
|
|
matches := re.FindStringSubmatch(lastSegment)
|
|
|
|
if len(matches) < 2 {
|
|
return "", fmt.Errorf("job ID not found in URL path: %s", jobURL)
|
|
}
|
|
|
|
return matches[1], nil
|
|
}
|