570 lines
15 KiB
Go
570 lines
15 KiB
Go
package scraper
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jobs-scraper/internal/pkg/domain"
|
|
"github.com/jobs-scraper/libs/repo"
|
|
"github.com/jobs-scraper/services/scraper-hiringcafe/api"
|
|
"github.com/jobs-scraper/services/scraper-hiringcafe/browser"
|
|
"github.com/jobs-scraper/services/scraper-hiringcafe/models"
|
|
)
|
|
|
|
// HiringCafeScraper handles scraping jobs from hiring.cafe
|
|
type HiringCafeScraper struct {
|
|
jobRepo *repo.JobRepository
|
|
jobDescRepo *repo.JobDescriptionRepository
|
|
apiClient *api.Client
|
|
browser *browser.Browser
|
|
}
|
|
|
|
// New creates a new scraper instance
|
|
func New(jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository) *HiringCafeScraper {
|
|
return &HiringCafeScraper{
|
|
jobRepo: jobRepo,
|
|
jobDescRepo: jobDescRepo,
|
|
apiClient: api.NewClient(),
|
|
}
|
|
}
|
|
|
|
// Scrape performs the main scraping operation using the API
|
|
func (s *HiringCafeScraper) Scrape(ctx context.Context, filters models.SearchFilters) error {
|
|
log.Println("Starting hiring.cafe scraper using API...")
|
|
|
|
// Convert models.SearchFilters to api.SearchFilters
|
|
apiFilters := api.SearchFilters{
|
|
Query: filters.Keywords,
|
|
Country: "United States",
|
|
Remote: filters.Remote,
|
|
Hybrid: true,
|
|
Onsite: true,
|
|
FullTime: true,
|
|
PartTime: false,
|
|
Contract: false,
|
|
Internship: false,
|
|
EntryLevel: true,
|
|
MidLevel: true,
|
|
SeniorLevel: false,
|
|
DateRangeDays: 30,
|
|
MaxPages: 10,
|
|
}
|
|
|
|
if filters.Location != "" {
|
|
apiFilters.Country = filters.Location
|
|
}
|
|
|
|
log.Printf("Searching for: %s in %s", apiFilters.Query, apiFilters.Country)
|
|
|
|
// Fetch jobs from API
|
|
apiJobs, err := s.apiClient.Search(apiFilters)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to fetch jobs from API: %w", err)
|
|
}
|
|
|
|
log.Printf("Found %d jobs from API", len(apiJobs))
|
|
|
|
if len(apiJobs) == 0 {
|
|
log.Println("No jobs found")
|
|
return nil
|
|
}
|
|
|
|
// Convert API jobs to scraped jobs
|
|
var scrapedJobs []models.ScrapedJob
|
|
for _, job := range apiJobs {
|
|
scrapedJob := models.ScrapedJob{
|
|
Title: job.Title,
|
|
Company: job.Company,
|
|
CompanyLink: job.CompanyURL,
|
|
Location: job.Location,
|
|
JobLink: job.ApplyURL,
|
|
PostedTime: job.PostedAt.Format("2006-01-02"),
|
|
Description: job.DescriptionClean,
|
|
Criteria: make(map[string]string),
|
|
}
|
|
|
|
if job.Salary != "" {
|
|
scrapedJob.Criteria["Salary"] = job.Salary
|
|
}
|
|
if job.Remote {
|
|
scrapedJob.Criteria["Remote"] = "Yes"
|
|
}
|
|
|
|
scrapedJobs = append(scrapedJobs, scrapedJob)
|
|
}
|
|
|
|
// Save jobs to database
|
|
if err := s.saveJobs(scrapedJobs); err != nil {
|
|
return fmt.Errorf("failed to save jobs: %w", err)
|
|
}
|
|
|
|
log.Printf("Successfully saved %d jobs to database", len(scrapedJobs))
|
|
return nil
|
|
}
|
|
|
|
// ScrapeWithBrowser performs scraping using browser automation (fallback if API fails)
|
|
func (s *HiringCafeScraper) ScrapeWithBrowser(ctx context.Context, filters models.SearchFilters) error {
|
|
// Create browser with anti-detection
|
|
config := browser.DefaultConfig()
|
|
config.Headless = false
|
|
|
|
b, err := browser.New(ctx, config)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create browser: %w", err)
|
|
}
|
|
defer b.Close()
|
|
s.browser = b
|
|
|
|
log.Println("Starting browser and navigating to hiring.cafe...")
|
|
|
|
// Navigate to hiring.cafe
|
|
if err := b.Navigate("https://hiring.cafe/"); err != nil {
|
|
if err == browser.ErrCaptchaDetected {
|
|
log.Println("CAPTCHA/Bot detection encountered - terminating scraper")
|
|
return err
|
|
}
|
|
return fmt.Errorf("failed to navigate to hiring.cafe: %w", err)
|
|
}
|
|
|
|
log.Println("Page loaded successfully, no CAPTCHA detected")
|
|
|
|
// Apply filters
|
|
if err := s.applyFilters(filters); err != nil {
|
|
return fmt.Errorf("failed to apply filters: %w", err)
|
|
}
|
|
|
|
log.Println("Filters applied, extracting job listings...")
|
|
|
|
// Extract job listings
|
|
jobs, err := s.extractJobListings()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to extract job listings: %w", err)
|
|
}
|
|
|
|
log.Printf("Found %d jobs", len(jobs))
|
|
|
|
// Save jobs to database
|
|
if err := s.saveJobs(jobs); err != nil {
|
|
return fmt.Errorf("failed to save jobs: %w", err)
|
|
}
|
|
|
|
log.Printf("Successfully saved %d jobs to database", len(jobs))
|
|
return nil
|
|
}
|
|
|
|
// applyFilters applies search filters on hiring.cafe
|
|
func (s *HiringCafeScraper) applyFilters(filters models.SearchFilters) error {
|
|
if err := s.browser.Sleep(2 * time.Second); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Apply keyword search if provided
|
|
if filters.Keywords != "" {
|
|
log.Printf("Searching for: %s", filters.Keywords)
|
|
|
|
// Try multiple possible search input selectors
|
|
searchSelectors := []string{
|
|
`input[type="search"]`,
|
|
`input[placeholder*="search" i]`,
|
|
`input[placeholder*="job" i]`,
|
|
`input[type="text"]`,
|
|
`input[name="q"]`,
|
|
`input[name="search"]`,
|
|
}
|
|
|
|
var searchApplied bool
|
|
for _, selector := range searchSelectors {
|
|
if err := s.browser.Type(selector, filters.Keywords); err == nil {
|
|
searchApplied = true
|
|
log.Printf("Applied search using selector: %s", selector)
|
|
break
|
|
}
|
|
}
|
|
|
|
if searchApplied {
|
|
browser.RandomDelay(500, 1000)
|
|
// Press Enter to search
|
|
var result interface{}
|
|
s.browser.Evaluate(`
|
|
const input = document.querySelector('input[type="search"], input[placeholder*="search" i], input[type="text"]');
|
|
if (input) {
|
|
input.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true}));
|
|
input.form?.submit();
|
|
}
|
|
`, &result)
|
|
s.browser.Sleep(3 * time.Second)
|
|
} else {
|
|
log.Println("Warning: could not find search input")
|
|
}
|
|
}
|
|
|
|
// Apply remote filter if enabled
|
|
if filters.Remote {
|
|
log.Println("Applying remote filter...")
|
|
remoteSelectors := []string{
|
|
`[data-filter="remote"]`,
|
|
`button:contains("Remote")`,
|
|
`label:contains("Remote")`,
|
|
`input[value="remote"]`,
|
|
`[class*="remote"]`,
|
|
`a[href*="remote"]`,
|
|
}
|
|
|
|
for _, selector := range remoteSelectors {
|
|
if err := s.browser.WaitAndClick(selector); err == nil {
|
|
log.Printf("Applied remote filter using selector: %s", selector)
|
|
s.browser.Sleep(2 * time.Second)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Wait for results to load
|
|
return s.browser.Sleep(3 * time.Second)
|
|
}
|
|
|
|
// extractJobListings extracts job listings from the current page
|
|
func (s *HiringCafeScraper) extractJobListings() ([]models.ScrapedJob, error) {
|
|
if err := s.browser.Sleep(2 * time.Second); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// Simulate scrolling to load more jobs
|
|
s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 2)`, nil)
|
|
s.browser.Sleep(1 * time.Second)
|
|
s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil)
|
|
s.browser.Sleep(2 * time.Second)
|
|
|
|
// Extract job data using JavaScript
|
|
var jobData []map[string]string
|
|
err := s.browser.Evaluate(`
|
|
(function() {
|
|
const jobs = [];
|
|
|
|
// Try multiple selectors for job cards
|
|
const selectors = [
|
|
'[class*="job-card"]',
|
|
'[class*="job-listing"]',
|
|
'[class*="JobCard"]',
|
|
'[class*="job-item"]',
|
|
'article[class*="job"]',
|
|
'div[data-job-id]',
|
|
'a[href*="/jobs/"]',
|
|
'a[href*="/job/"]',
|
|
'[class*="listing-card"]',
|
|
'[class*="position-card"]',
|
|
'li[class*="job"]',
|
|
'tr[class*="job"]'
|
|
];
|
|
|
|
let elements = [];
|
|
for (const selector of selectors) {
|
|
const found = document.querySelectorAll(selector);
|
|
if (found.length > 0) {
|
|
elements = Array.from(found);
|
|
console.log('Found jobs with selector:', selector, found.length);
|
|
break;
|
|
}
|
|
}
|
|
|
|
// If no specific job cards found, try to find any links that look like job postings
|
|
if (elements.length === 0) {
|
|
const allLinks = document.querySelectorAll('a[href*="job"], a[href*="position"], a[href*="career"]');
|
|
elements = Array.from(allLinks).filter(el => {
|
|
const text = el.textContent.toLowerCase();
|
|
return text.length > 10 && text.length < 200;
|
|
});
|
|
}
|
|
|
|
elements.forEach((el, index) => {
|
|
try {
|
|
// Try to extract job information
|
|
const titleSelectors = ['h1', 'h2', 'h3', 'h4', '[class*="title"]', '[class*="Title"]', '[class*="name"]'];
|
|
const companySelectors = ['[class*="company"]', '[class*="Company"]', '[class*="employer"]', '[class*="org"]'];
|
|
const locationSelectors = ['[class*="location"]', '[class*="Location"]', '[class*="place"]', '[class*="city"]'];
|
|
const timeSelectors = ['[class*="time"]', '[class*="date"]', '[class*="posted"]', 'time', '[class*="ago"]'];
|
|
|
|
let title = '';
|
|
let company = '';
|
|
let location = '';
|
|
let postedTime = '';
|
|
|
|
for (const sel of titleSelectors) {
|
|
const titleEl = el.querySelector(sel);
|
|
if (titleEl && titleEl.textContent.trim()) {
|
|
title = titleEl.textContent.trim();
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (const sel of companySelectors) {
|
|
const companyEl = el.querySelector(sel);
|
|
if (companyEl && companyEl.textContent.trim()) {
|
|
company = companyEl.textContent.trim();
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (const sel of locationSelectors) {
|
|
const locationEl = el.querySelector(sel);
|
|
if (locationEl && locationEl.textContent.trim()) {
|
|
location = locationEl.textContent.trim();
|
|
break;
|
|
}
|
|
}
|
|
|
|
for (const sel of timeSelectors) {
|
|
const timeEl = el.querySelector(sel);
|
|
if (timeEl && timeEl.textContent.trim()) {
|
|
postedTime = timeEl.textContent.trim();
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Get job link
|
|
let jobLink = '';
|
|
const linkEl = el.tagName === 'A' ? el : el.querySelector('a[href*="job"], a[href*="position"], a');
|
|
if (linkEl && linkEl.href) {
|
|
jobLink = linkEl.href;
|
|
}
|
|
|
|
// Get company link
|
|
let companyLink = '';
|
|
const companyLinkEl = el.querySelector('a[href*="company"], a[href*="employer"]');
|
|
if (companyLinkEl && companyLinkEl.href) {
|
|
companyLink = companyLinkEl.href;
|
|
}
|
|
|
|
// If no title found, use link text
|
|
if (!title && linkEl) {
|
|
title = linkEl.textContent.trim();
|
|
}
|
|
|
|
const job = {
|
|
title: title.substring(0, 500),
|
|
company: company.substring(0, 200),
|
|
location: location.substring(0, 200),
|
|
jobLink: jobLink,
|
|
postedTime: postedTime.substring(0, 100),
|
|
companyLink: companyLink
|
|
};
|
|
|
|
// Only add if we have at least a title and link
|
|
if (job.title && job.jobLink) {
|
|
jobs.push(job);
|
|
}
|
|
} catch (error) {
|
|
console.error('Error extracting job ' + index + ':', error);
|
|
}
|
|
});
|
|
|
|
// Remove duplicates based on jobLink
|
|
const seen = new Set();
|
|
return jobs.filter(job => {
|
|
if (seen.has(job.jobLink)) return false;
|
|
seen.add(job.jobLink);
|
|
return true;
|
|
});
|
|
})()
|
|
`, &jobData)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to extract job data: %w", err)
|
|
}
|
|
|
|
// Convert to ScrapedJob structs
|
|
var jobs []models.ScrapedJob
|
|
for _, data := range jobData {
|
|
job := models.ScrapedJob{
|
|
Title: data["title"],
|
|
Company: data["company"],
|
|
CompanyLink: data["companyLink"],
|
|
Location: data["location"],
|
|
JobLink: data["jobLink"],
|
|
PostedTime: data["postedTime"],
|
|
Criteria: make(map[string]string),
|
|
}
|
|
jobs = append(jobs, job)
|
|
}
|
|
|
|
return jobs, nil
|
|
}
|
|
|
|
// getJobDetails navigates to a job page and extracts detailed information
|
|
func (s *HiringCafeScraper) getJobDetails(job models.ScrapedJob) (models.ScrapedJob, error) {
|
|
if job.JobLink == "" {
|
|
return job, nil
|
|
}
|
|
|
|
// Navigate to job detail page
|
|
if err := s.browser.Navigate(job.JobLink); err != nil {
|
|
return job, fmt.Errorf("failed to navigate to job page: %w", err)
|
|
}
|
|
|
|
s.browser.Sleep(2 * time.Second)
|
|
s.browser.SimulateHumanBehavior()
|
|
|
|
// Extract job description
|
|
var description string
|
|
s.browser.Evaluate(`
|
|
(function() {
|
|
const descSelectors = [
|
|
'[class*="description"]',
|
|
'[class*="Description"]',
|
|
'[class*="job-content"]',
|
|
'[class*="job-details"]',
|
|
'[class*="details"]',
|
|
'[id*="description"]',
|
|
'article',
|
|
'main',
|
|
'.content'
|
|
];
|
|
|
|
for (const selector of descSelectors) {
|
|
const el = document.querySelector(selector);
|
|
if (el && el.textContent.trim().length > 100) {
|
|
return el.textContent.trim().substring(0, 10000);
|
|
}
|
|
}
|
|
|
|
// Fallback: get body text
|
|
return document.body.innerText.substring(0, 10000);
|
|
})()
|
|
`, &description)
|
|
|
|
// Extract criteria
|
|
var criteriaData map[string]string
|
|
s.browser.Evaluate(`
|
|
(function() {
|
|
const criteria = {};
|
|
|
|
// Look for common job criteria patterns
|
|
const patterns = [
|
|
{key: 'Experience', selectors: ['[class*="experience"]', '[class*="Experience"]']},
|
|
{key: 'Education', selectors: ['[class*="education"]', '[class*="Education"]']},
|
|
{key: 'Skills', selectors: ['[class*="skills"]', '[class*="Skills"]']},
|
|
{key: 'Salary', selectors: ['[class*="salary"]', '[class*="Salary"]', '[class*="compensation"]']},
|
|
{key: 'JobType', selectors: ['[class*="job-type"]', '[class*="employment"]', '[class*="type"]']},
|
|
];
|
|
|
|
patterns.forEach(({key, selectors}) => {
|
|
for (const selector of selectors) {
|
|
const el = document.querySelector(selector);
|
|
if (el && el.textContent.trim()) {
|
|
criteria[key] = el.textContent.trim().substring(0, 500);
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
|
|
return criteria;
|
|
})()
|
|
`, &criteriaData)
|
|
|
|
job.Description = description
|
|
if criteriaData != nil {
|
|
job.Criteria = criteriaData
|
|
}
|
|
|
|
return job, nil
|
|
}
|
|
|
|
// saveJobs saves the scraped jobs to the database
|
|
func (s *HiringCafeScraper) saveJobs(scrapedJobs []models.ScrapedJob) error {
|
|
if len(scrapedJobs) == 0 {
|
|
return nil
|
|
}
|
|
|
|
var domainJobs []domain.Job
|
|
var jobDescriptions []domain.JobDescription
|
|
|
|
for _, sj := range scrapedJobs {
|
|
postTime := parsePostedTime(sj.PostedTime)
|
|
|
|
job := domain.Job{
|
|
Title: sj.Title,
|
|
Company: sj.Company,
|
|
CompanyLink: sj.CompanyLink,
|
|
Location: sj.Location,
|
|
JobLink: sj.JobLink,
|
|
Provider: domain.HiringCafe,
|
|
JobPostTime: postTime,
|
|
Status: domain.JobStatusCreated,
|
|
}
|
|
domainJobs = append(domainJobs, job)
|
|
}
|
|
|
|
// Save jobs
|
|
if err := s.jobRepo.SaveJobs(domainJobs); err != nil {
|
|
return fmt.Errorf("failed to save jobs: %w", err)
|
|
}
|
|
|
|
// Collect job descriptions
|
|
for _, sj := range scrapedJobs {
|
|
if sj.Description == "" {
|
|
continue
|
|
}
|
|
|
|
jobDesc := domain.JobDescription{
|
|
Description: sj.Description,
|
|
Criteria: sj.Criteria,
|
|
}
|
|
jobDescriptions = append(jobDescriptions, jobDesc)
|
|
}
|
|
|
|
log.Printf("Job descriptions collected: %d (saving requires job IDs)", len(jobDescriptions))
|
|
|
|
return nil
|
|
}
|
|
|
|
// parsePostedTime converts relative time strings to actual dates
|
|
func parsePostedTime(timeStr string) *time.Time {
|
|
now := time.Now()
|
|
timeStr = strings.ToLower(strings.TrimSpace(timeStr))
|
|
|
|
if timeStr == "" {
|
|
return &now
|
|
}
|
|
|
|
// Parse common patterns
|
|
if strings.Contains(timeStr, "hour") || strings.HasSuffix(timeStr, "h") {
|
|
var hours int
|
|
fmt.Sscanf(timeStr, "%d", &hours)
|
|
if hours > 0 {
|
|
t := now.Add(-time.Duration(hours) * time.Hour)
|
|
return &t
|
|
}
|
|
}
|
|
|
|
if strings.Contains(timeStr, "day") || strings.HasSuffix(timeStr, "d") {
|
|
var days int
|
|
fmt.Sscanf(timeStr, "%d", &days)
|
|
if days > 0 {
|
|
t := now.AddDate(0, 0, -days)
|
|
return &t
|
|
}
|
|
}
|
|
|
|
if strings.Contains(timeStr, "week") || strings.HasSuffix(timeStr, "w") {
|
|
var weeks int
|
|
fmt.Sscanf(timeStr, "%d", &weeks)
|
|
if weeks > 0 {
|
|
t := now.AddDate(0, 0, -weeks*7)
|
|
return &t
|
|
}
|
|
}
|
|
|
|
if strings.Contains(timeStr, "month") || strings.HasSuffix(timeStr, "m") {
|
|
var months int
|
|
fmt.Sscanf(timeStr, "%d", &months)
|
|
if months > 0 {
|
|
t := now.AddDate(0, -months, 0)
|
|
return &t
|
|
}
|
|
}
|
|
|
|
return &now
|
|
}
|