package scraper import ( "context" "fmt" "log" "strings" "time" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/scraper-hiringcafe/api" "github.com/jobs-scraper/services/scraper-hiringcafe/browser" "github.com/jobs-scraper/services/scraper-hiringcafe/models" ) // HiringCafeScraper handles scraping jobs from hiring.cafe type HiringCafeScraper struct { jobRepo *repo.JobRepository jobDescRepo *repo.JobDescriptionRepository apiClient *api.Client browser *browser.Browser } // New creates a new scraper instance func New(jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository) *HiringCafeScraper { return &HiringCafeScraper{ jobRepo: jobRepo, jobDescRepo: jobDescRepo, apiClient: api.NewClient(), } } // Scrape performs the main scraping operation using the API func (s *HiringCafeScraper) Scrape(ctx context.Context, filters models.SearchFilters) error { log.Println("Starting hiring.cafe scraper using API...") // Convert models.SearchFilters to api.SearchFilters apiFilters := api.SearchFilters{ Query: filters.Keywords, Country: "United States", Remote: filters.Remote, Hybrid: true, Onsite: true, FullTime: true, PartTime: false, Contract: false, Internship: false, EntryLevel: true, MidLevel: true, SeniorLevel: false, DateRangeDays: 30, MaxPages: 10, } if filters.Location != "" { apiFilters.Country = filters.Location } log.Printf("Searching for: %s in %s", apiFilters.Query, apiFilters.Country) // Fetch jobs from API apiJobs, err := s.apiClient.Search(apiFilters) if err != nil { return fmt.Errorf("failed to fetch jobs from API: %w", err) } log.Printf("Found %d jobs from API", len(apiJobs)) if len(apiJobs) == 0 { log.Println("No jobs found") return nil } // Convert API jobs to scraped jobs var scrapedJobs []models.ScrapedJob for _, job := range apiJobs { scrapedJob := models.ScrapedJob{ Title: job.Title, Company: job.Company, CompanyLink: job.CompanyURL, Location: job.Location, JobLink: job.ApplyURL, PostedTime: job.PostedAt.Format("2006-01-02"), Description: job.DescriptionClean, Criteria: make(map[string]string), } if job.Salary != "" { scrapedJob.Criteria["Salary"] = job.Salary } if job.Remote { scrapedJob.Criteria["Remote"] = "Yes" } scrapedJobs = append(scrapedJobs, scrapedJob) } // Save jobs to database if err := s.saveJobs(scrapedJobs); err != nil { return fmt.Errorf("failed to save jobs: %w", err) } log.Printf("Successfully saved %d jobs to database", len(scrapedJobs)) return nil } // ScrapeWithBrowser performs scraping using browser automation (fallback if API fails) func (s *HiringCafeScraper) ScrapeWithBrowser(ctx context.Context, filters models.SearchFilters) error { // Create browser with anti-detection config := browser.DefaultConfig() config.Headless = false b, err := browser.New(ctx, config) if err != nil { return fmt.Errorf("failed to create browser: %w", err) } defer b.Close() s.browser = b log.Println("Starting browser and navigating to hiring.cafe...") // Navigate to hiring.cafe if err := b.Navigate("https://hiring.cafe/"); err != nil { if err == browser.ErrCaptchaDetected { log.Println("CAPTCHA/Bot detection encountered - terminating scraper") return err } return fmt.Errorf("failed to navigate to hiring.cafe: %w", err) } log.Println("Page loaded successfully, no CAPTCHA detected") // Apply filters if err := s.applyFilters(filters); err != nil { return fmt.Errorf("failed to apply filters: %w", err) } log.Println("Filters applied, extracting job listings...") // Extract job listings jobs, err := s.extractJobListings() if err != nil { return fmt.Errorf("failed to extract job listings: %w", err) } log.Printf("Found %d jobs", len(jobs)) // Save jobs to database if err := s.saveJobs(jobs); err != nil { return fmt.Errorf("failed to save jobs: %w", err) } log.Printf("Successfully saved %d jobs to database", len(jobs)) return nil } // applyFilters applies search filters on hiring.cafe func (s *HiringCafeScraper) applyFilters(filters models.SearchFilters) error { if err := s.browser.Sleep(2 * time.Second); err != nil { return err } // Apply keyword search if provided if filters.Keywords != "" { log.Printf("Searching for: %s", filters.Keywords) // Try multiple possible search input selectors searchSelectors := []string{ `input[type="search"]`, `input[placeholder*="search" i]`, `input[placeholder*="job" i]`, `input[type="text"]`, `input[name="q"]`, `input[name="search"]`, } var searchApplied bool for _, selector := range searchSelectors { if err := s.browser.Type(selector, filters.Keywords); err == nil { searchApplied = true log.Printf("Applied search using selector: %s", selector) break } } if searchApplied { browser.RandomDelay(500, 1000) // Press Enter to search var result interface{} s.browser.Evaluate(` const input = document.querySelector('input[type="search"], input[placeholder*="search" i], input[type="text"]'); if (input) { input.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true})); input.form?.submit(); } `, &result) s.browser.Sleep(3 * time.Second) } else { log.Println("Warning: could not find search input") } } // Apply remote filter if enabled if filters.Remote { log.Println("Applying remote filter...") remoteSelectors := []string{ `[data-filter="remote"]`, `button:contains("Remote")`, `label:contains("Remote")`, `input[value="remote"]`, `[class*="remote"]`, `a[href*="remote"]`, } for _, selector := range remoteSelectors { if err := s.browser.WaitAndClick(selector); err == nil { log.Printf("Applied remote filter using selector: %s", selector) s.browser.Sleep(2 * time.Second) break } } } // Wait for results to load return s.browser.Sleep(3 * time.Second) } // extractJobListings extracts job listings from the current page func (s *HiringCafeScraper) extractJobListings() ([]models.ScrapedJob, error) { if err := s.browser.Sleep(2 * time.Second); err != nil { return nil, err } // Simulate scrolling to load more jobs s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 2)`, nil) s.browser.Sleep(1 * time.Second) s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil) s.browser.Sleep(2 * time.Second) // Extract job data using JavaScript var jobData []map[string]string err := s.browser.Evaluate(` (function() { const jobs = []; // Try multiple selectors for job cards const selectors = [ '[class*="job-card"]', '[class*="job-listing"]', '[class*="JobCard"]', '[class*="job-item"]', 'article[class*="job"]', 'div[data-job-id]', 'a[href*="/jobs/"]', 'a[href*="/job/"]', '[class*="listing-card"]', '[class*="position-card"]', 'li[class*="job"]', 'tr[class*="job"]' ]; let elements = []; for (const selector of selectors) { const found = document.querySelectorAll(selector); if (found.length > 0) { elements = Array.from(found); console.log('Found jobs with selector:', selector, found.length); break; } } // If no specific job cards found, try to find any links that look like job postings if (elements.length === 0) { const allLinks = document.querySelectorAll('a[href*="job"], a[href*="position"], a[href*="career"]'); elements = Array.from(allLinks).filter(el => { const text = el.textContent.toLowerCase(); return text.length > 10 && text.length < 200; }); } elements.forEach((el, index) => { try { // Try to extract job information const titleSelectors = ['h1', 'h2', 'h3', 'h4', '[class*="title"]', '[class*="Title"]', '[class*="name"]']; const companySelectors = ['[class*="company"]', '[class*="Company"]', '[class*="employer"]', '[class*="org"]']; const locationSelectors = ['[class*="location"]', '[class*="Location"]', '[class*="place"]', '[class*="city"]']; const timeSelectors = ['[class*="time"]', '[class*="date"]', '[class*="posted"]', 'time', '[class*="ago"]']; let title = ''; let company = ''; let location = ''; let postedTime = ''; for (const sel of titleSelectors) { const titleEl = el.querySelector(sel); if (titleEl && titleEl.textContent.trim()) { title = titleEl.textContent.trim(); break; } } for (const sel of companySelectors) { const companyEl = el.querySelector(sel); if (companyEl && companyEl.textContent.trim()) { company = companyEl.textContent.trim(); break; } } for (const sel of locationSelectors) { const locationEl = el.querySelector(sel); if (locationEl && locationEl.textContent.trim()) { location = locationEl.textContent.trim(); break; } } for (const sel of timeSelectors) { const timeEl = el.querySelector(sel); if (timeEl && timeEl.textContent.trim()) { postedTime = timeEl.textContent.trim(); break; } } // Get job link let jobLink = ''; const linkEl = el.tagName === 'A' ? el : el.querySelector('a[href*="job"], a[href*="position"], a'); if (linkEl && linkEl.href) { jobLink = linkEl.href; } // Get company link let companyLink = ''; const companyLinkEl = el.querySelector('a[href*="company"], a[href*="employer"]'); if (companyLinkEl && companyLinkEl.href) { companyLink = companyLinkEl.href; } // If no title found, use link text if (!title && linkEl) { title = linkEl.textContent.trim(); } const job = { title: title.substring(0, 500), company: company.substring(0, 200), location: location.substring(0, 200), jobLink: jobLink, postedTime: postedTime.substring(0, 100), companyLink: companyLink }; // Only add if we have at least a title and link if (job.title && job.jobLink) { jobs.push(job); } } catch (error) { console.error('Error extracting job ' + index + ':', error); } }); // Remove duplicates based on jobLink const seen = new Set(); return jobs.filter(job => { if (seen.has(job.jobLink)) return false; seen.add(job.jobLink); return true; }); })() `, &jobData) if err != nil { return nil, fmt.Errorf("failed to extract job data: %w", err) } // Convert to ScrapedJob structs var jobs []models.ScrapedJob for _, data := range jobData { job := models.ScrapedJob{ Title: data["title"], Company: data["company"], CompanyLink: data["companyLink"], Location: data["location"], JobLink: data["jobLink"], PostedTime: data["postedTime"], Criteria: make(map[string]string), } jobs = append(jobs, job) } return jobs, nil } // getJobDetails navigates to a job page and extracts detailed information func (s *HiringCafeScraper) getJobDetails(job models.ScrapedJob) (models.ScrapedJob, error) { if job.JobLink == "" { return job, nil } // Navigate to job detail page if err := s.browser.Navigate(job.JobLink); err != nil { return job, fmt.Errorf("failed to navigate to job page: %w", err) } s.browser.Sleep(2 * time.Second) s.browser.SimulateHumanBehavior() // Extract job description var description string s.browser.Evaluate(` (function() { const descSelectors = [ '[class*="description"]', '[class*="Description"]', '[class*="job-content"]', '[class*="job-details"]', '[class*="details"]', '[id*="description"]', 'article', 'main', '.content' ]; for (const selector of descSelectors) { const el = document.querySelector(selector); if (el && el.textContent.trim().length > 100) { return el.textContent.trim().substring(0, 10000); } } // Fallback: get body text return document.body.innerText.substring(0, 10000); })() `, &description) // Extract criteria var criteriaData map[string]string s.browser.Evaluate(` (function() { const criteria = {}; // Look for common job criteria patterns const patterns = [ {key: 'Experience', selectors: ['[class*="experience"]', '[class*="Experience"]']}, {key: 'Education', selectors: ['[class*="education"]', '[class*="Education"]']}, {key: 'Skills', selectors: ['[class*="skills"]', '[class*="Skills"]']}, {key: 'Salary', selectors: ['[class*="salary"]', '[class*="Salary"]', '[class*="compensation"]']}, {key: 'JobType', selectors: ['[class*="job-type"]', '[class*="employment"]', '[class*="type"]']}, ]; patterns.forEach(({key, selectors}) => { for (const selector of selectors) { const el = document.querySelector(selector); if (el && el.textContent.trim()) { criteria[key] = el.textContent.trim().substring(0, 500); break; } } }); return criteria; })() `, &criteriaData) job.Description = description if criteriaData != nil { job.Criteria = criteriaData } return job, nil } // saveJobs saves the scraped jobs to the database func (s *HiringCafeScraper) saveJobs(scrapedJobs []models.ScrapedJob) error { if len(scrapedJobs) == 0 { return nil } var domainJobs []domain.Job var jobDescriptions []domain.JobDescription for _, sj := range scrapedJobs { postTime := parsePostedTime(sj.PostedTime) job := domain.Job{ Title: sj.Title, Company: sj.Company, CompanyLink: sj.CompanyLink, Location: sj.Location, JobLink: sj.JobLink, Provider: domain.HiringCafe, JobPostTime: postTime, Status: domain.JobStatusCreated, } domainJobs = append(domainJobs, job) } // Save jobs if err := s.jobRepo.SaveJobs(domainJobs); err != nil { return fmt.Errorf("failed to save jobs: %w", err) } // Collect job descriptions for _, sj := range scrapedJobs { if sj.Description == "" { continue } jobDesc := domain.JobDescription{ Description: sj.Description, Criteria: sj.Criteria, } jobDescriptions = append(jobDescriptions, jobDesc) } log.Printf("Job descriptions collected: %d (saving requires job IDs)", len(jobDescriptions)) return nil } // parsePostedTime converts relative time strings to actual dates func parsePostedTime(timeStr string) *time.Time { now := time.Now() timeStr = strings.ToLower(strings.TrimSpace(timeStr)) if timeStr == "" { return &now } // Parse common patterns if strings.Contains(timeStr, "hour") || strings.HasSuffix(timeStr, "h") { var hours int fmt.Sscanf(timeStr, "%d", &hours) if hours > 0 { t := now.Add(-time.Duration(hours) * time.Hour) return &t } } if strings.Contains(timeStr, "day") || strings.HasSuffix(timeStr, "d") { var days int fmt.Sscanf(timeStr, "%d", &days) if days > 0 { t := now.AddDate(0, 0, -days) return &t } } if strings.Contains(timeStr, "week") || strings.HasSuffix(timeStr, "w") { var weeks int fmt.Sscanf(timeStr, "%d", &weeks) if weeks > 0 { t := now.AddDate(0, 0, -weeks*7) return &t } } if strings.Contains(timeStr, "month") || strings.HasSuffix(timeStr, "m") { var months int fmt.Sscanf(timeStr, "%d", &months) if months > 0 { t := now.AddDate(0, -months, 0) return &t } } return &now }