jobs-monorepo/services/scraper-hiringcafe/api/client.go

package api

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"time"
)

const (
	BaseURL   = "https://hiring.cafe"
	SearchURL = BaseURL + "/api/search-jobs"
	UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
	PageSize  = 20
)

// Client handles API requests to hiring.cafe
type Client struct {
	httpClient *http.Client
}

// NewClient creates a new API client
func NewClient() *Client {
	return &Client{
		httpClient: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

// SearchRequest represents the search request payload (based on RSSHub implementation)
type SearchRequest struct {
	Size        int         `json:"size"`
	Page        int         `json:"page"`
	SearchState SearchState `json:"searchState"`
}

// SearchState contains search parameters
type SearchState struct {
	SearchQuery string `json:"searchQuery"`
	SortBy      string `json:"sortBy"` // date, default, compensation_desc, experience_asc
}

// SearchResponse represents the API response
type SearchResponse struct {
	Results []JobResult `json:"results"`
	Total   int         `json:"total"`
}

// JobResult represents a job result from the API
type JobResult struct {
	ID               string           `json:"id"`
	ApplyURL         string           `json:"apply_url"`
	JobInformation   JobInformation   `json:"job_information"`
	ProcessedJobData ProcessedJobData `json:"v5_processed_job_data"`
}

// JobInformation contains basic job info
type JobInformation struct {
	Title       string `json:"title"`
	Description string `json:"description"`
}

// ProcessedJobData contains processed job details
type ProcessedJobData struct {
	CompanyName                string   `json:"company_name"`
	IsCompensationTransparent  bool     `json:"is_compensation_transparent"`
	YearlyMinCompensation      *float64 `json:"yearly_min_compensation"`
	YearlyMaxCompensation      *float64 `json:"yearly_max_compensation"`
	WorkplaceType              string   `json:"workplace_type"`
	RequirementsSummary        string   `json:"requirements_summary"`
	JobCategory                string   `json:"job_category"`
	RoleActivities             []string `json:"role_activities"`
	FormattedWorkplaceLocation string   `json:"formatted_workplace_location"`
	EstimatedPublishDateMillis string   `json:"estimated_publish_date_millis"`
}

// Job is a simplified job structure for external use
type Job struct {
	ID               string
	ApplyURL         string
	Title            string
	Description      string
	Company          string
	CompanyURL       string
	Location         string
	Remote           bool
	Salary           string
	PostedAt         time.Time
	DescriptionClean string
}

// SearchFilters contains user-configurable search filters
type SearchFilters struct {
	Query         string
	Country       string
	Remote        bool
	Hybrid        bool
	Onsite        bool
	FullTime      bool
	PartTime      bool
	Contract      bool
	Internship    bool
	EntryLevel    bool
	MidLevel      bool
	SeniorLevel   bool
	DateRangeDays int
	MaxPages      int
}

// DefaultFilters returns default search filters
func DefaultFilters() SearchFilters {
	return SearchFilters{
		Query:         "",
		Country:       "United States",
		Remote:        true,
		Hybrid:        true,
		Onsite:        true,
		FullTime:      true,
		PartTime:      false,
		Contract:      false,
		Internship:    false,
		EntryLevel:    true,
		MidLevel:      true,
		SeniorLevel:   false,
		DateRangeDays: 30,
		MaxPages:      10,
	}
}

// Search performs a job search with the given filters
func (c *Client) Search(filters SearchFilters) ([]Job, error) {
	var allJobs []Job
	page := 1

	for {
		jobs, totalCount, err := c.searchPage(filters, page)
		if err != nil {
			return nil, fmt.Errorf("failed to search page %d: %w", page, err)
		}

		allJobs = append(allJobs, jobs...)

		// Check if we've fetched all jobs or reached max pages
		if len(allJobs) >= totalCount || page >= filters.MaxPages || len(jobs) == 0 {
			break
		}

		page++
		// Rate limiting - be nice to the API
		time.Sleep(500 * time.Millisecond)
	}

	return allJobs, nil
}

// searchPage performs a single page search using the correct API format
func (c *Client) searchPage(filters SearchFilters, page int) ([]Job, int, error) {
	reqBody := SearchRequest{
		Size: PageSize,
		Page: page - 1, // API uses 0-based indexing
		SearchState: SearchState{
			SearchQuery: filters.Query,
			SortBy:      "date",
		},
	}

	jsonBody, err := json.Marshal(reqBody)
	if err != nil {
		return nil, 0, fmt.Errorf("failed to marshal request: %w", err)
	}

	log.Printf("Sending request to %s with body: %s", SearchURL, string(jsonBody))

	req, err := http.NewRequest("POST", SearchURL, bytes.NewBuffer(jsonBody))
	if err != nil {
		return nil, 0, fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("User-Agent", UserAgent)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Origin", BaseURL)
	req.Header.Set("Referer", BaseURL+"/")

	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, 0, fmt.Errorf("failed to send request: %w", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, 0, fmt.Errorf("failed to read response: %w", err)
	}

	if resp.StatusCode != http.StatusOK {
		return nil, 0, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body))
	}

	var searchResp SearchResponse
	if err := json.Unmarshal(body, &searchResp); err != nil {
		return nil, 0, fmt.Errorf("failed to unmarshal response: %w", err)
	}

	// Convert JobResult to Job
	var jobs []Job
	for _, result := range searchResp.Results {
		job := Job{
			ID:               result.ID,
			ApplyURL:         result.ApplyURL,
			Title:            result.JobInformation.Title,
			Description:      result.JobInformation.Description,
			Company:          result.ProcessedJobData.CompanyName,
			Location:         result.ProcessedJobData.FormattedWorkplaceLocation,
			Remote:           result.ProcessedJobData.WorkplaceType == "remote",
			DescriptionClean: result.JobInformation.Description,
		}

		// Parse salary if available
		if result.ProcessedJobData.IsCompensationTransparent &&
			result.ProcessedJobData.YearlyMinCompensation != nil &&
			result.ProcessedJobData.YearlyMaxCompensation != nil {
			job.Salary = fmt.Sprintf("$%.0f - $%.0f",
				*result.ProcessedJobData.YearlyMinCompensation,
				*result.ProcessedJobData.YearlyMaxCompensation)
		}

		// Parse posted date
		if result.ProcessedJobData.EstimatedPublishDateMillis != "" {
			// Try to parse as milliseconds timestamp
			var millis int64
			if _, err := fmt.Sscanf(result.ProcessedJobData.EstimatedPublishDateMillis, "%d", &millis); err == nil {
				job.PostedAt = time.UnixMilli(millis)
			}
		}

		jobs = append(jobs, job)
	}

	return jobs, searchResp.Total, nil
}