diff --git a/Makefile b/Makefile index ccf6782..1b462c0 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ API_DIR=./services/api SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright +JOB_APPLIER_BROWSER_USE_DIR=./services/job-applier-browser-use CRON_ANALYZER_DIR=./apps/cron-analyzer CV_ANALYZER_DIR=./apps/cv-analyzer DOCS_DIR=./services/api/pkg/swagger @@ -71,6 +72,18 @@ docker-scraper-playwright: ## Build and run Playwright scraper in Docker @echo "Running Playwright scraper in Docker..." @docker run --rm --name playwright-scraper playwright-scraper +.PHONY: setup-job-applier-browser-use +setup-job-applier-browser-use: ## Setup the browser-use job applier service + @echo "Setting up browser-use job applier service..." + @cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv venv && uv pip install -e . + @echo "Installing Chromium browser..." + @cd $(JOB_APPLIER_BROWSER_USE_DIR) && uvx browser-use install + +.PHONY: run-job-applier-browser-use +run-job-applier-browser-use: ## Run the browser-use job applier service + @echo "Running browser-use job applier service..." + @cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv run python -m src.main + # Apps .PHONY: run-cron-analyzer run-cron-analyzer: ## Run the job analysis cron service diff --git a/apps/cron-analyzer/cmd/cli/main.go b/apps/cron-analyzer/cmd/cli/main.go index 41124c8..1b9b2c4 100644 --- a/apps/cron-analyzer/cmd/cli/main.go +++ b/apps/cron-analyzer/cmd/cli/main.go @@ -4,7 +4,7 @@ import ( "log" "github.com/jobs-scraper/apps/cron-analyzer/internal" - "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/infrastructure" "github.com/joho/godotenv" ) diff --git a/apps/cron-analyzer/go.mod b/apps/cron-analyzer/go.mod index ec40919..7cff5f2 100644 --- a/apps/cron-analyzer/go.mod +++ b/apps/cron-analyzer/go.mod @@ -3,9 +3,9 @@ module github.com/jobs-scraper/apps/cron-analyzer go 1.24.0 require ( - github.com/jobs-scraper/internal/pkg/domain v0.0.0 - github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 - github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/domain v0.0.0 + github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/openrouter v0.0.0-00010101000000-000000000000 github.com/jobs-scraper/libs/repo v0.0.0 github.com/joho/godotenv v1.5.1 ) @@ -20,10 +20,10 @@ require ( github.com/rabbitmq/amqp091-go v1.10.0 // indirect ) -replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain +replace github.com/jobs-scraper/internal/domain => ../../internal/domain -replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure +replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure -replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter +replace github.com/jobs-scraper/internal/openrouter => ../../internal/openrouter replace github.com/jobs-scraper/libs/repo => ../../libs/repo diff --git a/apps/cron-analyzer/internal/analyze-jobs.go b/apps/cron-analyzer/internal/analyze-jobs.go index d04d3fe..aca8d45 100644 --- a/apps/cron-analyzer/internal/analyze-jobs.go +++ b/apps/cron-analyzer/internal/analyze-jobs.go @@ -8,10 +8,10 @@ import ( // "path/filepath" - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" - "github.com/jobs-scraper/internal/pkg/openai" - "github.com/jobs-scraper/internal/pkg/openrouter" + "github.com/jobs-scraper/internal/domain" + "github.com/jobs-scraper/internal/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/openai" + "github.com/jobs-scraper/internal/openrouter" "github.com/jobs-scraper/libs/repo" ) diff --git a/go.work b/go.work index 998a0ea..782eee9 100644 --- a/go.work +++ b/go.work @@ -3,12 +3,12 @@ go 1.24.0 use ( . ./apps/cron-analyzer - ./internal/pkg/domain - ./internal/pkg/gemini - ./internal/pkg/infrastructure - ./internal/pkg/openai - ./internal/pkg/openrouter - ./internal/pkg/utils + ./internal/domain + ./internal/gemini + ./internal/infrastructure + ./internal/openai + ./internal/openrouter + ./internal/utils ./libs/ports ./libs/repo ./libs/server diff --git a/internal/pkg/browser/browser.go b/internal/browser/browser.go similarity index 58% rename from internal/pkg/browser/browser.go rename to internal/browser/browser.go index d8f4a62..47fe16d 100644 --- a/internal/pkg/browser/browser.go +++ b/internal/browser/browser.go @@ -5,9 +5,11 @@ import ( "log" "math/rand" "os/exec" + // "strings" "time" "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/cdproto/dom" "github.com/chromedp/cdproto/page" "github.com/chromedp/chromedp" ) @@ -116,7 +118,8 @@ func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) { var nodes []*cdp.Node err := chromedp.Run(b.ctx, - chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), + chromedp.WaitVisible(selector, chromedp.ByQueryAll), + chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll), ) if err != nil { @@ -126,6 +129,22 @@ func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) { return nodes, nil } +// GetOuterHTML gets the outer HTML of the first element matching the selector +func (b *Browser) GetOuterHTML(selector string) (string, error) { + var html string + + err := chromedp.Run(b.ctx, + chromedp.WaitVisible(selector, chromedp.ByQuery), + chromedp.OuterHTML(selector, &html, chromedp.ByQuery), + ) + + if err != nil { + return "", err + } + + return html, nil +} + func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) { var text string @@ -151,6 +170,8 @@ func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr str return "", err } + // value = strings.ReplaceAll(value, "/apply", "") + return value, nil } @@ -229,13 +250,154 @@ func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error { } } +func (b *Browser) NodeToString(node *cdp.Node) (string, error) { + var outerHTML string + + err := chromedp.Run(b.ctx, + chromedp.ActionFunc(func(ctx context.Context) error { + h, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx) + if err != nil { + return err + } + outerHTML = h + return nil + }), + ) + + if err != nil { + return "", err + } + + return outerHTML, err +} + +func (b *Browser) Type(selector string, value string) error { + err := chromedp.Run(b.ctx, + chromedp.WaitVisible(selector), + chromedp.SendKeys(selector, value), + ) + if err != nil { + return err + } + + return nil +} + +func (b *Browser) Click(selector string) error { + err := chromedp.Run(b.ctx, + chromedp.WaitVisible(selector), + chromedp.Click(selector), + ) + if err != nil { + return err + } + + return nil +} + +// Select chooses an option from a element by value + */ + async select(selector: string, value: string): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + await page.select(selector, value); + } + + /** + * Set a checkbox to checked or unchecked state + */ + async setCheckbox(selector: string, checked: boolean): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + + const isChecked = await page.$eval( + selector, + (el) => (el as HTMLInputElement).checked + ); + + if (isChecked !== checked) { + await page.click(selector); + } + } + + /** + * Click a radio button to select it + */ + async setRadio(selector: string): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + await page.click(selector); + } + + /** + * Upload a file to a file input + */ + async uploadFile(selector: string, filePath: string): Promise { + const page = this.getPage(); + await this.waitReady(selector); + + const input = (await page.$(selector)) as ElementHandle | null; + if (!input) { + throw new Error(`File input not found: ${selector}`); + } + + await input.uploadFile(filePath); + } + + /** + * Set the value of an input field directly (useful for hidden, date, color, etc.) + */ + async setValue(selector: string, value: string): Promise { + const page = this.getPage(); + await this.waitReady(selector); + + await page.$eval( + selector, + (el, val) => { + (el as HTMLInputElement).value = val; + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + }, + value + ); + } + + /** + * Clear the value of an input field + */ + async clear(selector: string): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + + // Triple-click to select all text, then delete + await page.click(selector, { clickCount: 3 }); + await page.keyboard.press("Backspace"); + } + + /** + * Focus on an element + */ + async focus(selector: string): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + await page.focus(selector); + } + + /** + * Take a screenshot + */ + async screenshot(path?: string): Promise { + const page = this.getPage(); + const buffer = await page.screenshot({ path }); + return buffer as Buffer; + } + + /** + * Random delay between min and max milliseconds + */ + async randomDelay(minMs: number, maxMs: number): Promise { + await randomDelay(minMs, maxMs); + } + + /** + * Scroll to an element + */ + async scrollToElement(selector: string): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + await page.$eval(selector, (el) => { + el.scrollIntoView({ behavior: "smooth", block: "center" }); + }); + } + + /** + * Check if an element exists + */ + async exists(selector: string): Promise { + const page = this.getPage(); + const element = await page.$(selector); + return element !== null; + } + + /** + * Get the count of elements matching a selector + */ + async count(selector: string): Promise { + const page = this.getPage(); + const elements = await page.$$(selector); + return elements.length; + } + + /** + * Press a keyboard key + */ + async pressKey(key: string): Promise { + const page = this.getPage(); + await page.keyboard.press(key as any); + } + + /** + * Triple-click to select all text in an input + */ + async selectAllText(selector: string): Promise { + const page = this.getPage(); + await this.waitVisible(selector); + await page.click(selector, { clickCount: 3 }); + } + + /** + * Clear and type new value (common pattern) + */ + async clearAndType(selector: string, value: string): Promise { + await this.clear(selector); + await this.type(selector, value); + } +} diff --git a/libs/browser-automation/src/index.ts b/libs/browser-automation/src/index.ts new file mode 100644 index 0000000..b8f71b2 --- /dev/null +++ b/libs/browser-automation/src/index.ts @@ -0,0 +1,18 @@ +export { Browser, BrowserOptions } from "./browser"; +export { + launchChromeWithDebugging, + waitForChromeReady, + getChromeWebSocketUrl, + ChromeVersionResponse, +} from "./utils/chrome-launcher"; +export { randomDelay, sleep } from "./utils/helpers"; +export { + OpenRouterService, + OpenRouterServiceOptions, + FieldType, + FormField, + FormExtractionResult, + JobAnalysisResult, + JobDescription, + getMockFormData, +} from "./openrouter"; diff --git a/libs/browser-automation/src/openrouter/index.ts b/libs/browser-automation/src/openrouter/index.ts new file mode 100644 index 0000000..ae7ffaf --- /dev/null +++ b/libs/browser-automation/src/openrouter/index.ts @@ -0,0 +1,12 @@ +export { OpenRouterService, OpenRouterServiceOptions } from "./openrouter"; +export { + FieldType, + FormField, + FormExtractionResult, + JobAnalysisResult, + JobDescription, + OpenRouterMessage, + OpenRouterRequest, + OpenRouterResponse, +} from "./types"; +export { getMockFormData } from "./mock-data"; diff --git a/libs/browser-automation/src/openrouter/mock-data.ts b/libs/browser-automation/src/openrouter/mock-data.ts new file mode 100644 index 0000000..d1492d6 --- /dev/null +++ b/libs/browser-automation/src/openrouter/mock-data.ts @@ -0,0 +1,102 @@ +import { FormExtractionResult } from "./types"; + +/** + * Returns mock form extraction result for testing + */ +export function getMockFormData(): FormExtractionResult { + return { + fields: [ + { + label: "Resume", + field_name: "resume", + field_type: "file", + value: "", + placeholder: "", + required: true, + selector: "input[data-ui='resume']", + }, + { + label: "Cover letter", + field_name: "cover_letter", + field_type: "textarea", + value: + "Dear KOMOJU Hiring Team,\n\nI am excited to apply for the Fullstack Developer position at KOMOJU. With over 3 years of experience building web applications and leading development projects, I am confident I can contribute to your Fraud Prevention Team's mission of protecting merchants and customers across borders.\n\nMy experience includes leading the development of online shopping platforms, mentoring junior developers, and coordinating complex projects. I have hands-on experience with modern web technologies including React, Node.js, and microservices architecture. I am a motivated self-starter who can work independently while collaborating effectively with cross-functional teams.\n\nI am eager to bring my technical skills and passion for product-focused development to KOMOJU's innovative payment gateway solutions.\n\nThank you for your consideration.\n\nBest regards,\nZiad Elshimy", + placeholder: "", + required: false, + selector: "#cover_letter", + }, + { + label: + "Do you have at least 2 years of personal or professional experience with Ruby?", + field_name: "QA_10609776", + field_type: "radio", + value: "false", + placeholder: "", + required: true, + selector: "input[name='QA_10609776']", + }, + { + label: "Could you tell us more about your Ruby experience?", + field_name: "QA_10609777", + field_type: "textarea", + value: + "I do not have professional Ruby experience, but I have extensive experience with JavaScript (React, Node.js), TypeScript, and Go. I am a fast learner with strong fundamentals in software development best practices, data structures, and algorithms. I am excited about the opportunity to learn Ruby and contribute to KOMOJU's fraud prevention systems.", + placeholder: "", + required: true, + selector: "#QA_10609777", + }, + { + label: + "Have you ever investigated and resolved a production issue that wasn't reproducible in a development or staging environment?", + field_name: "QA_10609885", + field_type: "radio", + value: "true", + placeholder: "", + required: true, + selector: "input[name='QA_10609885']", + }, + { + label: + "If the answer for above question is yes, can you tell us how did you approach it?", + field_name: "QA_10609886", + field_type: "textarea", + value: + "Yes, I have experience debugging production issues. My approach includes: 1) Analyzing production logs and error reports to identify patterns, 2) Using monitoring tools to track system behavior and performance metrics, 3) Creating targeted tests to reproduce the issue in isolated environments, 4) Collaborating with team members to validate hypotheses, and 5) Implementing and deploying fixes with proper testing and monitoring. I understand the importance of being systematic and thorough when diagnosing complex production issues.", + placeholder: "", + required: true, + selector: "#QA_10609886", + }, + { + label: "Do you currently reside in Japan?", + field_name: "QA_10609774", + field_type: "radio", + value: "false", + placeholder: "", + required: true, + selector: "input[name='QA_10609774']", + }, + { + label: + "This role requires at least 5 hours of overlap with Japan Standard Time (JST) business hours. Are you able to accommodate this?", + field_name: "QA_10609887", + field_type: "radio", + value: "true", + placeholder: "", + required: true, + selector: "input[name='QA_10609887']", + }, + { + label: + "Are you willing to relocate to Japan? If so, what are your motivations for doing so?", + field_name: "QA_10609888", + field_type: "textarea", + value: + "Yes, I am willing to relocate to Japan. My motivations include: 1) Joining KOMOJU's innovative team working on cutting-edge payment technology, 2) Experiencing Japan's world-class technology culture and work ethic, 3) Contributing to a product that powers payments for major platforms like Steam and TikTok, 4) Growing my career in an international environment with diverse perspectives, and 5) Embracing the opportunity to learn Japanese culture and language while working on globally impactful projects.", + placeholder: "", + required: true, + selector: "#QA_10609888", + }, + ], + apply_button: `button[data-ui="apply-button"]`, + }; +} diff --git a/libs/browser-automation/src/openrouter/openrouter.ts b/libs/browser-automation/src/openrouter/openrouter.ts new file mode 100644 index 0000000..a797380 --- /dev/null +++ b/libs/browser-automation/src/openrouter/openrouter.ts @@ -0,0 +1,256 @@ +import { + FormExtractionResult, + JobAnalysisResult, + JobDescription, + OpenRouterRequest, + OpenRouterResponse, +} from "./types"; + +export interface OpenRouterServiceOptions { + model: string; + apiKey: string; +} + +export class OpenRouterService { + private model: string; + private apiKey: string; + private baseUrl = "https://openrouter.ai/api/v1/chat/completions"; + + constructor(options: OpenRouterServiceOptions) { + this.model = options.model; + this.apiKey = options.apiKey; + } + + /** + * Clean markdown code blocks from response + */ + private cleanMarkdownCodeBlocks(content: string): string { + content = content.trim(); + + if (content.startsWith("```json")) { + content = content.slice(7); + } else if (content.startsWith("```")) { + content = content.slice(3); + } + + if (content.endsWith("```")) { + content = content.slice(0, -3); + } + + return content.trim(); + } + + /** + * Make a request to OpenRouter API + */ + private async makeRequest( + systemMessage: string, + userMessage: string + ): Promise { + const request: OpenRouterRequest = { + model: this.model, + messages: [ + { role: "system", content: systemMessage }, + { role: "user", content: userMessage }, + ], + }; + + const response = await fetch(this.baseUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${this.apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(request), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `OpenRouter API error: ${response.status} - ${errorText}` + ); + } + + const data = (await response.json()) as OpenRouterResponse; + + if (!data.choices || data.choices.length === 0) { + throw new Error("No response choices received from API"); + } + + const content = data.choices[0].message.content; + + if (content.startsWith("<")) { + throw new Error( + `API returned HTML/XML instead of JSON: ${content.slice(0, 200)}` + ); + } + + return this.cleanMarkdownCodeBlocks(content); + } + + /** + * Analyze a job description against a CV + */ + async analyzeJobDescription( + cv: string, + jobDesc: JobDescription + ): Promise { + const userMessage = `Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below. +1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv. +for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills. + +2) The job shouldn't require any language skills, preferbly only english. + +3) The job should be remote, or provide relocation to the country. + +CV: +${cv} + +Job Description: +${jobDesc.description} + +Job Criteria (key-value): +${JSON.stringify(jobDesc.criteria, null, 2)} + +CRITICAL OUTPUT REQUIREMENTS: +- Return ONLY raw JSON - NO markdown formatting whatsoever +- NO backticks, NO code blocks, NO json prefix +- NO additional text before or after the JSON +- Start your response directly with { and end with } +- Use this exact schema and key names: +{ + "recommendation": "apply" | "do_not_apply", + "confidence_score": number, // integer 0-100 + "matching_skills": [string], + "missing_skills": [string], + "experience_match": "excellent" | "good" | "fair" | "poor", + "summary": string, + "improvement_suggestions": [string] +}`; + + const systemMessage = + "You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }."; + + const jsonContent = await this.makeRequest(systemMessage, userMessage); + + console.log("Raw API response:", jsonContent); + + const result = JSON.parse(jsonContent) as JobAnalysisResult; + return result; + } + + /** + * Create a job description from HTML content + */ + async createJobDescription(htmlContent: string): Promise { + const userMessage = `You are a job posting data extractor. Analyze the HTML content and extract data according to this JSON structure: +{"description":"job description text","criteria":{"title":"Job Title","company":"Company","location":"Location","salary":"Salary","skills":"Skills","experience":"Experience","job_type":"Job Type","remote":"Remote"}} + +CRITICAL RULES: +1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations +2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON) +3. All criteria values MUST be strings (never use booleans, numbers, or arrays) +4. Properly escape all quotes inside strings using \\" +5. Use only standard ASCII quotes ("), never smart quotes (" " ' ') +7. If no job description is found, use empty string for "description": "" +8. All criteria fields must be present with empty string "" if not found + +Example output: +{"description":"Develop software applications...","criteria":{"title":"Backend Engineer","company":"Tech Corp","location":"Remote","salary":"$120k","skills":"Go, Docker","experience":"3+ years","job_type":"Full-time","remote":"Yes"}} + +Extract from this HTML Content: +${htmlContent}`; + + const systemMessage = + "You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }."; + + const jsonContent = await this.makeRequest(systemMessage, userMessage); + + console.log("Raw API response for job description:", jsonContent); + + const result = JSON.parse(jsonContent) as JobDescription; + return result; + } + + /** + * Extract form fields from HTML and populate them based on CV data. + * Also adjusts the CV based on job type (frontend/backend/fullstack). + */ + async applyForJob( + htmlForm: string, + cvMarkdown: string, + jobDescription: string + ): Promise { + const userMessage = `You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then: +1. Extract ALL form fields and populate them with appropriate values from the CV + +HTML FORM: +${htmlForm} + +JOB DESCRIPTION: +${jobDescription} + +ORIGINAL CV (Markdown): +${cvMarkdown} + +CV ADJUSTMENT RULES: +- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description +- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content. +- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content. +- If FULLSTACK: Keep both frontend and backend content. +- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default. +- Maintain the same markdown structure and formatting as the original CV. +- Do NOT invent new skills or experience - only filter existing content. + +CRITICAL RULES: +1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations +2. Extract ALL input fields, textareas, selects, and buttons from the form +3. For each field, determine the appropriate value from the ADJUSTED CV +4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string +5. Identify the apply/submit button text +6. Use the exact JSON schema below +7. If a field is already populated, ignore it (don't send it in the results) + +JSON Schema: +{ + "fields": [ + { + "label": "Field label or name attribute", + "field_name": "name or id attribute of the input", + "field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset", + "value": "Value to populate based on CV data", + "placeholder": "Placeholder text if any", + "required": true|false, + "selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined." + } + ], + "apply_button": "Unique CSS selector for the form's submit button", +} + +FIELD MAPPING GUIDELINES: +- Name fields: Extract full name, first name, last name from CV +- Email: Use email from CV contact info +- Phone: Use phone number from CV +- LinkedIn/Portfolio/Website: Use URLs from CV +- Experience/Years: Calculate from CV work history +- Current company/title: Use most recent from CV +- Skills: List relevant skills from ADJUSTED CV +- Education: Use education details from CV +- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV +- Salary expectations: Leave empty unless specified in CV +- Location/Address: Use from CV contact info +- Resume/CV upload: Leave value empty (file upload) + +Start your response with { and end with }`; + + const systemMessage = + "You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }."; + + const jsonContent = await this.makeRequest(systemMessage, userMessage); + + console.log("Raw API response for form extraction:", jsonContent); + + const result = JSON.parse(jsonContent) as FormExtractionResult; + return result; + } +} diff --git a/libs/browser-automation/src/openrouter/types.ts b/libs/browser-automation/src/openrouter/types.ts new file mode 100644 index 0000000..be432fc --- /dev/null +++ b/libs/browser-automation/src/openrouter/types.ts @@ -0,0 +1,104 @@ +/** + * FieldType represents HTML input field types + */ +export type FieldType = + | "text" + | "email" + | "tel" + | "password" + | "number" + | "date" + | "datetime-local" + | "time" + | "month" + | "week" + | "url" + | "search" + | "color" + | "range" + | "file" + | "hidden" + | "checkbox" + | "radio" + | "select" + | "textarea" + | "button" + | "submit" + | "reset"; + +/** + * FormField represents a single form field extracted from HTML + */ +export interface FormField { + label: string; + field_name: string; + field_type: FieldType; + value: string; + placeholder?: string; + required: boolean; + selector: string; +} + +/** + * FormExtractionResult represents the structured response from form extraction + */ +export interface FormExtractionResult { + fields: FormField[]; + apply_button: string; +} + +/** + * JobAnalysisResult represents the structured response from job analysis + */ +export interface JobAnalysisResult { + recommendation: "apply" | "do_not_apply"; + confidence_score: number; + matching_skills: string[]; + missing_skills: string[]; + experience_match: "excellent" | "good" | "fair" | "poor"; + summary: string; + improvement_suggestions: string[]; +} + +/** + * JobDescription represents a parsed job description + */ +export interface JobDescription { + description: string; + criteria: Record; +} + +/** + * OpenRouter API message format + */ +export interface OpenRouterMessage { + role: "system" | "user" | "assistant"; + content: string; +} + +/** + * OpenRouter API request body + */ +export interface OpenRouterRequest { + model: string; + messages: OpenRouterMessage[]; +} + +/** + * OpenRouter API response + */ +export interface OpenRouterResponse { + id: string; + choices: { + message: { + role: string; + content: string; + }; + finish_reason: string; + }[]; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; +} diff --git a/libs/browser-automation/src/utils/chrome-launcher.ts b/libs/browser-automation/src/utils/chrome-launcher.ts new file mode 100644 index 0000000..dbc4d8c --- /dev/null +++ b/libs/browser-automation/src/utils/chrome-launcher.ts @@ -0,0 +1,144 @@ +import fs from "fs"; +import path from "path"; +import os from "os"; +import { spawn, ChildProcess } from "child_process"; +import net from "net"; + +export interface ChromeVersionResponse { + webSocketDebuggerUrl: string; +} + +const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile"); + +function getChromePath(): string { + const platform = os.platform(); + + const paths: Record = { + darwin: [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ], + win32: [ + process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe", + process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe", + process.env["PROGRAMFILES(X86)"] + + "\\Google\\Chrome\\Application\\chrome.exe", + ], + linux: [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + ], + aix: [], + freebsd: [], + openbsd: [], + sunos: [], + android: [], + haiku: [], + cygwin: [], + netbsd: [], + }; + + const platformPaths = paths[platform] || []; + + for (const chromePath of platformPaths) { + try { + if (fs.existsSync(chromePath)) { + return chromePath; + } + } catch { + continue; + } + } + + throw new Error("Could not find Chrome/Chromium installation"); +} + +export function launchChromeWithDebugging( + port: number = 9222, + headless: boolean = false +): ChildProcess { + try { + const chromePath = getChromePath(); + const args = [ + `--remote-debugging-port=${port}`, + `--user-data-dir=${userDataDir}`, + "--remote-allow-origins=*", + "--incognito", + ]; + + if (headless) { + args.push("--headless=new"); + } + + console.log(`Launching Chrome at: ${chromePath}`); + + const chromeProcess = spawn(chromePath, args, { + detached: true, + stdio: "ignore", + }); + + chromeProcess.unref(); + + chromeProcess.on("error", (err) => { + console.error("Failed to start Chrome:", err); + }); + + chromeProcess.on("exit", (code) => { + if (code !== 0) { + console.error(`Chrome process exited with code ${code}`); + } + }); + + console.log(`Chrome launched successfully with debugging port ${port}`); + return chromeProcess; + } catch (error) { + console.error("Failed to launch Chrome:", error); + throw error; + } +} + +export async function waitForChromeReady( + port: number = 9222, + timeout: number = 30000 +): Promise { + const startTime = Date.now(); + + return new Promise((resolve, reject) => { + function check() { + const client = net.createConnection({ port }, () => { + client.end(); + resolve(); + }); + + client.on("error", () => { + if (Date.now() - startTime > timeout) { + reject(new Error("Timeout waiting for Chrome to start")); + } else { + setTimeout(check, 500); + } + }); + } + + check(); + }); +} + +export async function getChromeWebSocketUrl( + port: number = 9222 +): Promise { + const response = await fetch(`http://localhost:${port}/json/version`, { + headers: { + Origin: "", + }, + }); + + if (!response.ok) { + throw new Error("Failed to connect to Chrome debugging port"); + } + + const chromeInstance = (await response.json()) as ChromeVersionResponse; + return chromeInstance.webSocketDebuggerUrl; +} diff --git a/libs/browser-automation/src/utils/helpers.ts b/libs/browser-automation/src/utils/helpers.ts new file mode 100644 index 0000000..8a31c84 --- /dev/null +++ b/libs/browser-automation/src/utils/helpers.ts @@ -0,0 +1,14 @@ +/** + * Random delay between min and max milliseconds + */ +export function randomDelay(minMs: number, maxMs: number): Promise { + const delay = Math.floor(Math.random() * (maxMs - minMs) + minMs); + return new Promise((resolve) => setTimeout(resolve, delay)); +} + +/** + * Sleep for a specified number of milliseconds + */ +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/libs/browser-automation/tsconfig.json b/libs/browser-automation/tsconfig.json new file mode 100644 index 0000000..2c37fd3 --- /dev/null +++ b/libs/browser-automation/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020", "DOM"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "removeComments": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "exactOptionalPropertyTypes": false, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": false, + "noUncheckedIndexedAccess": false + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ] +} diff --git a/libs/ports/go.mod b/libs/ports/go.mod index 1bb16ea..e1bee67 100644 --- a/libs/ports/go.mod +++ b/libs/ports/go.mod @@ -4,6 +4,6 @@ go 1.24.0 toolchain go1.24.7 -require github.com/jobs-scraper/internal/pkg/domain v0.0.0 +require github.com/jobs-scraper/internal/domain v0.0.0 -replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain +replace github.com/jobs-scraper/internal/domain => ../../internal/domain diff --git a/libs/ports/job-commands.go b/libs/ports/job-commands.go index 9a834c0..32988b9 100644 --- a/libs/ports/job-commands.go +++ b/libs/ports/job-commands.go @@ -1,9 +1,10 @@ package ports -import "github.com/jobs-scraper/internal/pkg/domain" +import "github.com/jobs-scraper/internal/domain" type JobCommands struct { - CreateJob JobCommandHandler + CreateJob JobCommandHandler + ApplyForJob ApplyForJobCommandHandler } // CreateJobCommand represents the command to create a job @@ -19,3 +20,7 @@ type CreateJobCommand struct { type JobCommandHandler interface { Handle(cmd CreateJobCommand) error } + +type ApplyForJobCommandHandler interface { + Handle(jobId int) error +} diff --git a/libs/ports/job-queries.go b/libs/ports/job-queries.go index 6a37de0..9f6886c 100644 --- a/libs/ports/job-queries.go +++ b/libs/ports/job-queries.go @@ -2,7 +2,7 @@ package ports import ( "github.com/jobs-scraper/internal/dto" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/domain" ) type JobQueries struct { diff --git a/libs/rabbitmq-ts/package-lock.json b/libs/rabbitmq-ts/package-lock.json new file mode 100644 index 0000000..8c962f7 --- /dev/null +++ b/libs/rabbitmq-ts/package-lock.json @@ -0,0 +1,102 @@ +{ + "name": "@jobs-scraper/rabbitmq-ts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@jobs-scraper/rabbitmq-ts", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "amqplib": "^0.10.9" + }, + "devDependencies": { + "@types/amqplib": "^0.10.8", + "typescript": "^5.9.3" + } + }, + "node_modules/@types/amqplib": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz", + "integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "25.0.3", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.3.tgz", + "integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/amqplib": { + "version": "0.10.9", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz", + "integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==", + "license": "MIT", + "dependencies": { + "buffer-more-ints": "~1.0.0", + "url-parse": "~1.5.10" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==", + "license": "MIT" + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "license": "MIT" + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "license": "MIT", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + } + } +} diff --git a/libs/rabbitmq-ts/package.json b/libs/rabbitmq-ts/package.json new file mode 100644 index 0000000..892b739 --- /dev/null +++ b/libs/rabbitmq-ts/package.json @@ -0,0 +1,21 @@ +{ + "name": "@jobs-scraper/rabbitmq-ts", + "version": "1.0.0", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "amqplib": "^0.10.9" + }, + "devDependencies": { + "@types/amqplib": "^0.10.8", + "typescript": "^5.9.3" + } +} diff --git a/libs/rabbitmq-ts/src/index.ts b/libs/rabbitmq-ts/src/index.ts new file mode 100644 index 0000000..c17d39b --- /dev/null +++ b/libs/rabbitmq-ts/src/index.ts @@ -0,0 +1,173 @@ +import * as amqp from "amqplib"; + +export interface RabbitMQConfig { + url: string; + queueName: string; + exchangeName: string; + exchangeType: "direct" | "topic" | "fanout" | "headers"; + durable: boolean; +} + +export interface SearchQuery { + keywords: string; + location: string; + fwt?: string; + numPages: number; +} + +export class RabbitMQClient { + private connection: amqp.ChannelModel | null = null; + private channel: amqp.Channel | null = null; + private config: RabbitMQConfig; + + constructor(config: RabbitMQConfig) { + this.config = config; + } + + async connect(): Promise { + try { + console.log(`Connecting to RabbitMQ at: ${this.config.url}`); + this.connection = await amqp.connect(this.config.url); + console.log("Connected to RabbitMQ successfully"); + + if (!this.connection) { + throw new Error("Failed to establish RabbitMQ connection"); + } + + this.channel = await this.connection.createChannel(); + + if (!this.channel) { + throw new Error("Failed to create RabbitMQ channel"); + } + + // Ensure exchange and queue exist + await this.channel.assertExchange( + this.config.exchangeName, + this.config.exchangeType, + { durable: this.config.durable } + ); + + await this.channel.assertQueue(this.config.queueName, { + durable: this.config.durable, + arguments: { + "x-dead-letter-exchange": "scraper_dlx", + "x-dead-letter-routing-key": this.config.queueName, + "x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds + "x-max-retries": 3, + }, + }); + + await this.channel.bindQueue( + this.config.queueName, + this.config.exchangeName, + this.config.queueName + ); + + // Set prefetch to process one message at a time + await this.channel.prefetch(1); + + // Handle connection events + this.connection.on("close", () => { + console.log("RabbitMQ connection closed"); + this.connection = null; + this.channel = null; + }); + + this.connection.on("error", (error: Error) => { + console.error("RabbitMQ connection error:", error); + this.connection = null; + this.channel = null; + }); + } catch (error) { + console.error("Failed to connect to RabbitMQ:", error); + throw error; + } + } + + async subscribe( + messageHandler: (message: T) => Promise + ): Promise { + if (!this.channel) { + throw new Error( + "RabbitMQ channel not initialized. Call connect() first." + ); + } + + console.log( + `Waiting for messages from ${this.config.queueName}. To exit press CTRL+C` + ); + + await this.channel.consume( + this.config.queueName, + async (msg: amqp.ConsumeMessage | null) => { + if (msg && this.channel) { + try { + const messageContent = msg.content.toString(); + console.log(`Received message: ${messageContent}`); + + const message: T = JSON.parse(messageContent); + console.log(`Processing message:`, message); + + // Process the message using the provided handler + await messageHandler(message); + + // Acknowledge the message on success + this.channel.ack(msg); + console.log("Message processed successfully"); + } catch (error) { + console.error("Error processing message:", error); + // Reject the message and don't requeue it + if (this.channel) { + this.channel.nack(msg, false, false); + } + } + } + } + ); + } + + async close(): Promise { + try { + if (this.channel) { + try { + await this.channel.close(); + } catch { + // Channel may already be closing + } + this.channel = null; + } + if (this.connection) { + try { + await this.connection.close(); + } catch { + // Connection may already be closing + } + this.connection = null; + } + console.log("RabbitMQ connection closed gracefully"); + } catch (error) { + console.error("Error closing RabbitMQ connection:", error); + } + } + + isConnected(): boolean { + return this.connection !== null && this.channel !== null; + } +} + +export function createRabbitMQConfig(): RabbitMQConfig { + return { + url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/", + queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor", + exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange", + exchangeType: "topic", + durable: true, + }; +} + +export async function createRabbitMQClient(): Promise { + const config = createRabbitMQConfig(); + const client = new RabbitMQClient(config); + await client.connect(); + return client; +} diff --git a/libs/rabbitmq-ts/tsconfig.json b/libs/rabbitmq-ts/tsconfig.json new file mode 100644 index 0000000..86d80d6 --- /dev/null +++ b/libs/rabbitmq-ts/tsconfig.json @@ -0,0 +1,29 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020", "DOM"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "removeComments": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "exactOptionalPropertyTypes": false, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": false, + "noUncheckedIndexedAccess": false + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/libs/repo/go.mod b/libs/repo/go.mod index 18e388c..c134728 100644 --- a/libs/repo/go.mod +++ b/libs/repo/go.mod @@ -5,8 +5,8 @@ go 1.24.0 toolchain go1.24.7 require ( - github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/domain v0.0.0 github.com/lib/pq v1.10.9 ) -replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain +replace github.com/jobs-scraper/internal/domain => ../../internal/domain diff --git a/libs/repo/job-analysis-result.go b/libs/repo/job-analysis-result.go index 8ba0f28..8f46e37 100644 --- a/libs/repo/job-analysis-result.go +++ b/libs/repo/job-analysis-result.go @@ -4,7 +4,7 @@ import ( "database/sql" "fmt" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/domain" "github.com/lib/pq" ) diff --git a/libs/repo/job-description.go b/libs/repo/job-description.go index f1cbc68..24c0b30 100644 --- a/libs/repo/job-description.go +++ b/libs/repo/job-description.go @@ -6,7 +6,7 @@ import ( "fmt" "strings" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/domain" ) type JobDescriptionRepository struct { diff --git a/libs/repo/job.go b/libs/repo/job.go index 650f4d7..1d923ea 100644 --- a/libs/repo/job.go +++ b/libs/repo/job.go @@ -3,11 +3,12 @@ package repo import ( "bytes" "database/sql" + "encoding/json" "fmt" "strconv" "strings" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/domain" ) type JobRepository struct { @@ -104,7 +105,7 @@ type JobFilter struct { } func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) { - query := "SELECT id, title, company, company_link, location, job_link, job_timestamp FROM jobs" + query := "SELECT id, title, company, company_link, location, job_link, job_timestamp, provider FROM jobs" countQuery := "SELECT COUNT(*) FROM jobs" var conditions []string var args []interface{} @@ -164,7 +165,7 @@ func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) var jobs []domain.Job for rows.Next() { var job domain.Job - if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime); err != nil { + if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime, &job.Provider); err != nil { return nil, 0, fmt.Errorf("error scanning job row: %v", err) } jobs = append(jobs, job) @@ -180,7 +181,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) { var job domain.Job sqlStatement := ` - SELECT id, title, company, company_link, location, job_link, job_timestamp + SELECT id, title, company, company_link, location, job_link, job_timestamp, provider FROM jobs WHERE id = $1 ` @@ -193,6 +194,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) { &job.Location, &job.JobLink, &job.JobPostTime, + &job.Provider, ) if err == sql.ErrNoRows { @@ -206,6 +208,49 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) { return &job, nil } +func (r *JobRepository) GetJobWithDescription(id int) (*domain.JobWithDescription, error) { + var job domain.JobWithDescription + var criteriaJSON []byte + + sqlStatement := ` + SELECT j.id, j.title, j.company, j.company_link, j.location, j.job_link, j.job_timestamp, j.provider, + COALESCE(jd.description, ''), COALESCE(jd.job_criteria, '{}'::jsonb) + FROM jobs j + LEFT JOIN job_descriptions jd ON j.id = jd.job_id + WHERE j.id = $1 + ` + + err := r.db.QueryRow(sqlStatement, id).Scan( + &job.Job.ID, + &job.Job.Title, + &job.Job.Company, + &job.Job.CompanyLink, + &job.Job.Location, + &job.Job.JobLink, + &job.Job.JobPostTime, + &job.Job.Provider, + &job.JobDescription.Description, + &criteriaJSON, + ) + + if err == sql.ErrNoRows { + return nil, fmt.Errorf("job with ID %d not found", id) + } + + if err != nil { + return nil, fmt.Errorf("error querying job: %v", err) + } + + // Parse criteria JSON + if err := json.Unmarshal(criteriaJSON, &job.JobDescription.Criteria); err != nil { + job.JobDescription.Criteria = make(map[string]string) + } + + job.JobDescription.JobID = job.Job.ID + + return &job, nil +} + func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) { bf := bytes.Buffer{} values := make([]interface{}, 0, len(models)*7) diff --git a/services/api/cmd/server/main.go b/services/api/cmd/server/main.go index 8cbc436..9c11df6 100644 --- a/services/api/cmd/server/main.go +++ b/services/api/cmd/server/main.go @@ -10,8 +10,8 @@ import ( "time" "github.com/gorilla/mux" - "github.com/jobs-scraper/internal/pkg/infrastructure" - "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/infrastructure" + "github.com/jobs-scraper/internal/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/api/internal/app" httpHandler "github.com/jobs-scraper/services/api/pkg/http" @@ -64,12 +64,12 @@ func main() { router := mux.NewRouter() - // Swagger endpoint - router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) - router.Use(httpHandler.CORSMiddleware) router.Use(httpHandler.LogsMiddleware) + // Swagger endpoint + router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) + app := app.NewApplication(router, db, rmq) // Create job analysis result repository diff --git a/services/api/go.mod b/services/api/go.mod index b586643..020b5b2 100644 --- a/services/api/go.mod +++ b/services/api/go.mod @@ -5,9 +5,9 @@ go 1.24.0 require ( github.com/gorilla/mux v1.8.1 github.com/gorilla/schema v1.4.1 - github.com/jobs-scraper/internal/pkg/domain v0.0.0 - github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 - github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/jobs-scraper/internal/domain v0.0.0 + github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/utils v0.0.0 github.com/jobs-scraper/libs/ports v0.0.0 github.com/jobs-scraper/libs/repo v0.0.0 github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000 @@ -39,11 +39,11 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect ) -replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain +replace github.com/jobs-scraper/internal/domain => ../../internal/domain -replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure +replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure -replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils +replace github.com/jobs-scraper/internal/utils => ../../internal/utils replace github.com/jobs-scraper/libs/ports => ../../libs/ports diff --git a/services/api/internal/app/app.go b/services/api/internal/app/app.go index 95c7bbc..84699b0 100644 --- a/services/api/internal/app/app.go +++ b/services/api/internal/app/app.go @@ -4,7 +4,7 @@ import ( "database/sql" "github.com/gorilla/mux" - "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/server" @@ -30,7 +30,8 @@ func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient Router: router, DB: db, JobCommands: &ports.JobCommands{ - CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq), + CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq), + ApplyForJob: jobCommands.NewApplyForJobHandler(jobRepo, rmq), }, JobQueries: &ports.JobQueries{ GetJobs: jobQueries.NewGetJobsHandler(jobRepo), diff --git a/services/api/internal/commands/job/apply-for-job.go b/services/api/internal/commands/job/apply-for-job.go new file mode 100644 index 0000000..668a2ce --- /dev/null +++ b/services/api/internal/commands/job/apply-for-job.go @@ -0,0 +1,44 @@ +package job + +import ( + "encoding/json" + + "github.com/jobs-scraper/internal/dto" + "github.com/jobs-scraper/internal/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/repo" +) + +type ApplyForJob struct { + jobRepo *repo.JobRepository + rmq *rabbitmq.RabbitMQClient +} + +func NewApplyForJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *ApplyForJob { + return &ApplyForJob{ + jobRepo: jobRepo, + rmq: rmq, + } +} + +func (aj *ApplyForJob) Handle(jobId int) error { + job, err := aj.jobRepo.GetJobWithDescription(jobId) + + if err != nil { + return err + } + + jobDTO := dto.JobWithDescriptionFromDomain(*job) + jsonData, err := json.Marshal(jobDTO) + + if err != nil { + return err + } + + err = aj.rmq.Publish(rabbitmq.JobApplierQueue, rabbitmq.ScraperExchange, jsonData) + + if err != nil { + return err + } + + return nil +} diff --git a/services/api/internal/commands/job/create-job.go b/services/api/internal/commands/job/create-job.go index d0a1bb7..c730fe4 100644 --- a/services/api/internal/commands/job/create-job.go +++ b/services/api/internal/commands/job/create-job.go @@ -5,8 +5,8 @@ import ( "errors" "log" - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/domain" + "github.com/jobs-scraper/internal/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/repo" ) @@ -33,25 +33,21 @@ func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error { jsonData, err := json.Marshal(jobRequest) if err != nil { - log.Printf("Error marshaling JSON: %v", err) return err } switch cmd.Provider { case domain.LinkedIn: if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil { - log.Printf("Error publishing LinkedIn message: %v", err) return err } log.Printf("Published LinkedIn job request: %s", string(jsonData)) case domain.Glassdoor: if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil { - log.Printf("Error publishing Glassdoor message: %v", err) return err } log.Printf("Published Glassdoor job request: %s", string(jsonData)) default: - log.Printf("Unsupported job provider: %v", cmd.Provider) return errors.New("unsupported job provider") } diff --git a/services/api/pkg/http/job.go b/services/api/pkg/http/job.go index a4daa6b..45c10dc 100644 --- a/services/api/pkg/http/job.go +++ b/services/api/pkg/http/job.go @@ -9,7 +9,7 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/schema" - "github.com/jobs-scraper/internal/pkg/utils" + "github.com/jobs-scraper/internal/utils" "github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/repo" ) @@ -43,6 +43,7 @@ func (h *JobHandler) RegisterRoutes(router *mux.Router) { jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET") + jobs.HandleFunc("/apply/{id}", utils.Make(h.ApplyForJob)).Methods("POST") } // CreateJob handles the creation of a new job @@ -73,6 +74,44 @@ func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error { return nil } +// Apply applies for a job by id +// @Summary applies for a job +// @Description Applies for a job with the provided id +// @Tags jobs +// @Param id query string false "Job ID" +// @Accept json +// @Produce json +// @Success 201 {object} map[string]string "Successfully applied for job" +// @Failure 400 {object} map[string]string "Invalid request data" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/apply/{id} [post] +func (h *JobHandler) ApplyForJob(w http.ResponseWriter, r *http.Request) error { + jobId := r.URL.Query().Get("id") + + if jobId == "" { + slog.Error("Job ID is required") + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("job ID is required")) + } + + jobIdInt, err := strconv.Atoi(jobId) + + if err != nil { + slog.Error("Invalid Job ID", "error", err) + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID")) + } + + err = h.jobCommands.ApplyForJob.Handle(jobIdInt) + + if err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusInternalServerError, fmt.Errorf("Internal server error")) + } + + utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"}) + + return nil +} + // GetJobs handles the retrieval of all jobs // @Summary Gets all jobs // @Description Gets all jobs with the provided specifications diff --git a/services/api/pkg/http/middlewares.go b/services/api/pkg/http/middlewares.go index 3b6e0da..bcf4c7d 100644 --- a/services/api/pkg/http/middlewares.go +++ b/services/api/pkg/http/middlewares.go @@ -13,11 +13,17 @@ import ( func CORSMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Set CORS headers - w.Header().Set("Access-Control-Allow-Origin", "*") - w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + origin := r.Header.Get("Origin") + if origin == "" { + origin = "*" + } + + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With, X-API-Key") w.Header().Set("Access-Control-Allow-Credentials", "true") w.Header().Set("Access-Control-Max-Age", "86400") + w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range") // Handle preflight OPTIONS requests if r.Method == "OPTIONS" { diff --git a/services/api/pkg/swagger/docs.go b/services/api/pkg/swagger/docs.go index 1cf7ef5..10a6e2d 100644 --- a/services/api/pkg/swagger/docs.go +++ b/services/api/pkg/swagger/docs.go @@ -77,31 +77,25 @@ const docTemplate = `{ } ], "responses": { - "201": { + "200": { "description": "Successfully retrieved jobs", "schema": { "type": "object", - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } }, "400": { "description": "Invalid request data", "schema": { "type": "object", - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } }, "500": { "description": "Internal server error", "schema": { "type": "object", - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } } } @@ -241,6 +235,58 @@ const docTemplate = `{ } } }, + "/jobs/apply/{id}": { + "post": { + "description": "Applies for a job with the provided id", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "applies for a job", + "parameters": [ + { + "type": "string", + "description": "Job ID", + "name": "id", + "in": "query" + } + ], + "responses": { + "201": { + "description": "Successfully applied for job", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request data", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/jobs/{id}/analysis": { "get": { "description": "Retrieves the analysis result for a specific job", diff --git a/services/api/pkg/swagger/swagger.json b/services/api/pkg/swagger/swagger.json index 8ec2ac5..e8c25fb 100644 --- a/services/api/pkg/swagger/swagger.json +++ b/services/api/pkg/swagger/swagger.json @@ -70,31 +70,25 @@ } ], "responses": { - "201": { + "200": { "description": "Successfully retrieved jobs", "schema": { "type": "object", - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } }, "400": { "description": "Invalid request data", "schema": { "type": "object", - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } }, "500": { "description": "Internal server error", "schema": { "type": "object", - "additionalProperties": { - "type": "string" - } + "additionalProperties": true } } } @@ -234,6 +228,58 @@ } } }, + "/jobs/apply/{id}": { + "post": { + "description": "Applies for a job with the provided id", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "applies for a job", + "parameters": [ + { + "type": "string", + "description": "Job ID", + "name": "id", + "in": "query" + } + ], + "responses": { + "201": { + "description": "Successfully applied for job", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request data", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, "/jobs/{id}/analysis": { "get": { "description": "Retrieves the analysis result for a specific job", diff --git a/services/api/pkg/swagger/swagger.yaml b/services/api/pkg/swagger/swagger.yaml index 103a931..511fa1b 100644 --- a/services/api/pkg/swagger/swagger.yaml +++ b/services/api/pkg/swagger/swagger.yaml @@ -107,23 +107,20 @@ paths: produces: - application/json responses: - "201": + "200": description: Successfully retrieved jobs schema: - additionalProperties: - type: string + additionalProperties: true type: object "400": description: Invalid request data schema: - additionalProperties: - type: string + additionalProperties: true type: object "500": description: Internal server error schema: - additionalProperties: - type: string + additionalProperties: true type: object summary: Gets all jobs tags: @@ -253,4 +250,38 @@ paths: summary: Get top matching jobs tags: - jobs + /jobs/apply/{id}: + post: + consumes: + - application/json + description: Applies for a job with the provided id + parameters: + - description: Job ID + in: query + name: id + type: string + produces: + - application/json + responses: + "201": + description: Successfully applied for job + schema: + additionalProperties: + type: string + type: object + "400": + description: Invalid request data + schema: + additionalProperties: + type: string + type: object + "500": + description: Internal server error + schema: + additionalProperties: + type: string + type: object + summary: applies for a job + tags: + - jobs swagger: "2.0" diff --git a/services/job-applier-ts/.env b/services/job-applier-ts/.env new file mode 100644 index 0000000..d1f2c0a --- /dev/null +++ b/services/job-applier-ts/.env @@ -0,0 +1,17 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini + +CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free +OPENROUTER_API_KEY=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 +GEMINI_API_KEY=AIzaSyBkGcsPdqhIrjg-wbLOuhAYmjOCt3A_zxM \ No newline at end of file diff --git a/services/job-applier-ts/CV.pdf b/services/job-applier-ts/CV.pdf new file mode 100644 index 0000000..9e072c3 Binary files /dev/null and b/services/job-applier-ts/CV.pdf differ diff --git a/services/job-applier-ts/cv.txt b/services/job-applier-ts/cv.txt new file mode 100644 index 0000000..11bdd02 --- /dev/null +++ b/services/job-applier-ts/cv.txt @@ -0,0 +1,65 @@ +# Ziad Elshimy +## Frontend Developer + +Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects. + +`ziadshimy7@gmail.com` +`01223381370` +`Alexandria` +[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6) +[GitHub](https://github.com/ziadshimy7) + +--- + +## Work Experience + +### kari - Fullstack Developer +**Jun 2022 - current** +- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates. +- Led the development of multiple internal projects. +- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills. +- Led task planning and coordinated project timelines, reducing overall project duration by 20%. + +### Callibri - Frontend Developer +**Jan 2022 - Mar 2022** +- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement. +- Collaborated closely with senior developers to manage a complex design project, increasing efficiency. + +### EJADA - Frontend Developer +**Mar 2021 - Dec 2021** +- Troubleshooted the website's problems and stay up to date on technology. + +--- + +## Education + +### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General +**Jan 2017 - Dec 2021** + +--- + +## Skills +- HTML5 +- Cascading Style Sheets (CSS) +- SCSS +- Tailwind css +- JavaScript +- TypeScript +- React.js +- Redux.js +- Next.js +- Node.js +- SSR +- Webpack +- Vite +- Go (Golang) +- Microservices +- docker +- docker-compose + +--- + +## Certifications +- CCNA +- The Complete JavaScript Course 2023- Udemy +- React - The Complete Guide - Udemy diff --git a/services/job-applier-ts/package-lock.json b/services/job-applier-ts/package-lock.json new file mode 100644 index 0000000..1063696 --- /dev/null +++ b/services/job-applier-ts/package-lock.json @@ -0,0 +1,287 @@ +{ + "name": "job-applier", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "job-applier", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@jobs-scraper/browser-automation": "file:../../libs/browser-automation", + "@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts", + "dotenv": "^17.2.3" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } + }, + "../../libs/browser-automation": { + "name": "@jobs-scraper/browser-automation", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "dotenv": "^17.2.3", + "puppeteer-core": "^24.25.0" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } + }, + "../../libs/rabbitmq-ts": { + "name": "@jobs-scraper/rabbitmq-ts", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "amqplib": "^0.10.9" + }, + "devDependencies": { + "@types/amqplib": "^0.10.8", + "typescript": "^5.9.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jobs-scraper/browser-automation": { + "resolved": "../../libs/browser-automation", + "link": true + }, + "node_modules/@jobs-scraper/rabbitmq-ts": { + "resolved": "../../libs/rabbitmq-ts", + "link": true + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.10.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz", + "integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + } + } +} diff --git a/services/job-applier-ts/package.json b/services/job-applier-ts/package.json new file mode 100644 index 0000000..8bf1f03 --- /dev/null +++ b/services/job-applier-ts/package.json @@ -0,0 +1,25 @@ +{ + "name": "job-applier", + "version": "1.0.0", + "description": "Job application automation service using Puppeteer", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node src/index.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": ["job", "automation", "puppeteer", "application"], + "author": "", + "license": "ISC", + "dependencies": { + "@jobs-scraper/browser-automation": "file:../../libs/browser-automation", + "@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts", + "dotenv": "^17.2.3" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } +} diff --git a/services/job-applier-ts/src/applier/work.ts b/services/job-applier-ts/src/applier/work.ts new file mode 100644 index 0000000..8f46596 --- /dev/null +++ b/services/job-applier-ts/src/applier/work.ts @@ -0,0 +1,347 @@ +import fs from "fs"; +import path from "path"; +import { + Browser, + randomDelay, + OpenRouterService, + FormField, + getMockFormData, + FormExtractionResult, +} from "@jobs-scraper/browser-automation"; + +/** + * Job represents a job posting + */ +export interface Job { + id: number; + title: string; + company: string; + companyLink: string; + location: string; + jobLink: string; + provider: number; + jobPostTime: string; +} + +export interface JobWithDescription { + job: Job; + jobDescription: JobDescription; +} + +export interface JobDescription { + description: string; + criteria: Record; +} + +/** + * Read CV from file + */ +export function readCV(cvPath: string = "cv.txt"): string { + const absolutePath = path.isAbsolute(cvPath) + ? cvPath + : path.join(process.cwd(), cvPath); + + return fs.readFileSync(absolutePath, "utf-8"); +} + +/** + * Read CV from file + */ +export function getCVPath(cvPath: string = "CV.pdf"): string { + return path.isAbsolute(cvPath) ? cvPath : path.join(process.cwd(), cvPath); +} + +/** + * Handle text, email, tel, url, search, password inputs + */ +async function handleTextInput( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.clear(field.selector); + await browser.type(field.selector, ""); + await browser.type(field.selector, field.value); +} + +/** + * Handle textarea inputs + */ +async function handleTextarea( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.clear(field.selector); + await browser.type(field.selector, field.value); +} + +/** + * Handle number and range inputs + */ +async function handleNumberInput( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.setValue(field.selector, field.value); +} + +/** + * Handle date, datetime-local, time, month, week inputs + */ +async function handleDateInput( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.setValue(field.selector, field.value); +} + +/** + * Handle select dropdown inputs + */ +async function handleSelect(browser: Browser, field: FormField): Promise { + if (field.value === "") { + return; + } + await browser.select(field.selector, field.value); +} + +/** + * Handle radio button inputs + */ +async function handleRadio(browser: Browser, field: FormField): Promise { + if (field.value === "") { + return; + } + await browser.setRadio(field.selector); +} + +/** + * Handle checkbox inputs + */ +async function handleCheckbox( + browser: Browser, + field: FormField +): Promise { + const checked = + field.value === "true" || field.value === "1" || field.value === "yes"; + await browser.setCheckbox(field.selector, checked); +} + +/** + * Handle file input fields + */ +async function handleFileUpload( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.uploadFile(field.selector, field.value); +} + +/** + * Handle color picker inputs + */ +async function handleColorInput( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.setValue(field.selector, field.value); +} + +/** + * Handle hidden input fields + */ +async function handleHiddenInput( + browser: Browser, + field: FormField +): Promise { + if (field.value === "") { + return; + } + await browser.setValue(field.selector, field.value); +} + +export interface WorkOptions { + useMockData?: boolean; + openRouterApiKey?: string; + openRouterModel?: string; +} + +/** + * Main work function that fills out a job application form + */ +export async function work( + browser: Browser, + job: JobWithDescription, + options: WorkOptions = {} +): Promise { + const { openRouterApiKey = "", openRouterModel = "", useMockData } = options; + + try { + await browser.init(); + + await browser.navigate(`${job.job.jobLink}apply`); + + await browser.evaluate( + ` + (() => { + const acceptBtn = document.querySelector('[data-ui="cookie-consent-accept"]'); + if (acceptBtn) { + acceptBtn.click(); + return true; + } + return false; + })() + ` + ); + + let result: FormExtractionResult; + + const formHtml = await browser.getOuterHTML("form"); + + // Read the CV + const cv = readCV(); + + const cvPDFPath = getCVPath(); + + console.log({ cvPDFPath }); + + // let resumeButtonClicked: boolean = false; + + // while (!resumeButtonClicked) { + // resumeButtonClicked = await browser.evaluate( + // ` + // (() => { + // const buttons = document.querySelectorAll('button'); + // const resumeBtn = Array.from(buttons).find(btn => + // btn.innerText.toLowerCase().includes('resume') || + // btn.innerText.toLowerCase().includes('import cv') + // ); + // if (resumeBtn) { + // resumeBtn.click(); + // return true; + // } + // return false; + // })() + // ` + // ); + // } + + // console.log({ resumeButtonClicked }); + // await browser.uploadFile("input#file-upload", cvPDFPath); + await randomDelay(5000, 7000); + // await browser.uploadFile("input#resume-upload-input", cvPDFPath); + + if (!useMockData) { + const openRouterService = new OpenRouterService({ + apiKey: openRouterApiKey, + model: openRouterModel, + }); + console.log("Calling OpenRouter service..."); + result = await openRouterService.applyForJob(formHtml, cv, ""); + + console.log("Form extraction result:", result); + } else { + result = getMockFormData(); + } + + console.log({ result }); + + if (result.fields.length === 0) { + console.log("No fields extracted, using mock data"); + } else { + for (const field of result.fields) { + await randomDelay(1000, 2000); + await processField(browser, field); + } + } + + await randomDelay(160000, 180000); + } finally { + await browser.close(); + } +} + +/** + * Process a single form field + */ +async function processField(browser: Browser, field: FormField): Promise { + if (field.selector === "") { + return; + } + + console.log("Processing field:", field); + + try { + switch (field.field_type) { + case "text": + case "email": + case "tel": + case "url": + case "search": + case "password": + await handleTextInput(browser, field); + break; + case "textarea": + await handleTextarea(browser, field); + break; + case "number": + case "range": + await handleNumberInput(browser, field); + break; + case "date": + case "datetime-local": + case "time": + case "month": + case "week": + await handleDateInput(browser, field); + break; + case "select": + await handleSelect(browser, field); + break; + case "radio": + await handleRadio(browser, field); + break; + case "checkbox": + await handleCheckbox(browser, field); + break; + case "file": + await handleFileUpload(browser, field); + break; + case "color": + await handleColorInput(browser, field); + break; + case "hidden": + await handleHiddenInput(browser, field); + break; + case "button": + case "submit": + case "reset": + // Skip buttons - they're handled separately + return; + default: + return; + } + } catch (error) { + console.error(`Error processing field ${field.field_name}:`, error); + // Continue with other fields + } + + await randomDelay(1500, 2000); +} diff --git a/services/job-applier-ts/src/index.ts b/services/job-applier-ts/src/index.ts new file mode 100644 index 0000000..3de44f7 --- /dev/null +++ b/services/job-applier-ts/src/index.ts @@ -0,0 +1,91 @@ +import * as dotenv from "dotenv"; +import { RabbitMQClient, RabbitMQConfig } from "@jobs-scraper/rabbitmq-ts"; +import { work, JobWithDescription } from "./applier/work"; +import { Browser } from "@jobs-scraper/browser-automation"; + +if (dotenv.config({ path: ".local.env" }).error) { + console.log("No .local.env file found, trying .env"); + if (dotenv.config().error) { + console.log("No .env file found, using system environment variables"); + } +} + +const JOB_APPLIER_QUEUE = "job.applier"; +const SCRAPER_EXCHANGE = "scraper_exchange"; + +let browser = new Browser({ port: 9223 }); +let rabbitMQClient: RabbitMQClient | null = null; + +function createRabbitMQConfig(): RabbitMQConfig { + return { + url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/", + queueName: JOB_APPLIER_QUEUE, + exchangeName: SCRAPER_EXCHANGE, + exchangeType: "topic", + durable: true, + }; +} + +async function processJob(job: JobWithDescription): Promise { + console.log( + `Processing job application for: ${job.job.title} at ${job.job.company}` + ); + + await work(browser, job, { + useMockData: true, + openRouterApiKey: process.env.OPENROUTER_API_KEY, + openRouterModel: process.env.OPENROUTER_MODEL, + }); + + console.log("Job application completed successfully!"); +} + +async function main() { + console.log("Starting Job Applier service..."); + + try { + await browser.init(); + + const config = createRabbitMQConfig(); + rabbitMQClient = new RabbitMQClient(config); + await rabbitMQClient.connect(); + + await rabbitMQClient.subscribe(processJob); + } catch (error) { + console.error("Error starting service:", error); + await cleanup(); + process.exit(1); + } +} + +async function cleanup(): Promise { + try { + if (rabbitMQClient) { + await rabbitMQClient.close(); + rabbitMQClient = null; + } + await browser.close(); + console.log("Cleanup completed"); + } catch (error) { + console.error("Error during cleanup:", error); + } +} + +// Handle graceful shutdown +process.on("SIGINT", async () => { + console.log("Received SIGINT signal"); + await cleanup(); + process.exit(0); +}); + +process.on("SIGTERM", async () => { + console.log("Received SIGTERM signal"); + await cleanup(); + process.exit(0); +}); + +main().catch(async (error) => { + console.error("Fatal error:", error); + await cleanup(); + process.exit(1); +}); diff --git a/services/job-applier-ts/tsconfig.json b/services/job-applier-ts/tsconfig.json new file mode 100644 index 0000000..2c37fd3 --- /dev/null +++ b/services/job-applier-ts/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020", "DOM"], + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "removeComments": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "exactOptionalPropertyTypes": false, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": false, + "noUncheckedIndexedAccess": false + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist" + ] +} diff --git a/services/scraper-google/analyzer/get-job-description.go b/services/scraper-google/analyzer/get-job-description.go index b5cb22a..8b6a30c 100644 --- a/services/scraper-google/analyzer/get-job-description.go +++ b/services/scraper-google/analyzer/get-job-description.go @@ -1,19 +1,15 @@ package analyzer import ( - "fmt" "os" - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/internal/pkg/openrouter" - // "github.com/jobs-scraper/internal/pkg/openai" - // "github.com/jobs-scraper/internal/pkg/openrouter" + "github.com/jobs-scraper/internal/domain" + "github.com/jobs-scraper/internal/openrouter" ) func GetJobDescription(htmlContent string) (*domain.JobDescription, error) { openRouterApiKey := os.Getenv("OPENROUTER_API_KEY") openRouterModel := os.Getenv("OPENROUTER_MODEL") - fmt.Println("key", openRouterApiKey) client := openrouter.NewOpenRouterService(openRouterModel, openRouterApiKey) result, err := client.CreateJobDescription(htmlContent) diff --git a/services/scraper-google/go.mod b/services/scraper-google/go.mod index 644817b..7533b06 100644 --- a/services/scraper-google/go.mod +++ b/services/scraper-google/go.mod @@ -4,7 +4,7 @@ go 1.24.0 require ( github.com/PuerkitoBio/goquery v1.10.3 - github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 + github.com/jobs-scraper/internal/infrastructure v0.0.0 github.com/joho/godotenv v1.5.1 github.com/levmv/sked v0.2.1 ) @@ -18,4 +18,4 @@ require ( golang.org/x/net v0.44.0 // indirect ) -replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure +replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go index 498d820..b3c8552 100644 --- a/services/scraper-google/main.go +++ b/services/scraper-google/main.go @@ -8,8 +8,8 @@ import ( "syscall" "time" - "github.com/jobs-scraper/internal/pkg/browser" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/browser" + "github.com/jobs-scraper/internal/domain" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/scraper-google/analyzer" si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure" @@ -49,7 +49,6 @@ func main() { sched := sked.New(ctx) googleLinkStream := make(chan domain.GoogleLink, 100) pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} - // pages := []int{50, 60, 70, 80, 90, 100} log.Println("Starting scraper with scheduler...") diff --git a/services/scraper-google/setup-infrastructure/setup-infrastructure.go b/services/scraper-google/setup-infrastructure/setup-infrastructure.go index 269f121..608b69f 100644 --- a/services/scraper-google/setup-infrastructure/setup-infrastructure.go +++ b/services/scraper-google/setup-infrastructure/setup-infrastructure.go @@ -4,7 +4,7 @@ import ( "database/sql" "log" - "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/infrastructure" "github.com/joho/godotenv" ) diff --git a/services/scraper-google/utils/build-url.go b/services/scraper-google/utils/build-url.go index fac437c..352b695 100644 --- a/services/scraper-google/utils/build-url.go +++ b/services/scraper-google/utils/build-url.go @@ -1,15 +1,28 @@ package utils import ( + "net/url" "strconv" - "strings" + "time" ) func BuildUrl(query string, page int) string { - var url strings.Builder - url.WriteString("https://www.google.com/search?q=") - url.WriteString(query) - url.WriteString("&start=") - url.WriteString(strconv.Itoa(page)) - return url.String() + params := url.Values{} + params.Add("q", query) + params.Add("start", strconv.Itoa(page)) + + now := time.Now() + twoWeeksAgo := now.AddDate(0, 0, -14) + + startDate := twoWeeksAgo.Format("01/02/2006") + endDate := now.Format("01/02/2006") + + tbs := "cdr:1,cd_min:" + startDate + ",cd_max:" + endDate + params.Add("tbs", tbs) + + // Optional language bias + // params.Add("hl", "en") + // params.Add("lr", "lang_en") + + return "https://www.google.com/search?" + params.Encode() } diff --git a/services/scraper-google/utils/get-html-raw.go b/services/scraper-google/utils/get-html-raw.go index d05fe67..4b07c37 100644 --- a/services/scraper-google/utils/get-html-raw.go +++ b/services/scraper-google/utils/get-html-raw.go @@ -9,8 +9,8 @@ import ( "time" "github.com/PuerkitoBio/goquery" - "github.com/jobs-scraper/internal/pkg/browser" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/browser" + "github.com/jobs-scraper/internal/domain" ) func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) { diff --git a/services/scraper-google/work.go b/services/scraper-google/work.go index bcdb3f1..750c65d 100644 --- a/services/scraper-google/work.go +++ b/services/scraper-google/work.go @@ -2,10 +2,12 @@ package main import ( "context" + "fmt" "log" + "time" - "github.com/jobs-scraper/internal/pkg/browser" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/browser" + "github.com/jobs-scraper/internal/domain" "github.com/jobs-scraper/services/scraper-google/utils" ) @@ -21,9 +23,9 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha browser.RandomDelay(1000, 3000) - // query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid" - // query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` - query := `site:workable.com (Software OR Backend OR Full-Stack) Engineer lang:en` + // quer := `site: ashbyhq.com (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en` + // query := `site:lever.co (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en` + query := `site:workable.com (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en` // url := utils.BuildUrl(query, page) err = tab.Navigate("https://linkedin.com") @@ -32,8 +34,9 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha } browser.RandomDelay(2000, 2500) - + fmt.Println(utils.BuildUrl(query, page)) err = tab.Navigate(utils.BuildUrl(query, page)) + err = tab.WaitForNetworkIdle(10 * time.Second) if err != nil { log.Fatal(err) diff --git a/services/scraper-hiringcafe/index.ts b/services/scraper-hiringcafe/index.ts index 70a31ab..8192d17 100644 --- a/services/scraper-hiringcafe/index.ts +++ b/services/scraper-hiringcafe/index.ts @@ -39,10 +39,11 @@ async function cleanup() { } async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { - chrome = launchChromeWithDebugging(); + let port = 9223; + chrome = launchChromeWithDebugging(port); await waitForChromeReady(); - const response = await fetch("http://localhost:9222/json/version", { + const response = await fetch(`http://localhost:${port}/json/version`, { headers: { Origin: "", }, diff --git a/services/scraper-hiringcafe/utils/browser.ts b/services/scraper-hiringcafe/utils/browser.ts index df532d7..33d20ab 100644 --- a/services/scraper-hiringcafe/utils/browser.ts +++ b/services/scraper-hiringcafe/utils/browser.ts @@ -70,11 +70,11 @@ function getChromePath(): string { throw new Error("Could not find Chrome/Chromium installation"); } -export function launchChromeWithDebugging() { +export function launchChromeWithDebugging(port: number) { try { const chromePath = getChromePath(); const args = [ - "--remote-debugging-port=9222", + `--remote-debugging-port=${port}`, `--user-data-dir=${userDataDir}`, "--remote-allow-origins=*", "--incognito", diff --git a/services/scraper-linkedin/go.mod b/services/scraper-linkedin/go.mod index f9e8ef3..a919a2f 100644 --- a/services/scraper-linkedin/go.mod +++ b/services/scraper-linkedin/go.mod @@ -6,9 +6,9 @@ toolchain go1.24.7 require ( github.com/PuerkitoBio/goquery v1.10.3 - github.com/jobs-scraper/internal/pkg/domain v0.0.0 - github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 - github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/jobs-scraper/internal/domain v0.0.0 + github.com/jobs-scraper/internal/infrastructure v0.0.0 + github.com/jobs-scraper/internal/utils v0.0.0 github.com/jobs-scraper/libs/repo v0.0.0 github.com/joho/godotenv v1.5.1 ) @@ -23,10 +23,10 @@ require ( golang.org/x/net v0.44.0 // indirect ) -replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain +replace github.com/jobs-scraper/internal/domain => ../../internal/domain -replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure +replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure replace github.com/jobs-scraper/libs/repo => ../../libs/repo -replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils +replace github.com/jobs-scraper/internal/utils => ../../internal/utils diff --git a/services/scraper-linkedin/main.go b/services/scraper-linkedin/main.go index 902c10b..e073553 100644 --- a/services/scraper-linkedin/main.go +++ b/services/scraper-linkedin/main.go @@ -10,9 +10,9 @@ import ( "time" - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/internal/pkg/infrastructure" - "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/domain" + "github.com/jobs-scraper/internal/infrastructure" + "github.com/jobs-scraper/internal/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/services/scraper/pipeline" "github.com/joho/godotenv" diff --git a/services/scraper-linkedin/pipeline/job_pipeline.go b/services/scraper-linkedin/pipeline/job_pipeline.go index bdbb779..c0b64f4 100644 --- a/services/scraper-linkedin/pipeline/job_pipeline.go +++ b/services/scraper-linkedin/pipeline/job_pipeline.go @@ -9,7 +9,7 @@ import ( "time" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/domain" "github.com/jobs-scraper/libs/repo" ) diff --git a/services/scraper-linkedin/pipeline/job_pipeline_workers.go b/services/scraper-linkedin/pipeline/job_pipeline_workers.go index 5b39bc0..f71c827 100644 --- a/services/scraper-linkedin/pipeline/job_pipeline_workers.go +++ b/services/scraper-linkedin/pipeline/job_pipeline_workers.go @@ -4,7 +4,7 @@ import ( "context" "log" - "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/domain" ) func GetJobs(context context.Context, scraperService *Scraper, searchQuery domain.SearchQuery) <-chan domain.Job { diff --git a/services/scraper-linkedin/pipeline/scraper.go b/services/scraper-linkedin/pipeline/scraper.go index 97f6b2f..e7404eb 100644 --- a/services/scraper-linkedin/pipeline/scraper.go +++ b/services/scraper-linkedin/pipeline/scraper.go @@ -12,8 +12,8 @@ import ( "time" "github.com/PuerkitoBio/goquery" - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/internal/pkg/utils" + "github.com/jobs-scraper/internal/domain" + "github.com/jobs-scraper/internal/utils" ) type Config struct {