fix errors and restructure project

This commit is contained in:
Elshimy Ziad Magdy Taha 2026-01-26 17:33:48 +05:00
parent a281363b61
commit af52085ea3
93 changed files with 4250 additions and 190 deletions

View file

@ -6,6 +6,7 @@ API_DIR=./services/api
SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin
SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor
SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright
JOB_APPLIER_BROWSER_USE_DIR=./services/job-applier-browser-use
CRON_ANALYZER_DIR=./apps/cron-analyzer CRON_ANALYZER_DIR=./apps/cron-analyzer
CV_ANALYZER_DIR=./apps/cv-analyzer CV_ANALYZER_DIR=./apps/cv-analyzer
DOCS_DIR=./services/api/pkg/swagger DOCS_DIR=./services/api/pkg/swagger
@ -71,6 +72,18 @@ docker-scraper-playwright: ## Build and run Playwright scraper in Docker
@echo "Running Playwright scraper in Docker..." @echo "Running Playwright scraper in Docker..."
@docker run --rm --name playwright-scraper playwright-scraper @docker run --rm --name playwright-scraper playwright-scraper
.PHONY: setup-job-applier-browser-use
setup-job-applier-browser-use: ## Setup the browser-use job applier service
@echo "Setting up browser-use job applier service..."
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv venv && uv pip install -e .
@echo "Installing Chromium browser..."
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uvx browser-use install
.PHONY: run-job-applier-browser-use
run-job-applier-browser-use: ## Run the browser-use job applier service
@echo "Running browser-use job applier service..."
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv run python -m src.main
# Apps # Apps
.PHONY: run-cron-analyzer .PHONY: run-cron-analyzer
run-cron-analyzer: ## Run the job analysis cron service run-cron-analyzer: ## Run the job analysis cron service

View file

@ -4,7 +4,7 @@ import (
"log" "log"
"github.com/jobs-scraper/apps/cron-analyzer/internal" "github.com/jobs-scraper/apps/cron-analyzer/internal"
"github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/infrastructure"
"github.com/joho/godotenv" "github.com/joho/godotenv"
) )

View file

@ -3,9 +3,9 @@ module github.com/jobs-scraper/apps/cron-analyzer
go 1.24.0 go 1.24.0
require ( require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0 github.com/jobs-scraper/internal/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000 github.com/jobs-scraper/internal/openrouter v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/libs/repo v0.0.0 github.com/jobs-scraper/libs/repo v0.0.0
github.com/joho/godotenv v1.5.1 github.com/joho/godotenv v1.5.1
) )
@ -20,10 +20,10 @@ require (
github.com/rabbitmq/amqp091-go v1.10.0 // indirect github.com/rabbitmq/amqp091-go v1.10.0 // indirect
) )
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain replace github.com/jobs-scraper/internal/domain => ../../internal/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter replace github.com/jobs-scraper/internal/openrouter => ../../internal/openrouter
replace github.com/jobs-scraper/libs/repo => ../../libs/repo replace github.com/jobs-scraper/libs/repo => ../../libs/repo

View file

@ -8,10 +8,10 @@ import (
// "path/filepath" // "path/filepath"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/pkg/openai" "github.com/jobs-scraper/internal/openai"
"github.com/jobs-scraper/internal/pkg/openrouter" "github.com/jobs-scraper/internal/openrouter"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
) )

12
go.work
View file

@ -3,12 +3,12 @@ go 1.24.0
use ( use (
. .
./apps/cron-analyzer ./apps/cron-analyzer
./internal/pkg/domain ./internal/domain
./internal/pkg/gemini ./internal/gemini
./internal/pkg/infrastructure ./internal/infrastructure
./internal/pkg/openai ./internal/openai
./internal/pkg/openrouter ./internal/openrouter
./internal/pkg/utils ./internal/utils
./libs/ports ./libs/ports
./libs/repo ./libs/repo
./libs/server ./libs/server

View file

@ -5,9 +5,11 @@ import (
"log" "log"
"math/rand" "math/rand"
"os/exec" "os/exec"
// "strings"
"time" "time"
"github.com/chromedp/cdproto/cdp" "github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/cdproto/page" "github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp" "github.com/chromedp/chromedp"
) )
@ -116,7 +118,8 @@ func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
var nodes []*cdp.Node var nodes []*cdp.Node
err := chromedp.Run(b.ctx, err := chromedp.Run(b.ctx,
chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), chromedp.WaitVisible(selector, chromedp.ByQueryAll),
chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
) )
if err != nil { if err != nil {
@ -126,6 +129,22 @@ func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
return nodes, nil return nodes, nil
} }
// GetOuterHTML gets the outer HTML of the first element matching the selector
func (b *Browser) GetOuterHTML(selector string) (string, error) {
var html string
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector, chromedp.ByQuery),
chromedp.OuterHTML(selector, &html, chromedp.ByQuery),
)
if err != nil {
return "", err
}
return html, nil
}
func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) { func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) {
var text string var text string
@ -151,6 +170,8 @@ func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr str
return "", err return "", err
} }
// value = strings.ReplaceAll(value, "/apply", "")
return value, nil return value, nil
} }
@ -229,13 +250,154 @@ func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error {
} }
} }
func (b *Browser) NodeToString(node *cdp.Node) (string, error) {
var outerHTML string
err := chromedp.Run(b.ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
h, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
if err != nil {
return err
}
outerHTML = h
return nil
}),
)
if err != nil {
return "", err
}
return outerHTML, err
}
func (b *Browser) Type(selector string, value string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.SendKeys(selector, value),
)
if err != nil {
return err
}
return nil
}
func (b *Browser) Click(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Click(selector),
)
if err != nil {
return err
}
return nil
}
// Select chooses an option from a <select> element by value
func (b *Browser) Select(selector string, value string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.SetValue(selector, value),
)
if err != nil {
return err
}
return nil
}
// SetCheckbox sets a checkbox to checked or unchecked state
func (b *Browser) SetCheckbox(selector string, checked bool) error {
var isChecked bool
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Evaluate(`document.querySelector('`+selector+`').checked`, &isChecked),
)
if err != nil {
return err
}
if isChecked != checked {
err = chromedp.Run(b.ctx,
chromedp.Click(selector),
)
if err != nil {
return err
}
}
return nil
}
// SetRadio clicks a radio button to select it
func (b *Browser) SetRadio(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Click(selector),
)
if err != nil {
return err
}
return nil
}
// UploadFile sets a file input to the specified file path
func (b *Browser) UploadFile(selector string, filePath string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.SetUploadFiles(selector, []string{filePath}),
)
if err != nil {
return err
}
return nil
}
// SetValue sets the value of an input field directly (useful for hidden, date, color, etc.)
func (b *Browser) SetValue(selector string, value string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitReady(selector),
chromedp.SetValue(selector, value),
)
if err != nil {
return err
}
return nil
}
// Clear clears the value of an input field
func (b *Browser) Clear(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Clear(selector),
)
if err != nil {
return err
}
return nil
}
// Focus focuses on an element
func (b *Browser) Focus(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Focus(selector),
)
if err != nil {
return err
}
return nil
}
// RandomDelay waits for a random duration between min and max milliseconds // RandomDelay waits for a random duration between min and max milliseconds
func RandomDelay(minMs, maxMs int) { func RandomDelay(minMs, maxMs int) {
delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond
time.Sleep(delay) time.Sleep(delay)
} }
// Sleep waits for the specified duration
func (b *Browser) Sleep(d time.Duration) {
time.Sleep(d)
}

5
internal/domain/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/internal/domain
go 1.24.0
toolchain go1.24.7

View file

@ -1,15 +1,16 @@
package dto package dto
import "github.com/jobs-scraper/internal/pkg/domain" import "github.com/jobs-scraper/internal/domain"
type JobDTO struct { type JobDTO struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Title string `json:"title"` Title string `json:"title"`
Company string `json:"company"` Company string `json:"company"`
CompanyLink string `json:"companyLink"` CompanyLink string `json:"companyLink"`
Location string `json:"location"` Location string `json:"location"`
JobLink string `json:"jobLink"` JobLink string `json:"jobLink"`
JobPostTime string `json:"jobPostTime,omitempty"` JobPostTime string `json:"jobPostTime,omitempty"`
Provider domain.JobProvider `json:"provider"`
} }
func JobFromDomain(j domain.Job) JobDTO { func JobFromDomain(j domain.Job) JobDTO {
@ -25,6 +26,7 @@ func JobFromDomain(j domain.Job) JobDTO {
Location: j.Location, Location: j.Location,
JobLink: j.JobLink, JobLink: j.JobLink,
JobPostTime: postTime, JobPostTime: postTime,
Provider: j.Provider,
} }
} }
@ -35,3 +37,17 @@ func JobsFromDomain(jobs []domain.Job) []JobDTO {
} }
return dtos return dtos
} }
type JobWithDescriptionDTO struct {
JobDTO
Description string `json:"description"`
Criteria map[string]string `json:"criteria,omitempty"`
}
func JobWithDescriptionFromDomain(j domain.JobWithDescription) JobWithDescriptionDTO {
return JobWithDescriptionDTO{
JobDTO: JobFromDomain(j.Job),
Description: j.JobDescription.Description,
Criteria: j.JobDescription.Criteria,
}
}

View file

@ -7,7 +7,7 @@ import (
"log" "log"
"strings" "strings"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"google.golang.org/genai" "google.golang.org/genai"
) )

View file

@ -1,4 +1,4 @@
module github.com/jobs-scraper/internal/pkg/gemini module github.com/jobs-scraper/internal/gemini
go 1.24.0 go 1.24.0

View file

@ -1,4 +1,4 @@
module github.com/jobs-scraper/internal/pkg/infrastructure module github.com/jobs-scraper/internal/infrastructure
go 1.24.0 go 1.24.0
@ -9,16 +9,16 @@ require (
github.com/gorilla/mux v1.8.1 github.com/gorilla/mux v1.8.1
github.com/jobs-scraper/libs/ports v0.0.0 github.com/jobs-scraper/libs/ports v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0 github.com/jobs-scraper/libs/repo v0.0.0
github.com/jobs-scraper/internal/pkg/utils v0.0.0 github.com/jobs-scraper/internal/utils v0.0.0
github.com/lib/pq v1.10.9 github.com/lib/pq v1.10.9
github.com/rabbitmq/amqp091-go v1.10.0 github.com/rabbitmq/amqp091-go v1.10.0
) )
replace github.com/jobs-scraper/libs/repo => ../../../libs/repo replace github.com/jobs-scraper/libs/repo => ../../libs/repo
replace github.com/jobs-scraper/internal/pkg/utils => ../utils replace github.com/jobs-scraper/internal/utils => ../utils
replace github.com/jobs-scraper/libs/ports => ../../../libs/ports replace github.com/jobs-scraper/libs/ports => ../../libs/ports
require ( require (
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect

View file

@ -9,7 +9,7 @@ import (
"github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/internal/pkg/utils" "github.com/jobs-scraper/internal/utils"
"github.com/gorilla/mux" "github.com/gorilla/mux"
) )

View file

@ -21,6 +21,7 @@ const (
DeadLetterExchange = "scraper_dlx" DeadLetterExchange = "scraper_dlx"
CvAnalyzeExchange = "cv_exchange" CvAnalyzeExchange = "cv_exchange"
CvAnalyzeQueue = "cv.analyze" CvAnalyzeQueue = "cv.analyze"
JobApplierQueue = "job.applier"
) )
type RabbitMQClient struct { type RabbitMQClient struct {
@ -122,13 +123,14 @@ func (r *RabbitMQClient) setupInfrastructure() error {
// Define queues with their routing keys // Define queues with their routing keys
queues := map[string]string{ queues := map[string]string{
LinkedInQueue: "scraper.linkedin", LinkedInQueue: "scraper.linkedin",
IndeedQueue: "scraper.indeed", IndeedQueue: "scraper.indeed",
BaytQueue: "scraper.bayt", BaytQueue: "scraper.bayt",
TokyoDevQueue: "scraper.tokyodev", TokyoDevQueue: "scraper.tokyodev",
JapanDevQueue: "scraper.japandev", JapanDevQueue: "scraper.japandev",
GlassDoorQueue: "scraper.glassdoor", GlassDoorQueue: "scraper.glassdoor",
GoogleQueue: "scraper.google", GoogleQueue: "scraper.google",
JobApplierQueue: "job.applier",
} }
// Declare queues with dead letter exchange and TTL // Declare queues with dead letter exchange and TTL

View file

@ -36,8 +36,8 @@ import (
"fmt" "fmt"
"os" "os"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/pkg/openai" "github.com/jobs-scraper/internal/openai"
) )
func main() { func main() {

10
internal/openai/go.mod Normal file
View file

@ -0,0 +1,10 @@
module github.com/jobs-scraper/internal/openai
go 1.24.0
require (
github.com/jobs-scraper/internal/domain v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.41.2
)
replace github.com/jobs-scraper/internal/domain => ../domain

View file

@ -1,6 +1,6 @@
package openai package openai
import "github.com/jobs-scraper/internal/pkg/domain" import "github.com/jobs-scraper/internal/domain"
// JobAnalysisService defines the interface for job analysis services // JobAnalysisService defines the interface for job analysis services
type JobAnalysisService interface { type JobAnalysisService interface {

View file

@ -8,7 +8,7 @@ import (
"os" "os"
"strings" "strings"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/sashabaranov/go-openai" "github.com/sashabaranov/go-openai"
) )

View file

@ -0,0 +1,12 @@
module github.com/jobs-scraper/internal/openrouter
go 1.24.0
require (
github.com/eduardolat/openroutergo v0.1.0
github.com/jobs-scraper/internal/domain v0.0.0
)
require github.com/orsinium-labs/enum v1.4.0 // indirect
replace github.com/jobs-scraper/internal/domain => ../domain

View file

@ -7,9 +7,56 @@ import (
"strings" "strings"
"github.com/eduardolat/openroutergo" "github.com/eduardolat/openroutergo"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
) )
// FieldType represents HTML input field types
type FieldType string
const (
FieldTypeText FieldType = "text"
FieldTypeEmail FieldType = "email"
FieldTypeTel FieldType = "tel"
FieldTypePassword FieldType = "password"
FieldTypeNumber FieldType = "number"
FieldTypeDate FieldType = "date"
FieldTypeDatetimeLocal FieldType = "datetime-local"
FieldTypeTime FieldType = "time"
FieldTypeMonth FieldType = "month"
FieldTypeWeek FieldType = "week"
FieldTypeURL FieldType = "url"
FieldTypeSearch FieldType = "search"
FieldTypeColor FieldType = "color"
FieldTypeRange FieldType = "range"
FieldTypeFile FieldType = "file"
FieldTypeHidden FieldType = "hidden"
FieldTypeCheckbox FieldType = "checkbox"
FieldTypeRadio FieldType = "radio"
FieldTypeSelect FieldType = "select"
FieldTypeTextarea FieldType = "textarea"
FieldTypeButton FieldType = "button"
FieldTypeSubmit FieldType = "submit"
FieldTypeReset FieldType = "reset"
)
// FormField represents a single form field extracted from HTML
type FormField struct {
Label string `json:"label"`
FieldName string `json:"field_name"`
FieldType FieldType `json:"field_type"`
Value string `json:"value"`
Placeholder string `json:"placeholder,omitempty"`
Required bool `json:"required"`
Selector string `json:"selector"`
}
// FormExtractionResult represents the structured response from form extraction
type FormExtractionResult struct {
Fields []FormField `json:"fields"`
ApplyButton string `json:"apply_button"`
AdjustedCV string `json:"adjusted_cv"`
}
// JobAnalysisResult represents the structured response from job analysis // JobAnalysisResult represents the structured response from job analysis
type JobAnalysisResult struct { type JobAnalysisResult struct {
Recommendation string `json:"recommendation"` Recommendation string `json:"recommendation"`
@ -199,3 +246,117 @@ Extract from this HTML Content:
return jobDescription, nil return jobDescription, nil
} }
// ExtractFormFields extracts form fields from HTML and populates them based on CV data.
// It also adjusts the CV based on job type (frontend/backend/fullstack).
func (s *OpenRouterService) CreateCV(htmlForm string, cvMarkdown string, jobDescription string) (*FormExtractionResult, error) {
client, err := openroutergo.
NewClient().
WithAPIKey(s.apiKey).
Create()
if err != nil {
return nil, fmt.Errorf("failed to create client: %v", err)
}
prompt := fmt.Sprintf(`You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
1. Extract ALL form fields and populate them with appropriate values from the CV
2. Adjust the CV based on the job type (frontend/backend/fullstack)
HTML FORM:
%s
JOB DESCRIPTION:
%s
ORIGINAL CV (Markdown):
%s
CV ADJUSTMENT RULES:
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
- If FULLSTACK: Keep both frontend and backend content.
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
- Maintain the same markdown structure and formatting as the original CV.
- Do NOT invent new skills or experience - only filter existing content.
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Extract ALL input fields, textareas, selects, and buttons from the form
3. For each field, determine the appropriate value from the ADJUSTED CV
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
5. Identify the apply/submit button text
6. Use the exact JSON schema below
JSON Schema:
{
"fields": [
{
"label": "Field label or name attribute",
"field_name": "name or id attribute of the input",
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
"value": "Value to populate based on CV data",
"placeholder": "Placeholder text if any",
"required": true|false,
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
}
],
"apply_button": "Unique CSS selector for the form's submit button",
"adjusted_cv": "The full adjusted CV in markdown format, tailored to the job type"
}
FIELD MAPPING GUIDELINES:
- Name fields: Extract full name, first name, last name from CV
- Email: Use email from CV contact info
- Phone: Use phone number from CV
- LinkedIn/Portfolio/Website: Use URLs from CV
- Experience/Years: Calculate from CV work history
- Current company/title: Use most recent from CV
- Skills: List relevant skills from ADJUSTED CV
- Education: Use education details from CV
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
- Salary expectations: Leave empty unless specified in CV
- Location/Address: Use from CV contact info
- Resume/CV upload: Leave value empty (file upload)
Start your response with { and end with }`, htmlForm, jobDescription, cvMarkdown)
_, resp, err := client.
NewChatCompletion().
WithModel(s.model).
WithSystemMessage("You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.").
WithUserMessage(prompt).
Execute()
if err != nil {
return nil, fmt.Errorf("failed to execute completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from API")
}
jsonContent := resp.Choices[0].Message.Content
log.Printf("Raw API response for form extraction: %s", jsonContent)
if len(jsonContent) > 0 && jsonContent[0] == '<' {
return nil, fmt.Errorf("API returned HTML/XML instead of JSON. Response: %s", jsonContent[:min(200, len(jsonContent))])
}
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
var result FormExtractionResult
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
if syntaxErr, ok := err.(*json.SyntaxError); ok {
start := max(int(syntaxErr.Offset)-50, 0)
end := int(syntaxErr.Offset) + 50
if end > len(jsonContent) {
end = len(jsonContent)
}
log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end])
}
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(500, len(jsonContent))])
}
return &result, nil
}

View file

@ -1,5 +0,0 @@
module github.com/jobs-scraper/internal/pkg/domain
go 1.24.0
toolchain go1.24.7

View file

@ -1,10 +0,0 @@
module github.com/jobs-scraper/internal/pkg/openai
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.41.2
)
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

View file

@ -1,12 +0,0 @@
module github.com/jobs-scraper/internal/pkg/openrouter
go 1.24.0
require (
github.com/eduardolat/openroutergo v0.1.0
github.com/jobs-scraper/internal/pkg/domain v0.0.0
)
require github.com/orsinium-labs/enum v1.4.0 // indirect
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

View file

@ -1,5 +0,0 @@
module github.com/jobs-scraper/internal/pkg/utils
go 1.24.0
toolchain go1.24.7

5
internal/utils/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/internal/utils
go 1.24.0
toolchain go1.24.7

1146
libs/browser-automation/package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,25 @@
{
"name": "@jobs-scraper/browser-automation",
"version": "1.0.0",
"description": "Browser automation library using Puppeteer - ported from Go chromedp implementation",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "ts-node src/index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": ["puppeteer", "browser", "automation", "scraping"],
"author": "",
"license": "ISC",
"dependencies": {
"dotenv": "^17.2.3",
"puppeteer-core": "^24.25.0"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,389 @@
import puppeteer, {
Browser as PuppeteerBrowser,
Page,
ElementHandle,
} from "puppeteer-core";
import { ChildProcess } from "child_process";
import {
launchChromeWithDebugging,
waitForChromeReady,
getChromeWebSocketUrl,
} from "./utils/chrome-launcher";
import { randomDelay } from "./utils/helpers";
export interface BrowserOptions {
headless?: boolean;
port?: number;
userAgent?: string;
}
export class Browser {
private browser: PuppeteerBrowser | null = null;
private page: Page | null = null;
private chromeProcess: ChildProcess | null = null;
private port: number;
private userAgent: string;
constructor(options: BrowserOptions = {}) {
this.port = options.port || 9222;
this.userAgent =
options.userAgent ||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
}
/**
* Initialize the browser by launching Chrome and connecting via Puppeteer
*/
async init(headless: boolean = false): Promise<void> {
this.chromeProcess = launchChromeWithDebugging(this.port, headless);
await waitForChromeReady(this.port);
const wsUrl = await getChromeWebSocketUrl(this.port);
this.browser = await puppeteer.connect({
browserWSEndpoint: wsUrl,
defaultViewport: null,
});
console.log("Connected to browser successfully");
const pages = await this.browser.pages();
if (pages.length === 0) {
this.page = await this.browser.newPage();
} else {
this.page = pages[0];
}
await this.page.setUserAgent(this.userAgent);
}
/**
* Close the browser and kill Chrome process
*/
async close(): Promise<void> {
if (this.browser) {
await this.browser.close();
this.browser = null;
}
if (this.chromeProcess) {
this.chromeProcess.kill("SIGINT");
this.chromeProcess = null;
}
console.log("Browser closed");
}
/**
* Create a new tab
*/
async newTab(): Promise<Page> {
if (!this.browser) {
throw new Error("Browser not initialized");
}
return await this.browser.newPage();
}
/**
* Get the current page
*/
getPage(): Page {
if (!this.page) {
throw new Error("Page not initialized");
}
return this.page;
}
/**
* Navigate to a URL
*/
async navigate(url: string): Promise<void> {
const page = this.getPage();
await page.goto(url, {
waitUntil: ["networkidle2", "domcontentloaded"],
timeout: 60000,
});
}
/**
* Wait for an element to be visible
*/
async waitVisible(
selector: string,
timeout: number = 30000
): Promise<ElementHandle<Element> | null> {
const page = this.getPage();
return await page.waitForSelector(selector, {
visible: true,
timeout,
});
}
/**
* Wait for an element to be ready in the DOM (not necessarily visible)
*/
async waitReady(
selector: string,
timeout: number = 30000
): Promise<ElementHandle<Element> | null> {
const page = this.getPage();
return await page.waitForSelector(selector, {
timeout,
});
}
/**
* Get all elements matching a selector
*/
async getNodes(selector: string): Promise<ElementHandle<Element>[]> {
const page = this.getPage();
await this.waitVisible(selector);
return await page.$$(selector);
}
/**
* Get the outer HTML of the first element matching the selector
*/
async getOuterHTML(selector: string): Promise<string> {
const page = this.getPage();
await this.waitVisible(selector);
const html = await page.$eval(selector, (el) => el.outerHTML);
return html;
}
/**
* Get text content from an element
*/
async getText(selector: string): Promise<string> {
const page = this.getPage();
await this.waitVisible(selector);
const text = await page.$eval(selector, (el) => el.textContent || "");
return text.trim();
}
/**
* Get an attribute value from an element
*/
async getAttribute(selector: string, attribute: string): Promise<string> {
const page = this.getPage();
await this.waitVisible(selector);
const value = await page.$eval(
selector,
(el, attr) => el.getAttribute(attr) || "",
attribute
);
return value;
}
/**
* Evaluate JavaScript in the browser context
*/
async evaluate<T>(expression: string): Promise<T> {
const page = this.getPage();
return (await page.evaluate(expression)) as T;
}
/**
* Get the current page URL
*/
async getCurrentLocation(): Promise<string> {
const page = this.getPage();
return page.url();
}
/**
* Get the full HTML of the page
*/
async getHTML(): Promise<string> {
const page = this.getPage();
return await page.content();
}
/**
* Wait for network to be idle
*/
async waitForNetworkIdle(
idleTime: number = 500,
timeout: number = 30000
): Promise<void> {
const page = this.getPage();
await page.waitForNetworkIdle({ idleTime, timeout });
}
/**
* Type text into an input field (simulates keystrokes)
*/
async type(selector: string, value: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.type(selector, value);
}
/**
* Click on an element
*/
async click(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.click(selector);
}
/**
* Select an option from a <select> element by value
*/
async select(selector: string, value: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.select(selector, value);
}
/**
* Set a checkbox to checked or unchecked state
*/
async setCheckbox(selector: string, checked: boolean): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
const isChecked = await page.$eval(
selector,
(el) => (el as HTMLInputElement).checked
);
if (isChecked !== checked) {
await page.click(selector);
}
}
/**
* Click a radio button to select it
*/
async setRadio(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.click(selector);
}
/**
* Upload a file to a file input
*/
async uploadFile(selector: string, filePath: string): Promise<void> {
const page = this.getPage();
await this.waitReady(selector);
const input = (await page.$(selector)) as ElementHandle<HTMLInputElement> | null;
if (!input) {
throw new Error(`File input not found: ${selector}`);
}
await input.uploadFile(filePath);
}
/**
* Set the value of an input field directly (useful for hidden, date, color, etc.)
*/
async setValue(selector: string, value: string): Promise<void> {
const page = this.getPage();
await this.waitReady(selector);
await page.$eval(
selector,
(el, val) => {
(el as HTMLInputElement).value = val;
el.dispatchEvent(new Event("input", { bubbles: true }));
el.dispatchEvent(new Event("change", { bubbles: true }));
},
value
);
}
/**
* Clear the value of an input field
*/
async clear(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
// Triple-click to select all text, then delete
await page.click(selector, { clickCount: 3 });
await page.keyboard.press("Backspace");
}
/**
* Focus on an element
*/
async focus(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.focus(selector);
}
/**
* Take a screenshot
*/
async screenshot(path?: string): Promise<Buffer> {
const page = this.getPage();
const buffer = await page.screenshot({ path });
return buffer as Buffer;
}
/**
* Random delay between min and max milliseconds
*/
async randomDelay(minMs: number, maxMs: number): Promise<void> {
await randomDelay(minMs, maxMs);
}
/**
* Scroll to an element
*/
async scrollToElement(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.$eval(selector, (el) => {
el.scrollIntoView({ behavior: "smooth", block: "center" });
});
}
/**
* Check if an element exists
*/
async exists(selector: string): Promise<boolean> {
const page = this.getPage();
const element = await page.$(selector);
return element !== null;
}
/**
* Get the count of elements matching a selector
*/
async count(selector: string): Promise<number> {
const page = this.getPage();
const elements = await page.$$(selector);
return elements.length;
}
/**
* Press a keyboard key
*/
async pressKey(key: string): Promise<void> {
const page = this.getPage();
await page.keyboard.press(key as any);
}
/**
* Triple-click to select all text in an input
*/
async selectAllText(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.click(selector, { clickCount: 3 });
}
/**
* Clear and type new value (common pattern)
*/
async clearAndType(selector: string, value: string): Promise<void> {
await this.clear(selector);
await this.type(selector, value);
}
}

View file

@ -0,0 +1,18 @@
export { Browser, BrowserOptions } from "./browser";
export {
launchChromeWithDebugging,
waitForChromeReady,
getChromeWebSocketUrl,
ChromeVersionResponse,
} from "./utils/chrome-launcher";
export { randomDelay, sleep } from "./utils/helpers";
export {
OpenRouterService,
OpenRouterServiceOptions,
FieldType,
FormField,
FormExtractionResult,
JobAnalysisResult,
JobDescription,
getMockFormData,
} from "./openrouter";

View file

@ -0,0 +1,12 @@
export { OpenRouterService, OpenRouterServiceOptions } from "./openrouter";
export {
FieldType,
FormField,
FormExtractionResult,
JobAnalysisResult,
JobDescription,
OpenRouterMessage,
OpenRouterRequest,
OpenRouterResponse,
} from "./types";
export { getMockFormData } from "./mock-data";

View file

@ -0,0 +1,102 @@
import { FormExtractionResult } from "./types";
/**
* Returns mock form extraction result for testing
*/
export function getMockFormData(): FormExtractionResult {
return {
fields: [
{
label: "Resume",
field_name: "resume",
field_type: "file",
value: "",
placeholder: "",
required: true,
selector: "input[data-ui='resume']",
},
{
label: "Cover letter",
field_name: "cover_letter",
field_type: "textarea",
value:
"Dear KOMOJU Hiring Team,\n\nI am excited to apply for the Fullstack Developer position at KOMOJU. With over 3 years of experience building web applications and leading development projects, I am confident I can contribute to your Fraud Prevention Team's mission of protecting merchants and customers across borders.\n\nMy experience includes leading the development of online shopping platforms, mentoring junior developers, and coordinating complex projects. I have hands-on experience with modern web technologies including React, Node.js, and microservices architecture. I am a motivated self-starter who can work independently while collaborating effectively with cross-functional teams.\n\nI am eager to bring my technical skills and passion for product-focused development to KOMOJU's innovative payment gateway solutions.\n\nThank you for your consideration.\n\nBest regards,\nZiad Elshimy",
placeholder: "",
required: false,
selector: "#cover_letter",
},
{
label:
"Do you have at least 2 years of personal or professional experience with Ruby?",
field_name: "QA_10609776",
field_type: "radio",
value: "false",
placeholder: "",
required: true,
selector: "input[name='QA_10609776']",
},
{
label: "Could you tell us more about your Ruby experience?",
field_name: "QA_10609777",
field_type: "textarea",
value:
"I do not have professional Ruby experience, but I have extensive experience with JavaScript (React, Node.js), TypeScript, and Go. I am a fast learner with strong fundamentals in software development best practices, data structures, and algorithms. I am excited about the opportunity to learn Ruby and contribute to KOMOJU's fraud prevention systems.",
placeholder: "",
required: true,
selector: "#QA_10609777",
},
{
label:
"Have you ever investigated and resolved a production issue that wasn't reproducible in a development or staging environment?",
field_name: "QA_10609885",
field_type: "radio",
value: "true",
placeholder: "",
required: true,
selector: "input[name='QA_10609885']",
},
{
label:
"If the answer for above question is yes, can you tell us how did you approach it?",
field_name: "QA_10609886",
field_type: "textarea",
value:
"Yes, I have experience debugging production issues. My approach includes: 1) Analyzing production logs and error reports to identify patterns, 2) Using monitoring tools to track system behavior and performance metrics, 3) Creating targeted tests to reproduce the issue in isolated environments, 4) Collaborating with team members to validate hypotheses, and 5) Implementing and deploying fixes with proper testing and monitoring. I understand the importance of being systematic and thorough when diagnosing complex production issues.",
placeholder: "",
required: true,
selector: "#QA_10609886",
},
{
label: "Do you currently reside in Japan?",
field_name: "QA_10609774",
field_type: "radio",
value: "false",
placeholder: "",
required: true,
selector: "input[name='QA_10609774']",
},
{
label:
"This role requires at least 5 hours of overlap with Japan Standard Time (JST) business hours. Are you able to accommodate this?",
field_name: "QA_10609887",
field_type: "radio",
value: "true",
placeholder: "",
required: true,
selector: "input[name='QA_10609887']",
},
{
label:
"Are you willing to relocate to Japan? If so, what are your motivations for doing so?",
field_name: "QA_10609888",
field_type: "textarea",
value:
"Yes, I am willing to relocate to Japan. My motivations include: 1) Joining KOMOJU's innovative team working on cutting-edge payment technology, 2) Experiencing Japan's world-class technology culture and work ethic, 3) Contributing to a product that powers payments for major platforms like Steam and TikTok, 4) Growing my career in an international environment with diverse perspectives, and 5) Embracing the opportunity to learn Japanese culture and language while working on globally impactful projects.",
placeholder: "",
required: true,
selector: "#QA_10609888",
},
],
apply_button: `button[data-ui="apply-button"]`,
};
}

View file

@ -0,0 +1,256 @@
import {
FormExtractionResult,
JobAnalysisResult,
JobDescription,
OpenRouterRequest,
OpenRouterResponse,
} from "./types";
export interface OpenRouterServiceOptions {
model: string;
apiKey: string;
}
export class OpenRouterService {
private model: string;
private apiKey: string;
private baseUrl = "https://openrouter.ai/api/v1/chat/completions";
constructor(options: OpenRouterServiceOptions) {
this.model = options.model;
this.apiKey = options.apiKey;
}
/**
* Clean markdown code blocks from response
*/
private cleanMarkdownCodeBlocks(content: string): string {
content = content.trim();
if (content.startsWith("```json")) {
content = content.slice(7);
} else if (content.startsWith("```")) {
content = content.slice(3);
}
if (content.endsWith("```")) {
content = content.slice(0, -3);
}
return content.trim();
}
/**
* Make a request to OpenRouter API
*/
private async makeRequest(
systemMessage: string,
userMessage: string
): Promise<string> {
const request: OpenRouterRequest = {
model: this.model,
messages: [
{ role: "system", content: systemMessage },
{ role: "user", content: userMessage },
],
};
const response = await fetch(this.baseUrl, {
method: "POST",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify(request),
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(
`OpenRouter API error: ${response.status} - ${errorText}`
);
}
const data = (await response.json()) as OpenRouterResponse;
if (!data.choices || data.choices.length === 0) {
throw new Error("No response choices received from API");
}
const content = data.choices[0].message.content;
if (content.startsWith("<")) {
throw new Error(
`API returned HTML/XML instead of JSON: ${content.slice(0, 200)}`
);
}
return this.cleanMarkdownCodeBlocks(content);
}
/**
* Analyze a job description against a CV
*/
async analyzeJobDescription(
cv: string,
jobDesc: JobDescription
): Promise<JobAnalysisResult> {
const userMessage = `Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
2) The job shouldn't require any language skills, preferbly only english.
3) The job should be remote, or provide relocation to the country.
CV:
${cv}
Job Description:
${jobDesc.description}
Job Criteria (key-value):
${JSON.stringify(jobDesc.criteria, null, 2)}
CRITICAL OUTPUT REQUIREMENTS:
- Return ONLY raw JSON - NO markdown formatting whatsoever
- NO backticks, NO code blocks, NO json prefix
- NO additional text before or after the JSON
- Start your response directly with { and end with }
- Use this exact schema and key names:
{
"recommendation": "apply" | "do_not_apply",
"confidence_score": number, // integer 0-100
"matching_skills": [string],
"missing_skills": [string],
"experience_match": "excellent" | "good" | "fair" | "poor",
"summary": string,
"improvement_suggestions": [string]
}`;
const systemMessage =
"You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
const jsonContent = await this.makeRequest(systemMessage, userMessage);
console.log("Raw API response:", jsonContent);
const result = JSON.parse(jsonContent) as JobAnalysisResult;
return result;
}
/**
* Create a job description from HTML content
*/
async createJobDescription(htmlContent: string): Promise<JobDescription> {
const userMessage = `You are a job posting data extractor. Analyze the HTML content and extract data according to this JSON structure:
{"description":"job description text","criteria":{"title":"Job Title","company":"Company","location":"Location","salary":"Salary","skills":"Skills","experience":"Experience","job_type":"Job Type","remote":"Remote"}}
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON)
3. All criteria values MUST be strings (never use booleans, numbers, or arrays)
4. Properly escape all quotes inside strings using \\"
5. Use only standard ASCII quotes ("), never smart quotes (" " ' ')
7. If no job description is found, use empty string for "description": ""
8. All criteria fields must be present with empty string "" if not found
Example output:
{"description":"Develop software applications...","criteria":{"title":"Backend Engineer","company":"Tech Corp","location":"Remote","salary":"$120k","skills":"Go, Docker","experience":"3+ years","job_type":"Full-time","remote":"Yes"}}
Extract from this HTML Content:
${htmlContent}`;
const systemMessage =
"You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
const jsonContent = await this.makeRequest(systemMessage, userMessage);
console.log("Raw API response for job description:", jsonContent);
const result = JSON.parse(jsonContent) as JobDescription;
return result;
}
/**
* Extract form fields from HTML and populate them based on CV data.
* Also adjusts the CV based on job type (frontend/backend/fullstack).
*/
async applyForJob(
htmlForm: string,
cvMarkdown: string,
jobDescription: string
): Promise<FormExtractionResult> {
const userMessage = `You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
1. Extract ALL form fields and populate them with appropriate values from the CV
HTML FORM:
${htmlForm}
JOB DESCRIPTION:
${jobDescription}
ORIGINAL CV (Markdown):
${cvMarkdown}
CV ADJUSTMENT RULES:
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
- If FULLSTACK: Keep both frontend and backend content.
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
- Maintain the same markdown structure and formatting as the original CV.
- Do NOT invent new skills or experience - only filter existing content.
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Extract ALL input fields, textareas, selects, and buttons from the form
3. For each field, determine the appropriate value from the ADJUSTED CV
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
5. Identify the apply/submit button text
6. Use the exact JSON schema below
7. If a field is already populated, ignore it (don't send it in the results)
JSON Schema:
{
"fields": [
{
"label": "Field label or name attribute",
"field_name": "name or id attribute of the input",
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
"value": "Value to populate based on CV data",
"placeholder": "Placeholder text if any",
"required": true|false,
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
}
],
"apply_button": "Unique CSS selector for the form's submit button",
}
FIELD MAPPING GUIDELINES:
- Name fields: Extract full name, first name, last name from CV
- Email: Use email from CV contact info
- Phone: Use phone number from CV
- LinkedIn/Portfolio/Website: Use URLs from CV
- Experience/Years: Calculate from CV work history
- Current company/title: Use most recent from CV
- Skills: List relevant skills from ADJUSTED CV
- Education: Use education details from CV
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
- Salary expectations: Leave empty unless specified in CV
- Location/Address: Use from CV contact info
- Resume/CV upload: Leave value empty (file upload)
Start your response with { and end with }`;
const systemMessage =
"You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
const jsonContent = await this.makeRequest(systemMessage, userMessage);
console.log("Raw API response for form extraction:", jsonContent);
const result = JSON.parse(jsonContent) as FormExtractionResult;
return result;
}
}

View file

@ -0,0 +1,104 @@
/**
* FieldType represents HTML input field types
*/
export type FieldType =
| "text"
| "email"
| "tel"
| "password"
| "number"
| "date"
| "datetime-local"
| "time"
| "month"
| "week"
| "url"
| "search"
| "color"
| "range"
| "file"
| "hidden"
| "checkbox"
| "radio"
| "select"
| "textarea"
| "button"
| "submit"
| "reset";
/**
* FormField represents a single form field extracted from HTML
*/
export interface FormField {
label: string;
field_name: string;
field_type: FieldType;
value: string;
placeholder?: string;
required: boolean;
selector: string;
}
/**
* FormExtractionResult represents the structured response from form extraction
*/
export interface FormExtractionResult {
fields: FormField[];
apply_button: string;
}
/**
* JobAnalysisResult represents the structured response from job analysis
*/
export interface JobAnalysisResult {
recommendation: "apply" | "do_not_apply";
confidence_score: number;
matching_skills: string[];
missing_skills: string[];
experience_match: "excellent" | "good" | "fair" | "poor";
summary: string;
improvement_suggestions: string[];
}
/**
* JobDescription represents a parsed job description
*/
export interface JobDescription {
description: string;
criteria: Record<string, string>;
}
/**
* OpenRouter API message format
*/
export interface OpenRouterMessage {
role: "system" | "user" | "assistant";
content: string;
}
/**
* OpenRouter API request body
*/
export interface OpenRouterRequest {
model: string;
messages: OpenRouterMessage[];
}
/**
* OpenRouter API response
*/
export interface OpenRouterResponse {
id: string;
choices: {
message: {
role: string;
content: string;
};
finish_reason: string;
}[];
usage?: {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
};
}

View file

@ -0,0 +1,144 @@
import fs from "fs";
import path from "path";
import os from "os";
import { spawn, ChildProcess } from "child_process";
import net from "net";
export interface ChromeVersionResponse {
webSocketDebuggerUrl: string;
}
const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile");
function getChromePath(): string {
const platform = os.platform();
const paths: Record<NodeJS.Platform, string[]> = {
darwin: [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
],
win32: [
process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
process.env["PROGRAMFILES(X86)"] +
"\\Google\\Chrome\\Application\\chrome.exe",
],
linux: [
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
],
aix: [],
freebsd: [],
openbsd: [],
sunos: [],
android: [],
haiku: [],
cygwin: [],
netbsd: [],
};
const platformPaths = paths[platform] || [];
for (const chromePath of platformPaths) {
try {
if (fs.existsSync(chromePath)) {
return chromePath;
}
} catch {
continue;
}
}
throw new Error("Could not find Chrome/Chromium installation");
}
export function launchChromeWithDebugging(
port: number = 9222,
headless: boolean = false
): ChildProcess {
try {
const chromePath = getChromePath();
const args = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${userDataDir}`,
"--remote-allow-origins=*",
"--incognito",
];
if (headless) {
args.push("--headless=new");
}
console.log(`Launching Chrome at: ${chromePath}`);
const chromeProcess = spawn(chromePath, args, {
detached: true,
stdio: "ignore",
});
chromeProcess.unref();
chromeProcess.on("error", (err) => {
console.error("Failed to start Chrome:", err);
});
chromeProcess.on("exit", (code) => {
if (code !== 0) {
console.error(`Chrome process exited with code ${code}`);
}
});
console.log(`Chrome launched successfully with debugging port ${port}`);
return chromeProcess;
} catch (error) {
console.error("Failed to launch Chrome:", error);
throw error;
}
}
export async function waitForChromeReady(
port: number = 9222,
timeout: number = 30000
): Promise<void> {
const startTime = Date.now();
return new Promise((resolve, reject) => {
function check() {
const client = net.createConnection({ port }, () => {
client.end();
resolve();
});
client.on("error", () => {
if (Date.now() - startTime > timeout) {
reject(new Error("Timeout waiting for Chrome to start"));
} else {
setTimeout(check, 500);
}
});
}
check();
});
}
export async function getChromeWebSocketUrl(
port: number = 9222
): Promise<string> {
const response = await fetch(`http://localhost:${port}/json/version`, {
headers: {
Origin: "",
},
});
if (!response.ok) {
throw new Error("Failed to connect to Chrome debugging port");
}
const chromeInstance = (await response.json()) as ChromeVersionResponse;
return chromeInstance.webSocketDebuggerUrl;
}

View file

@ -0,0 +1,14 @@
/**
* Random delay between min and max milliseconds
*/
export function randomDelay(minMs: number, maxMs: number): Promise<void> {
const delay = Math.floor(Math.random() * (maxMs - minMs) + minMs);
return new Promise((resolve) => setTimeout(resolve, delay));
}
/**
* Sleep for a specified number of milliseconds
*/
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View file

@ -0,0 +1,34 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": [
"src/**/*.ts"
],
"exclude": [
"node_modules",
"dist"
]
}

View file

@ -4,6 +4,6 @@ go 1.24.0
toolchain go1.24.7 toolchain go1.24.7
require github.com/jobs-scraper/internal/pkg/domain v0.0.0 require github.com/jobs-scraper/internal/domain v0.0.0
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain replace github.com/jobs-scraper/internal/domain => ../../internal/domain

View file

@ -1,9 +1,10 @@
package ports package ports
import "github.com/jobs-scraper/internal/pkg/domain" import "github.com/jobs-scraper/internal/domain"
type JobCommands struct { type JobCommands struct {
CreateJob JobCommandHandler CreateJob JobCommandHandler
ApplyForJob ApplyForJobCommandHandler
} }
// CreateJobCommand represents the command to create a job // CreateJobCommand represents the command to create a job
@ -19,3 +20,7 @@ type CreateJobCommand struct {
type JobCommandHandler interface { type JobCommandHandler interface {
Handle(cmd CreateJobCommand) error Handle(cmd CreateJobCommand) error
} }
type ApplyForJobCommandHandler interface {
Handle(jobId int) error
}

View file

@ -2,7 +2,7 @@ package ports
import ( import (
"github.com/jobs-scraper/internal/dto" "github.com/jobs-scraper/internal/dto"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
) )
type JobQueries struct { type JobQueries struct {

102
libs/rabbitmq-ts/package-lock.json generated Normal file
View file

@ -0,0 +1,102 @@
{
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"amqplib": "^0.10.9"
},
"devDependencies": {
"@types/amqplib": "^0.10.8",
"typescript": "^5.9.3"
}
},
"node_modules/@types/amqplib": {
"version": "0.10.8",
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz",
"integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/node": {
"version": "25.0.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.3.tgz",
"integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"node_modules/amqplib": {
"version": "0.10.9",
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz",
"integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==",
"license": "MIT",
"dependencies": {
"buffer-more-ints": "~1.0.0",
"url-parse": "~1.5.10"
},
"engines": {
"node": ">=10"
}
},
"node_modules/buffer-more-ints": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz",
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==",
"license": "MIT"
},
"node_modules/querystringify": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
"license": "MIT"
},
"node_modules/requires-port": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
"license": "MIT"
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"dev": true,
"license": "MIT"
},
"node_modules/url-parse": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
"license": "MIT",
"dependencies": {
"querystringify": "^2.1.1",
"requires-port": "^1.0.0"
}
}
}
}

View file

@ -0,0 +1,21 @@
{
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "tsc",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"amqplib": "^0.10.9"
},
"devDependencies": {
"@types/amqplib": "^0.10.8",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,173 @@
import * as amqp from "amqplib";
export interface RabbitMQConfig {
url: string;
queueName: string;
exchangeName: string;
exchangeType: "direct" | "topic" | "fanout" | "headers";
durable: boolean;
}
export interface SearchQuery {
keywords: string;
location: string;
fwt?: string;
numPages: number;
}
export class RabbitMQClient {
private connection: amqp.ChannelModel | null = null;
private channel: amqp.Channel | null = null;
private config: RabbitMQConfig;
constructor(config: RabbitMQConfig) {
this.config = config;
}
async connect(): Promise<void> {
try {
console.log(`Connecting to RabbitMQ at: ${this.config.url}`);
this.connection = await amqp.connect(this.config.url);
console.log("Connected to RabbitMQ successfully");
if (!this.connection) {
throw new Error("Failed to establish RabbitMQ connection");
}
this.channel = await this.connection.createChannel();
if (!this.channel) {
throw new Error("Failed to create RabbitMQ channel");
}
// Ensure exchange and queue exist
await this.channel.assertExchange(
this.config.exchangeName,
this.config.exchangeType,
{ durable: this.config.durable }
);
await this.channel.assertQueue(this.config.queueName, {
durable: this.config.durable,
arguments: {
"x-dead-letter-exchange": "scraper_dlx",
"x-dead-letter-routing-key": this.config.queueName,
"x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds
"x-max-retries": 3,
},
});
await this.channel.bindQueue(
this.config.queueName,
this.config.exchangeName,
this.config.queueName
);
// Set prefetch to process one message at a time
await this.channel.prefetch(1);
// Handle connection events
this.connection.on("close", () => {
console.log("RabbitMQ connection closed");
this.connection = null;
this.channel = null;
});
this.connection.on("error", (error: Error) => {
console.error("RabbitMQ connection error:", error);
this.connection = null;
this.channel = null;
});
} catch (error) {
console.error("Failed to connect to RabbitMQ:", error);
throw error;
}
}
async subscribe<T = SearchQuery>(
messageHandler: (message: T) => Promise<void>
): Promise<void> {
if (!this.channel) {
throw new Error(
"RabbitMQ channel not initialized. Call connect() first."
);
}
console.log(
`Waiting for messages from ${this.config.queueName}. To exit press CTRL+C`
);
await this.channel.consume(
this.config.queueName,
async (msg: amqp.ConsumeMessage | null) => {
if (msg && this.channel) {
try {
const messageContent = msg.content.toString();
console.log(`Received message: ${messageContent}`);
const message: T = JSON.parse(messageContent);
console.log(`Processing message:`, message);
// Process the message using the provided handler
await messageHandler(message);
// Acknowledge the message on success
this.channel.ack(msg);
console.log("Message processed successfully");
} catch (error) {
console.error("Error processing message:", error);
// Reject the message and don't requeue it
if (this.channel) {
this.channel.nack(msg, false, false);
}
}
}
}
);
}
async close(): Promise<void> {
try {
if (this.channel) {
try {
await this.channel.close();
} catch {
// Channel may already be closing
}
this.channel = null;
}
if (this.connection) {
try {
await this.connection.close();
} catch {
// Connection may already be closing
}
this.connection = null;
}
console.log("RabbitMQ connection closed gracefully");
} catch (error) {
console.error("Error closing RabbitMQ connection:", error);
}
}
isConnected(): boolean {
return this.connection !== null && this.channel !== null;
}
}
export function createRabbitMQConfig(): RabbitMQConfig {
return {
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor",
exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange",
exchangeType: "topic",
durable: true,
};
}
export async function createRabbitMQClient(): Promise<RabbitMQClient> {
const config = createRabbitMQConfig();
const client = new RabbitMQClient(config);
await client.connect();
return client;
}

View file

@ -0,0 +1,29 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist"]
}

View file

@ -5,8 +5,8 @@ go 1.24.0
toolchain go1.24.7 toolchain go1.24.7
require ( require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0 github.com/jobs-scraper/internal/domain v0.0.0
github.com/lib/pq v1.10.9 github.com/lib/pq v1.10.9
) )
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain replace github.com/jobs-scraper/internal/domain => ../../internal/domain

View file

@ -4,7 +4,7 @@ import (
"database/sql" "database/sql"
"fmt" "fmt"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/lib/pq" "github.com/lib/pq"
) )

View file

@ -6,7 +6,7 @@ import (
"fmt" "fmt"
"strings" "strings"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
) )
type JobDescriptionRepository struct { type JobDescriptionRepository struct {

View file

@ -3,11 +3,12 @@ package repo
import ( import (
"bytes" "bytes"
"database/sql" "database/sql"
"encoding/json"
"fmt" "fmt"
"strconv" "strconv"
"strings" "strings"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
) )
type JobRepository struct { type JobRepository struct {
@ -104,7 +105,7 @@ type JobFilter struct {
} }
func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) { func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) {
query := "SELECT id, title, company, company_link, location, job_link, job_timestamp FROM jobs" query := "SELECT id, title, company, company_link, location, job_link, job_timestamp, provider FROM jobs"
countQuery := "SELECT COUNT(*) FROM jobs" countQuery := "SELECT COUNT(*) FROM jobs"
var conditions []string var conditions []string
var args []interface{} var args []interface{}
@ -164,7 +165,7 @@ func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error)
var jobs []domain.Job var jobs []domain.Job
for rows.Next() { for rows.Next() {
var job domain.Job var job domain.Job
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime); err != nil { if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime, &job.Provider); err != nil {
return nil, 0, fmt.Errorf("error scanning job row: %v", err) return nil, 0, fmt.Errorf("error scanning job row: %v", err)
} }
jobs = append(jobs, job) jobs = append(jobs, job)
@ -180,7 +181,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
var job domain.Job var job domain.Job
sqlStatement := ` sqlStatement := `
SELECT id, title, company, company_link, location, job_link, job_timestamp SELECT id, title, company, company_link, location, job_link, job_timestamp, provider
FROM jobs FROM jobs
WHERE id = $1 WHERE id = $1
` `
@ -193,6 +194,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
&job.Location, &job.Location,
&job.JobLink, &job.JobLink,
&job.JobPostTime, &job.JobPostTime,
&job.Provider,
) )
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
@ -206,6 +208,49 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
return &job, nil return &job, nil
} }
func (r *JobRepository) GetJobWithDescription(id int) (*domain.JobWithDescription, error) {
var job domain.JobWithDescription
var criteriaJSON []byte
sqlStatement := `
SELECT j.id, j.title, j.company, j.company_link, j.location, j.job_link, j.job_timestamp, j.provider,
COALESCE(jd.description, ''), COALESCE(jd.job_criteria, '{}'::jsonb)
FROM jobs j
LEFT JOIN job_descriptions jd ON j.id = jd.job_id
WHERE j.id = $1
`
err := r.db.QueryRow(sqlStatement, id).Scan(
&job.Job.ID,
&job.Job.Title,
&job.Job.Company,
&job.Job.CompanyLink,
&job.Job.Location,
&job.Job.JobLink,
&job.Job.JobPostTime,
&job.Job.Provider,
&job.JobDescription.Description,
&criteriaJSON,
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("job with ID %d not found", id)
}
if err != nil {
return nil, fmt.Errorf("error querying job: %v", err)
}
// Parse criteria JSON
if err := json.Unmarshal(criteriaJSON, &job.JobDescription.Criteria); err != nil {
job.JobDescription.Criteria = make(map[string]string)
}
job.JobDescription.JobID = job.Job.ID
return &job, nil
}
func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) { func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) {
bf := bytes.Buffer{} bf := bytes.Buffer{}
values := make([]interface{}, 0, len(models)*7) values := make([]interface{}, 0, len(models)*7)

View file

@ -10,8 +10,8 @@ import (
"time" "time"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/infrastructure"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/api/internal/app" "github.com/jobs-scraper/services/api/internal/app"
httpHandler "github.com/jobs-scraper/services/api/pkg/http" httpHandler "github.com/jobs-scraper/services/api/pkg/http"
@ -64,12 +64,12 @@ func main() {
router := mux.NewRouter() router := mux.NewRouter()
// Swagger endpoint
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
router.Use(httpHandler.CORSMiddleware) router.Use(httpHandler.CORSMiddleware)
router.Use(httpHandler.LogsMiddleware) router.Use(httpHandler.LogsMiddleware)
// Swagger endpoint
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
app := app.NewApplication(router, db, rmq) app := app.NewApplication(router, db, rmq)
// Create job analysis result repository // Create job analysis result repository

View file

@ -5,9 +5,9 @@ go 1.24.0
require ( require (
github.com/gorilla/mux v1.8.1 github.com/gorilla/mux v1.8.1
github.com/gorilla/schema v1.4.1 github.com/gorilla/schema v1.4.1
github.com/jobs-scraper/internal/pkg/domain v0.0.0 github.com/jobs-scraper/internal/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/pkg/utils v0.0.0 github.com/jobs-scraper/internal/utils v0.0.0
github.com/jobs-scraper/libs/ports v0.0.0 github.com/jobs-scraper/libs/ports v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0 github.com/jobs-scraper/libs/repo v0.0.0
github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000 github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000
@ -39,11 +39,11 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
) )
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain replace github.com/jobs-scraper/internal/domain => ../../internal/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils replace github.com/jobs-scraper/internal/utils => ../../internal/utils
replace github.com/jobs-scraper/libs/ports => ../../libs/ports replace github.com/jobs-scraper/libs/ports => ../../libs/ports

View file

@ -4,7 +4,7 @@ import (
"database/sql" "database/sql"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/libs/server" "github.com/jobs-scraper/libs/server"
@ -30,7 +30,8 @@ func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient
Router: router, Router: router,
DB: db, DB: db,
JobCommands: &ports.JobCommands{ JobCommands: &ports.JobCommands{
CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq), CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq),
ApplyForJob: jobCommands.NewApplyForJobHandler(jobRepo, rmq),
}, },
JobQueries: &ports.JobQueries{ JobQueries: &ports.JobQueries{
GetJobs: jobQueries.NewGetJobsHandler(jobRepo), GetJobs: jobQueries.NewGetJobsHandler(jobRepo),

View file

@ -0,0 +1,44 @@
package job
import (
"encoding/json"
"github.com/jobs-scraper/internal/dto"
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo"
)
type ApplyForJob struct {
jobRepo *repo.JobRepository
rmq *rabbitmq.RabbitMQClient
}
func NewApplyForJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *ApplyForJob {
return &ApplyForJob{
jobRepo: jobRepo,
rmq: rmq,
}
}
func (aj *ApplyForJob) Handle(jobId int) error {
job, err := aj.jobRepo.GetJobWithDescription(jobId)
if err != nil {
return err
}
jobDTO := dto.JobWithDescriptionFromDomain(*job)
jsonData, err := json.Marshal(jobDTO)
if err != nil {
return err
}
err = aj.rmq.Publish(rabbitmq.JobApplierQueue, rabbitmq.ScraperExchange, jsonData)
if err != nil {
return err
}
return nil
}

View file

@ -5,8 +5,8 @@ import (
"errors" "errors"
"log" "log"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
) )
@ -33,25 +33,21 @@ func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error {
jsonData, err := json.Marshal(jobRequest) jsonData, err := json.Marshal(jobRequest)
if err != nil { if err != nil {
log.Printf("Error marshaling JSON: %v", err)
return err return err
} }
switch cmd.Provider { switch cmd.Provider {
case domain.LinkedIn: case domain.LinkedIn:
if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil { if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
log.Printf("Error publishing LinkedIn message: %v", err)
return err return err
} }
log.Printf("Published LinkedIn job request: %s", string(jsonData)) log.Printf("Published LinkedIn job request: %s", string(jsonData))
case domain.Glassdoor: case domain.Glassdoor:
if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil { if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
log.Printf("Error publishing Glassdoor message: %v", err)
return err return err
} }
log.Printf("Published Glassdoor job request: %s", string(jsonData)) log.Printf("Published Glassdoor job request: %s", string(jsonData))
default: default:
log.Printf("Unsupported job provider: %v", cmd.Provider)
return errors.New("unsupported job provider") return errors.New("unsupported job provider")
} }

View file

@ -9,7 +9,7 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/gorilla/schema" "github.com/gorilla/schema"
"github.com/jobs-scraper/internal/pkg/utils" "github.com/jobs-scraper/internal/utils"
"github.com/jobs-scraper/libs/ports" "github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
) )
@ -43,6 +43,7 @@ func (h *JobHandler) RegisterRoutes(router *mux.Router) {
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET") jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET")
jobs.HandleFunc("/apply/{id}", utils.Make(h.ApplyForJob)).Methods("POST")
} }
// CreateJob handles the creation of a new job // CreateJob handles the creation of a new job
@ -73,6 +74,44 @@ func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
return nil return nil
} }
// Apply applies for a job by id
// @Summary applies for a job
// @Description Applies for a job with the provided id
// @Tags jobs
// @Param id query string false "Job ID"
// @Accept json
// @Produce json
// @Success 201 {object} map[string]string "Successfully applied for job"
// @Failure 400 {object} map[string]string "Invalid request data"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/apply/{id} [post]
func (h *JobHandler) ApplyForJob(w http.ResponseWriter, r *http.Request) error {
jobId := r.URL.Query().Get("id")
if jobId == "" {
slog.Error("Job ID is required")
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("job ID is required"))
}
jobIdInt, err := strconv.Atoi(jobId)
if err != nil {
slog.Error("Invalid Job ID", "error", err)
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
}
err = h.jobCommands.ApplyForJob.Handle(jobIdInt)
if err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusInternalServerError, fmt.Errorf("Internal server error"))
}
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
return nil
}
// GetJobs handles the retrieval of all jobs // GetJobs handles the retrieval of all jobs
// @Summary Gets all jobs // @Summary Gets all jobs
// @Description Gets all jobs with the provided specifications // @Description Gets all jobs with the provided specifications

View file

@ -13,11 +13,17 @@ import (
func CORSMiddleware(next http.Handler) http.Handler { func CORSMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Set CORS headers // Set CORS headers
w.Header().Set("Access-Control-Allow-Origin", "*") origin := r.Header.Get("Origin")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") if origin == "" {
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") origin = "*"
}
w.Header().Set("Access-Control-Allow-Origin", origin)
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With, X-API-Key")
w.Header().Set("Access-Control-Allow-Credentials", "true") w.Header().Set("Access-Control-Allow-Credentials", "true")
w.Header().Set("Access-Control-Max-Age", "86400") w.Header().Set("Access-Control-Max-Age", "86400")
w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range")
// Handle preflight OPTIONS requests // Handle preflight OPTIONS requests
if r.Method == "OPTIONS" { if r.Method == "OPTIONS" {

View file

@ -77,31 +77,25 @@ const docTemplate = `{
} }
], ],
"responses": { "responses": {
"201": { "200": {
"description": "Successfully retrieved jobs", "description": "Successfully retrieved jobs",
"schema": { "schema": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": true
"type": "string"
}
} }
}, },
"400": { "400": {
"description": "Invalid request data", "description": "Invalid request data",
"schema": { "schema": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": true
"type": "string"
}
} }
}, },
"500": { "500": {
"description": "Internal server error", "description": "Internal server error",
"schema": { "schema": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": true
"type": "string"
}
} }
} }
} }
@ -241,6 +235,58 @@ const docTemplate = `{
} }
} }
}, },
"/jobs/apply/{id}": {
"post": {
"description": "Applies for a job with the provided id",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "applies for a job",
"parameters": [
{
"type": "string",
"description": "Job ID",
"name": "id",
"in": "query"
}
],
"responses": {
"201": {
"description": "Successfully applied for job",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/{id}/analysis": { "/jobs/{id}/analysis": {
"get": { "get": {
"description": "Retrieves the analysis result for a specific job", "description": "Retrieves the analysis result for a specific job",

View file

@ -70,31 +70,25 @@
} }
], ],
"responses": { "responses": {
"201": { "200": {
"description": "Successfully retrieved jobs", "description": "Successfully retrieved jobs",
"schema": { "schema": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": true
"type": "string"
}
} }
}, },
"400": { "400": {
"description": "Invalid request data", "description": "Invalid request data",
"schema": { "schema": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": true
"type": "string"
}
} }
}, },
"500": { "500": {
"description": "Internal server error", "description": "Internal server error",
"schema": { "schema": {
"type": "object", "type": "object",
"additionalProperties": { "additionalProperties": true
"type": "string"
}
} }
} }
} }
@ -234,6 +228,58 @@
} }
} }
}, },
"/jobs/apply/{id}": {
"post": {
"description": "Applies for a job with the provided id",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "applies for a job",
"parameters": [
{
"type": "string",
"description": "Job ID",
"name": "id",
"in": "query"
}
],
"responses": {
"201": {
"description": "Successfully applied for job",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/{id}/analysis": { "/jobs/{id}/analysis": {
"get": { "get": {
"description": "Retrieves the analysis result for a specific job", "description": "Retrieves the analysis result for a specific job",

View file

@ -107,23 +107,20 @@ paths:
produces: produces:
- application/json - application/json
responses: responses:
"201": "200":
description: Successfully retrieved jobs description: Successfully retrieved jobs
schema: schema:
additionalProperties: additionalProperties: true
type: string
type: object type: object
"400": "400":
description: Invalid request data description: Invalid request data
schema: schema:
additionalProperties: additionalProperties: true
type: string
type: object type: object
"500": "500":
description: Internal server error description: Internal server error
schema: schema:
additionalProperties: additionalProperties: true
type: string
type: object type: object
summary: Gets all jobs summary: Gets all jobs
tags: tags:
@ -253,4 +250,38 @@ paths:
summary: Get top matching jobs summary: Get top matching jobs
tags: tags:
- jobs - jobs
/jobs/apply/{id}:
post:
consumes:
- application/json
description: Applies for a job with the provided id
parameters:
- description: Job ID
in: query
name: id
type: string
produces:
- application/json
responses:
"201":
description: Successfully applied for job
schema:
additionalProperties:
type: string
type: object
"400":
description: Invalid request data
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal server error
schema:
additionalProperties:
type: string
type: object
summary: applies for a job
tags:
- jobs
swagger: "2.0" swagger: "2.0"

View file

@ -0,0 +1,17 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini
CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
OPENROUTER_API_KEY=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
GEMINI_API_KEY=AIzaSyBkGcsPdqhIrjg-wbLOuhAYmjOCt3A_zxM

Binary file not shown.

View file

@ -0,0 +1,65 @@
# Ziad Elshimy
## Frontend Developer
Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects.
`ziadshimy7@gmail.com`
`01223381370`
`Alexandria`
[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6)
[GitHub](https://github.com/ziadshimy7)
---
## Work Experience
### kari - Fullstack Developer
**Jun 2022 - current**
- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates.
- Led the development of multiple internal projects.
- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills.
- Led task planning and coordinated project timelines, reducing overall project duration by 20%.
### Callibri - Frontend Developer
**Jan 2022 - Mar 2022**
- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement.
- Collaborated closely with senior developers to manage a complex design project, increasing efficiency.
### EJADA - Frontend Developer
**Mar 2021 - Dec 2021**
- Troubleshooted the website's problems and stay up to date on technology.
---
## Education
### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General
**Jan 2017 - Dec 2021**
---
## Skills
- HTML5
- Cascading Style Sheets (CSS)
- SCSS
- Tailwind css
- JavaScript
- TypeScript
- React.js
- Redux.js
- Next.js
- Node.js
- SSR
- Webpack
- Vite
- Go (Golang)
- Microservices
- docker
- docker-compose
---
## Certifications
- CCNA
- The Complete JavaScript Course 2023- Udemy
- React - The Complete Guide - Udemy

287
services/job-applier-ts/package-lock.json generated Normal file
View file

@ -0,0 +1,287 @@
{
"name": "job-applier",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "job-applier",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
"dotenv": "^17.2.3"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
},
"../../libs/browser-automation": {
"name": "@jobs-scraper/browser-automation",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"dotenv": "^17.2.3",
"puppeteer-core": "^24.25.0"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
},
"../../libs/rabbitmq-ts": {
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"amqplib": "^0.10.9"
},
"devDependencies": {
"@types/amqplib": "^0.10.8",
"typescript": "^5.9.3"
}
},
"node_modules/@cspotcode/source-map-support": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
"integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/trace-mapping": "0.3.9"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@jobs-scraper/browser-automation": {
"resolved": "../../libs/browser-automation",
"link": true
},
"node_modules/@jobs-scraper/rabbitmq-ts": {
"resolved": "../../libs/rabbitmq-ts",
"link": true
},
"node_modules/@jridgewell/resolve-uri": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
"integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.5.5",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
"integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
"dev": true,
"license": "MIT"
},
"node_modules/@jridgewell/trace-mapping": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
"integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/resolve-uri": "^3.0.3",
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"node_modules/@tsconfig/node10": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz",
"integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node12": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
"integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node14": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
"integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node16": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "24.10.4",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz",
"integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"node_modules/acorn": {
"version": "8.15.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"dev": true,
"license": "MIT",
"bin": {
"acorn": "bin/acorn"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/acorn-walk": {
"version": "8.3.4",
"resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
"integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
"dev": true,
"license": "MIT",
"dependencies": {
"acorn": "^8.11.0"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/arg": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
"integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
"dev": true,
"license": "MIT"
},
"node_modules/create-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true,
"license": "MIT"
},
"node_modules/diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
"dev": true,
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.3.1"
}
},
"node_modules/dotenv": {
"version": "17.2.3",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
"integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/make-error": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
"dev": true,
"license": "ISC"
},
"node_modules/ts-node": {
"version": "10.9.2",
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
"@tsconfig/node12": "^1.0.7",
"@tsconfig/node14": "^1.0.0",
"@tsconfig/node16": "^1.0.2",
"acorn": "^8.4.1",
"acorn-walk": "^8.1.1",
"arg": "^4.1.0",
"create-require": "^1.1.0",
"diff": "^4.0.1",
"make-error": "^1.1.1",
"v8-compile-cache-lib": "^3.0.1",
"yn": "3.1.1"
},
"bin": {
"ts-node": "dist/bin.js",
"ts-node-cwd": "dist/bin-cwd.js",
"ts-node-esm": "dist/bin-esm.js",
"ts-node-script": "dist/bin-script.js",
"ts-node-transpile-only": "dist/bin-transpile.js",
"ts-script": "dist/bin-script-deprecated.js"
},
"peerDependencies": {
"@swc/core": ">=1.2.50",
"@swc/wasm": ">=1.2.50",
"@types/node": "*",
"typescript": ">=2.7"
},
"peerDependenciesMeta": {
"@swc/core": {
"optional": true
},
"@swc/wasm": {
"optional": true
}
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"dev": true,
"license": "MIT"
},
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
"dev": true,
"license": "MIT"
},
"node_modules/yn": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
"integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
}
}
}

View file

@ -0,0 +1,25 @@
{
"name": "job-applier",
"version": "1.0.0",
"description": "Job application automation service using Puppeteer",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "ts-node src/index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": ["job", "automation", "puppeteer", "application"],
"author": "",
"license": "ISC",
"dependencies": {
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
"dotenv": "^17.2.3"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,347 @@
import fs from "fs";
import path from "path";
import {
Browser,
randomDelay,
OpenRouterService,
FormField,
getMockFormData,
FormExtractionResult,
} from "@jobs-scraper/browser-automation";
/**
* Job represents a job posting
*/
export interface Job {
id: number;
title: string;
company: string;
companyLink: string;
location: string;
jobLink: string;
provider: number;
jobPostTime: string;
}
export interface JobWithDescription {
job: Job;
jobDescription: JobDescription;
}
export interface JobDescription {
description: string;
criteria: Record<string, string>;
}
/**
* Read CV from file
*/
export function readCV(cvPath: string = "cv.txt"): string {
const absolutePath = path.isAbsolute(cvPath)
? cvPath
: path.join(process.cwd(), cvPath);
return fs.readFileSync(absolutePath, "utf-8");
}
/**
* Read CV from file
*/
export function getCVPath(cvPath: string = "CV.pdf"): string {
return path.isAbsolute(cvPath) ? cvPath : path.join(process.cwd(), cvPath);
}
/**
* Handle text, email, tel, url, search, password inputs
*/
async function handleTextInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.clear(field.selector);
await browser.type(field.selector, "");
await browser.type(field.selector, field.value);
}
/**
* Handle textarea inputs
*/
async function handleTextarea(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.clear(field.selector);
await browser.type(field.selector, field.value);
}
/**
* Handle number and range inputs
*/
async function handleNumberInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
/**
* Handle date, datetime-local, time, month, week inputs
*/
async function handleDateInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
/**
* Handle select dropdown inputs
*/
async function handleSelect(browser: Browser, field: FormField): Promise<void> {
if (field.value === "") {
return;
}
await browser.select(field.selector, field.value);
}
/**
* Handle radio button inputs
*/
async function handleRadio(browser: Browser, field: FormField): Promise<void> {
if (field.value === "") {
return;
}
await browser.setRadio(field.selector);
}
/**
* Handle checkbox inputs
*/
async function handleCheckbox(
browser: Browser,
field: FormField
): Promise<void> {
const checked =
field.value === "true" || field.value === "1" || field.value === "yes";
await browser.setCheckbox(field.selector, checked);
}
/**
* Handle file input fields
*/
async function handleFileUpload(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.uploadFile(field.selector, field.value);
}
/**
* Handle color picker inputs
*/
async function handleColorInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
/**
* Handle hidden input fields
*/
async function handleHiddenInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
export interface WorkOptions {
useMockData?: boolean;
openRouterApiKey?: string;
openRouterModel?: string;
}
/**
* Main work function that fills out a job application form
*/
export async function work(
browser: Browser,
job: JobWithDescription,
options: WorkOptions = {}
): Promise<void> {
const { openRouterApiKey = "", openRouterModel = "", useMockData } = options;
try {
await browser.init();
await browser.navigate(`${job.job.jobLink}apply`);
await browser.evaluate<boolean>(
`
(() => {
const acceptBtn = document.querySelector('[data-ui="cookie-consent-accept"]');
if (acceptBtn) {
acceptBtn.click();
return true;
}
return false;
})()
`
);
let result: FormExtractionResult;
const formHtml = await browser.getOuterHTML("form");
// Read the CV
const cv = readCV();
const cvPDFPath = getCVPath();
console.log({ cvPDFPath });
// let resumeButtonClicked: boolean = false;
// while (!resumeButtonClicked) {
// resumeButtonClicked = await browser.evaluate<boolean>(
// `
// (() => {
// const buttons = document.querySelectorAll('button');
// const resumeBtn = Array.from(buttons).find(btn =>
// btn.innerText.toLowerCase().includes('resume') ||
// btn.innerText.toLowerCase().includes('import cv')
// );
// if (resumeBtn) {
// resumeBtn.click();
// return true;
// }
// return false;
// })()
// `
// );
// }
// console.log({ resumeButtonClicked });
// await browser.uploadFile("input#file-upload", cvPDFPath);
await randomDelay(5000, 7000);
// await browser.uploadFile("input#resume-upload-input", cvPDFPath);
if (!useMockData) {
const openRouterService = new OpenRouterService({
apiKey: openRouterApiKey,
model: openRouterModel,
});
console.log("Calling OpenRouter service...");
result = await openRouterService.applyForJob(formHtml, cv, "");
console.log("Form extraction result:", result);
} else {
result = getMockFormData();
}
console.log({ result });
if (result.fields.length === 0) {
console.log("No fields extracted, using mock data");
} else {
for (const field of result.fields) {
await randomDelay(1000, 2000);
await processField(browser, field);
}
}
await randomDelay(160000, 180000);
} finally {
await browser.close();
}
}
/**
* Process a single form field
*/
async function processField(browser: Browser, field: FormField): Promise<void> {
if (field.selector === "") {
return;
}
console.log("Processing field:", field);
try {
switch (field.field_type) {
case "text":
case "email":
case "tel":
case "url":
case "search":
case "password":
await handleTextInput(browser, field);
break;
case "textarea":
await handleTextarea(browser, field);
break;
case "number":
case "range":
await handleNumberInput(browser, field);
break;
case "date":
case "datetime-local":
case "time":
case "month":
case "week":
await handleDateInput(browser, field);
break;
case "select":
await handleSelect(browser, field);
break;
case "radio":
await handleRadio(browser, field);
break;
case "checkbox":
await handleCheckbox(browser, field);
break;
case "file":
await handleFileUpload(browser, field);
break;
case "color":
await handleColorInput(browser, field);
break;
case "hidden":
await handleHiddenInput(browser, field);
break;
case "button":
case "submit":
case "reset":
// Skip buttons - they're handled separately
return;
default:
return;
}
} catch (error) {
console.error(`Error processing field ${field.field_name}:`, error);
// Continue with other fields
}
await randomDelay(1500, 2000);
}

View file

@ -0,0 +1,91 @@
import * as dotenv from "dotenv";
import { RabbitMQClient, RabbitMQConfig } from "@jobs-scraper/rabbitmq-ts";
import { work, JobWithDescription } from "./applier/work";
import { Browser } from "@jobs-scraper/browser-automation";
if (dotenv.config({ path: ".local.env" }).error) {
console.log("No .local.env file found, trying .env");
if (dotenv.config().error) {
console.log("No .env file found, using system environment variables");
}
}
const JOB_APPLIER_QUEUE = "job.applier";
const SCRAPER_EXCHANGE = "scraper_exchange";
let browser = new Browser({ port: 9223 });
let rabbitMQClient: RabbitMQClient | null = null;
function createRabbitMQConfig(): RabbitMQConfig {
return {
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
queueName: JOB_APPLIER_QUEUE,
exchangeName: SCRAPER_EXCHANGE,
exchangeType: "topic",
durable: true,
};
}
async function processJob(job: JobWithDescription): Promise<void> {
console.log(
`Processing job application for: ${job.job.title} at ${job.job.company}`
);
await work(browser, job, {
useMockData: true,
openRouterApiKey: process.env.OPENROUTER_API_KEY,
openRouterModel: process.env.OPENROUTER_MODEL,
});
console.log("Job application completed successfully!");
}
async function main() {
console.log("Starting Job Applier service...");
try {
await browser.init();
const config = createRabbitMQConfig();
rabbitMQClient = new RabbitMQClient(config);
await rabbitMQClient.connect();
await rabbitMQClient.subscribe<JobWithDescription>(processJob);
} catch (error) {
console.error("Error starting service:", error);
await cleanup();
process.exit(1);
}
}
async function cleanup(): Promise<void> {
try {
if (rabbitMQClient) {
await rabbitMQClient.close();
rabbitMQClient = null;
}
await browser.close();
console.log("Cleanup completed");
} catch (error) {
console.error("Error during cleanup:", error);
}
}
// Handle graceful shutdown
process.on("SIGINT", async () => {
console.log("Received SIGINT signal");
await cleanup();
process.exit(0);
});
process.on("SIGTERM", async () => {
console.log("Received SIGTERM signal");
await cleanup();
process.exit(0);
});
main().catch(async (error) => {
console.error("Fatal error:", error);
await cleanup();
process.exit(1);
});

View file

@ -0,0 +1,34 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": [
"src/**/*.ts"
],
"exclude": [
"node_modules",
"dist"
]
}

View file

@ -1,19 +1,15 @@
package analyzer package analyzer
import ( import (
"fmt"
"os" "os"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/pkg/openrouter" "github.com/jobs-scraper/internal/openrouter"
// "github.com/jobs-scraper/internal/pkg/openai"
// "github.com/jobs-scraper/internal/pkg/openrouter"
) )
func GetJobDescription(htmlContent string) (*domain.JobDescription, error) { func GetJobDescription(htmlContent string) (*domain.JobDescription, error) {
openRouterApiKey := os.Getenv("OPENROUTER_API_KEY") openRouterApiKey := os.Getenv("OPENROUTER_API_KEY")
openRouterModel := os.Getenv("OPENROUTER_MODEL") openRouterModel := os.Getenv("OPENROUTER_MODEL")
fmt.Println("key", openRouterApiKey)
client := openrouter.NewOpenRouterService(openRouterModel, openRouterApiKey) client := openrouter.NewOpenRouterService(openRouterModel, openRouterApiKey)
result, err := client.CreateJobDescription(htmlContent) result, err := client.CreateJobDescription(htmlContent)

View file

@ -4,7 +4,7 @@ go 1.24.0
require ( require (
github.com/PuerkitoBio/goquery v1.10.3 github.com/PuerkitoBio/goquery v1.10.3
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 github.com/jobs-scraper/internal/infrastructure v0.0.0
github.com/joho/godotenv v1.5.1 github.com/joho/godotenv v1.5.1
github.com/levmv/sked v0.2.1 github.com/levmv/sked v0.2.1
) )
@ -18,4 +18,4 @@ require (
golang.org/x/net v0.44.0 // indirect golang.org/x/net v0.44.0 // indirect
) )
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure

View file

@ -8,8 +8,8 @@ import (
"syscall" "syscall"
"time" "time"
"github.com/jobs-scraper/internal/pkg/browser" "github.com/jobs-scraper/internal/browser"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper-google/analyzer" "github.com/jobs-scraper/services/scraper-google/analyzer"
si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure" si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
@ -49,7 +49,6 @@ func main() {
sched := sked.New(ctx) sched := sked.New(ctx)
googleLinkStream := make(chan domain.GoogleLink, 100) googleLinkStream := make(chan domain.GoogleLink, 100)
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
// pages := []int{50, 60, 70, 80, 90, 100}
log.Println("Starting scraper with scheduler...") log.Println("Starting scraper with scheduler...")

View file

@ -4,7 +4,7 @@ import (
"database/sql" "database/sql"
"log" "log"
"github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/infrastructure"
"github.com/joho/godotenv" "github.com/joho/godotenv"
) )

View file

@ -1,15 +1,28 @@
package utils package utils
import ( import (
"net/url"
"strconv" "strconv"
"strings" "time"
) )
func BuildUrl(query string, page int) string { func BuildUrl(query string, page int) string {
var url strings.Builder params := url.Values{}
url.WriteString("https://www.google.com/search?q=") params.Add("q", query)
url.WriteString(query) params.Add("start", strconv.Itoa(page))
url.WriteString("&start=")
url.WriteString(strconv.Itoa(page)) now := time.Now()
return url.String() twoWeeksAgo := now.AddDate(0, 0, -14)
startDate := twoWeeksAgo.Format("01/02/2006")
endDate := now.Format("01/02/2006")
tbs := "cdr:1,cd_min:" + startDate + ",cd_max:" + endDate
params.Add("tbs", tbs)
// Optional language bias
// params.Add("hl", "en")
// params.Add("lr", "lang_en")
return "https://www.google.com/search?" + params.Encode()
} }

View file

@ -9,8 +9,8 @@ import (
"time" "time"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/jobs-scraper/internal/pkg/browser" "github.com/jobs-scraper/internal/browser"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
) )
func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) { func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) {

View file

@ -2,10 +2,12 @@ package main
import ( import (
"context" "context"
"fmt"
"log" "log"
"time"
"github.com/jobs-scraper/internal/pkg/browser" "github.com/jobs-scraper/internal/browser"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/services/scraper-google/utils" "github.com/jobs-scraper/services/scraper-google/utils"
) )
@ -21,9 +23,9 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha
browser.RandomDelay(1000, 3000) browser.RandomDelay(1000, 3000)
// query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid" // quer := `site: ashbyhq.com (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en`
// query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` // query := `site:lever.co (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en`
query := `site:workable.com (Software OR Backend OR Full-Stack) Engineer lang:en` query := `site:workable.com (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en`
// url := utils.BuildUrl(query, page) // url := utils.BuildUrl(query, page)
err = tab.Navigate("https://linkedin.com") err = tab.Navigate("https://linkedin.com")
@ -32,8 +34,9 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha
} }
browser.RandomDelay(2000, 2500) browser.RandomDelay(2000, 2500)
fmt.Println(utils.BuildUrl(query, page))
err = tab.Navigate(utils.BuildUrl(query, page)) err = tab.Navigate(utils.BuildUrl(query, page))
err = tab.WaitForNetworkIdle(10 * time.Second)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)

View file

@ -39,10 +39,11 @@ async function cleanup() {
} }
async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
chrome = launchChromeWithDebugging(); let port = 9223;
chrome = launchChromeWithDebugging(port);
await waitForChromeReady(); await waitForChromeReady();
const response = await fetch("http://localhost:9222/json/version", { const response = await fetch(`http://localhost:${port}/json/version`, {
headers: { headers: {
Origin: "", Origin: "",
}, },

View file

@ -70,11 +70,11 @@ function getChromePath(): string {
throw new Error("Could not find Chrome/Chromium installation"); throw new Error("Could not find Chrome/Chromium installation");
} }
export function launchChromeWithDebugging() { export function launchChromeWithDebugging(port: number) {
try { try {
const chromePath = getChromePath(); const chromePath = getChromePath();
const args = [ const args = [
"--remote-debugging-port=9222", `--remote-debugging-port=${port}`,
`--user-data-dir=${userDataDir}`, `--user-data-dir=${userDataDir}`,
"--remote-allow-origins=*", "--remote-allow-origins=*",
"--incognito", "--incognito",

View file

@ -6,9 +6,9 @@ toolchain go1.24.7
require ( require (
github.com/PuerkitoBio/goquery v1.10.3 github.com/PuerkitoBio/goquery v1.10.3
github.com/jobs-scraper/internal/pkg/domain v0.0.0 github.com/jobs-scraper/internal/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 github.com/jobs-scraper/internal/infrastructure v0.0.0
github.com/jobs-scraper/internal/pkg/utils v0.0.0 github.com/jobs-scraper/internal/utils v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0 github.com/jobs-scraper/libs/repo v0.0.0
github.com/joho/godotenv v1.5.1 github.com/joho/godotenv v1.5.1
) )
@ -23,10 +23,10 @@ require (
golang.org/x/net v0.44.0 // indirect golang.org/x/net v0.44.0 // indirect
) )
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain replace github.com/jobs-scraper/internal/domain => ../../internal/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
replace github.com/jobs-scraper/libs/repo => ../../libs/repo replace github.com/jobs-scraper/libs/repo => ../../libs/repo
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils replace github.com/jobs-scraper/internal/utils => ../../internal/utils

View file

@ -10,9 +10,9 @@ import (
"time" "time"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/infrastructure"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper/pipeline" "github.com/jobs-scraper/services/scraper/pipeline"
"github.com/joho/godotenv" "github.com/joho/godotenv"

View file

@ -9,7 +9,7 @@ import (
"time" "time"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
) )

View file

@ -4,7 +4,7 @@ import (
"context" "context"
"log" "log"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
) )
func GetJobs(context context.Context, scraperService *Scraper, searchQuery domain.SearchQuery) <-chan domain.Job { func GetJobs(context context.Context, scraperService *Scraper, searchQuery domain.SearchQuery) <-chan domain.Job {

View file

@ -12,8 +12,8 @@ import (
"time" "time"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/pkg/utils" "github.com/jobs-scraper/internal/utils"
) )
type Config struct { type Config struct {