fix errors and restructure project
This commit is contained in:
Elshimy Ziad Magdy Taha
parent
a281363b61
commit
af52085ea3
93 changed files with 4250 additions and 190 deletions
13
Makefile
13
Makefile
|
|
@ -6,6 +6,7 @@ API_DIR=./services/api
|
|||
SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin
|
||||
SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor
|
||||
SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright
|
||||
JOB_APPLIER_BROWSER_USE_DIR=./services/job-applier-browser-use
|
||||
CRON_ANALYZER_DIR=./apps/cron-analyzer
|
||||
CV_ANALYZER_DIR=./apps/cv-analyzer
|
||||
DOCS_DIR=./services/api/pkg/swagger
|
||||
|
|
@ -71,6 +72,18 @@ docker-scraper-playwright: ## Build and run Playwright scraper in Docker
|
|||
@echo "Running Playwright scraper in Docker..."
|
||||
@docker run --rm --name playwright-scraper playwright-scraper
|
||||
|
||||
.PHONY: setup-job-applier-browser-use
|
||||
setup-job-applier-browser-use: ## Setup the browser-use job applier service
|
||||
@echo "Setting up browser-use job applier service..."
|
||||
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv venv && uv pip install -e .
|
||||
@echo "Installing Chromium browser..."
|
||||
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uvx browser-use install
|
||||
|
||||
.PHONY: run-job-applier-browser-use
|
||||
run-job-applier-browser-use: ## Run the browser-use job applier service
|
||||
@echo "Running browser-use job applier service..."
|
||||
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv run python -m src.main
|
||||
|
||||
# Apps
|
||||
.PHONY: run-cron-analyzer
|
||||
run-cron-analyzer: ## Run the job analysis cron service
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"log"
|
||||
|
||||
"github.com/jobs-scraper/apps/cron-analyzer/internal"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ module github.com/jobs-scraper/apps/cron-analyzer
|
|||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/openrouter v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
)
|
||||
|
|
@ -20,10 +20,10 @@ require (
|
|||
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter
|
||||
replace github.com/jobs-scraper/internal/openrouter => ../../internal/openrouter
|
||||
|
||||
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ import (
|
|||
|
||||
// "path/filepath"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/pkg/openai"
|
||||
"github.com/jobs-scraper/internal/pkg/openrouter"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/openai"
|
||||
"github.com/jobs-scraper/internal/openrouter"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
||||
|
|
|
|||
12
go.work
12
go.work
|
|
@ -3,12 +3,12 @@ go 1.24.0
|
|||
use (
|
||||
.
|
||||
./apps/cron-analyzer
|
||||
./internal/pkg/domain
|
||||
./internal/pkg/gemini
|
||||
./internal/pkg/infrastructure
|
||||
./internal/pkg/openai
|
||||
./internal/pkg/openrouter
|
||||
./internal/pkg/utils
|
||||
./internal/domain
|
||||
./internal/gemini
|
||||
./internal/infrastructure
|
||||
./internal/openai
|
||||
./internal/openrouter
|
||||
./internal/utils
|
||||
./libs/ports
|
||||
./libs/repo
|
||||
./libs/server
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ import (
|
|||
"log"
|
||||
"math/rand"
|
||||
"os/exec"
|
||||
// "strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/dom"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
|
@ -116,7 +118,8 @@ func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
|
|||
var nodes []*cdp.Node
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll),
|
||||
chromedp.WaitVisible(selector, chromedp.ByQueryAll),
|
||||
chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -126,6 +129,22 @@ func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
|
|||
return nodes, nil
|
||||
}
|
||||
|
||||
// GetOuterHTML gets the outer HTML of the first element matching the selector
|
||||
func (b *Browser) GetOuterHTML(selector string) (string, error) {
|
||||
var html string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector, chromedp.ByQuery),
|
||||
chromedp.OuterHTML(selector, &html, chromedp.ByQuery),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return html, nil
|
||||
}
|
||||
|
||||
func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) {
|
||||
var text string
|
||||
|
||||
|
|
@ -151,6 +170,8 @@ func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr str
|
|||
return "", err
|
||||
}
|
||||
|
||||
// value = strings.ReplaceAll(value, "/apply", "")
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
|
|
@ -229,13 +250,154 @@ func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error {
|
|||
}
|
||||
}
|
||||
|
||||
func (b *Browser) NodeToString(node *cdp.Node) (string, error) {
|
||||
var outerHTML string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
h, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outerHTML = h
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return outerHTML, err
|
||||
}
|
||||
|
||||
func (b *Browser) Type(selector string, value string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.SendKeys(selector, value),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Browser) Click(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Click(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Select chooses an option from a <select> element by value
|
||||
func (b *Browser) Select(selector string, value string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.SetValue(selector, value),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetCheckbox sets a checkbox to checked or unchecked state
|
||||
func (b *Browser) SetCheckbox(selector string, checked bool) error {
|
||||
var isChecked bool
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Evaluate(`document.querySelector('`+selector+`').checked`, &isChecked),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isChecked != checked {
|
||||
err = chromedp.Run(b.ctx,
|
||||
chromedp.Click(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetRadio clicks a radio button to select it
|
||||
func (b *Browser) SetRadio(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Click(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadFile sets a file input to the specified file path
|
||||
func (b *Browser) UploadFile(selector string, filePath string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.SetUploadFiles(selector, []string{filePath}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetValue sets the value of an input field directly (useful for hidden, date, color, etc.)
|
||||
func (b *Browser) SetValue(selector string, value string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitReady(selector),
|
||||
chromedp.SetValue(selector, value),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clear clears the value of an input field
|
||||
func (b *Browser) Clear(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Clear(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Focus focuses on an element
|
||||
func (b *Browser) Focus(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Focus(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RandomDelay waits for a random duration between min and max milliseconds
|
||||
func RandomDelay(minMs, maxMs int) {
|
||||
delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond
|
||||
time.Sleep(delay)
|
||||
}
|
||||
|
||||
// Sleep waits for the specified duration
|
||||
func (b *Browser) Sleep(d time.Duration) {
|
||||
time.Sleep(d)
|
||||
}
|
||||
5
internal/domain/go.mod
Normal file
5
internal/domain/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
module github.com/jobs-scraper/internal/domain
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
|
@ -1,15 +1,16 @@
|
|||
package dto
|
||||
|
||||
import "github.com/jobs-scraper/internal/pkg/domain"
|
||||
import "github.com/jobs-scraper/internal/domain"
|
||||
|
||||
type JobDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Company string `json:"company"`
|
||||
CompanyLink string `json:"companyLink"`
|
||||
Location string `json:"location"`
|
||||
JobLink string `json:"jobLink"`
|
||||
JobPostTime string `json:"jobPostTime,omitempty"`
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Company string `json:"company"`
|
||||
CompanyLink string `json:"companyLink"`
|
||||
Location string `json:"location"`
|
||||
JobLink string `json:"jobLink"`
|
||||
JobPostTime string `json:"jobPostTime,omitempty"`
|
||||
Provider domain.JobProvider `json:"provider"`
|
||||
}
|
||||
|
||||
func JobFromDomain(j domain.Job) JobDTO {
|
||||
|
|
@ -25,6 +26,7 @@ func JobFromDomain(j domain.Job) JobDTO {
|
|||
Location: j.Location,
|
||||
JobLink: j.JobLink,
|
||||
JobPostTime: postTime,
|
||||
Provider: j.Provider,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -35,3 +37,17 @@ func JobsFromDomain(jobs []domain.Job) []JobDTO {
|
|||
}
|
||||
return dtos
|
||||
}
|
||||
|
||||
type JobWithDescriptionDTO struct {
|
||||
JobDTO
|
||||
Description string `json:"description"`
|
||||
Criteria map[string]string `json:"criteria,omitempty"`
|
||||
}
|
||||
|
||||
func JobWithDescriptionFromDomain(j domain.JobWithDescription) JobWithDescriptionDTO {
|
||||
return JobWithDescriptionDTO{
|
||||
JobDTO: JobFromDomain(j.Job),
|
||||
Description: j.JobDescription.Description,
|
||||
Criteria: j.JobDescription.Criteria,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"google.golang.org/genai"
|
||||
)
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module github.com/jobs-scraper/internal/pkg/gemini
|
||||
module github.com/jobs-scraper/internal/gemini
|
||||
|
||||
go 1.24.0
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module github.com/jobs-scraper/internal/pkg/infrastructure
|
||||
module github.com/jobs-scraper/internal/infrastructure
|
||||
|
||||
go 1.24.0
|
||||
|
||||
|
|
@ -9,16 +9,16 @@ require (
|
|||
github.com/gorilla/mux v1.8.1
|
||||
github.com/jobs-scraper/libs/ports v0.0.0
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/utils v0.0.0
|
||||
github.com/jobs-scraper/internal/utils v0.0.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/libs/repo => ../../../libs/repo
|
||||
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/utils => ../utils
|
||||
replace github.com/jobs-scraper/internal/utils => ../utils
|
||||
|
||||
replace github.com/jobs-scraper/libs/ports => ../../../libs/ports
|
||||
replace github.com/jobs-scraper/libs/ports => ../../libs/ports
|
||||
|
||||
require (
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
|
|
@ -9,7 +9,7 @@ import (
|
|||
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/internal/pkg/utils"
|
||||
"github.com/jobs-scraper/internal/utils"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
|
@ -21,6 +21,7 @@ const (
|
|||
DeadLetterExchange = "scraper_dlx"
|
||||
CvAnalyzeExchange = "cv_exchange"
|
||||
CvAnalyzeQueue = "cv.analyze"
|
||||
JobApplierQueue = "job.applier"
|
||||
)
|
||||
|
||||
type RabbitMQClient struct {
|
||||
|
|
@ -122,13 +123,14 @@ func (r *RabbitMQClient) setupInfrastructure() error {
|
|||
|
||||
// Define queues with their routing keys
|
||||
queues := map[string]string{
|
||||
LinkedInQueue: "scraper.linkedin",
|
||||
IndeedQueue: "scraper.indeed",
|
||||
BaytQueue: "scraper.bayt",
|
||||
TokyoDevQueue: "scraper.tokyodev",
|
||||
JapanDevQueue: "scraper.japandev",
|
||||
GlassDoorQueue: "scraper.glassdoor",
|
||||
GoogleQueue: "scraper.google",
|
||||
LinkedInQueue: "scraper.linkedin",
|
||||
IndeedQueue: "scraper.indeed",
|
||||
BaytQueue: "scraper.bayt",
|
||||
TokyoDevQueue: "scraper.tokyodev",
|
||||
JapanDevQueue: "scraper.japandev",
|
||||
GlassDoorQueue: "scraper.glassdoor",
|
||||
GoogleQueue: "scraper.google",
|
||||
JobApplierQueue: "job.applier",
|
||||
}
|
||||
|
||||
// Declare queues with dead letter exchange and TTL
|
||||
|
|
@ -36,8 +36,8 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/openai"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/openai"
|
||||
)
|
||||
|
||||
func main() {
|
||||
10
internal/openai/go.mod
Normal file
10
internal/openai/go.mod
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
module github.com/jobs-scraper/internal/openai
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/domain v0.0.0-00010101000000-000000000000
|
||||
github.com/sashabaranov/go-openai v1.41.2
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/domain => ../domain
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package openai
|
||||
|
||||
import "github.com/jobs-scraper/internal/pkg/domain"
|
||||
import "github.com/jobs-scraper/internal/domain"
|
||||
|
||||
// JobAnalysisService defines the interface for job analysis services
|
||||
type JobAnalysisService interface {
|
||||
|
|
@ -8,7 +8,7 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
12
internal/openrouter/go.mod
Normal file
12
internal/openrouter/go.mod
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
module github.com/jobs-scraper/internal/openrouter
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/eduardolat/openroutergo v0.1.0
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
)
|
||||
|
||||
require github.com/orsinium-labs/enum v1.4.0 // indirect
|
||||
|
||||
replace github.com/jobs-scraper/internal/domain => ../domain
|
||||
|
|
@ -7,9 +7,56 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/eduardolat/openroutergo"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
// FieldType represents HTML input field types
|
||||
type FieldType string
|
||||
|
||||
const (
|
||||
FieldTypeText FieldType = "text"
|
||||
FieldTypeEmail FieldType = "email"
|
||||
FieldTypeTel FieldType = "tel"
|
||||
FieldTypePassword FieldType = "password"
|
||||
FieldTypeNumber FieldType = "number"
|
||||
FieldTypeDate FieldType = "date"
|
||||
FieldTypeDatetimeLocal FieldType = "datetime-local"
|
||||
FieldTypeTime FieldType = "time"
|
||||
FieldTypeMonth FieldType = "month"
|
||||
FieldTypeWeek FieldType = "week"
|
||||
FieldTypeURL FieldType = "url"
|
||||
FieldTypeSearch FieldType = "search"
|
||||
FieldTypeColor FieldType = "color"
|
||||
FieldTypeRange FieldType = "range"
|
||||
FieldTypeFile FieldType = "file"
|
||||
FieldTypeHidden FieldType = "hidden"
|
||||
FieldTypeCheckbox FieldType = "checkbox"
|
||||
FieldTypeRadio FieldType = "radio"
|
||||
FieldTypeSelect FieldType = "select"
|
||||
FieldTypeTextarea FieldType = "textarea"
|
||||
FieldTypeButton FieldType = "button"
|
||||
FieldTypeSubmit FieldType = "submit"
|
||||
FieldTypeReset FieldType = "reset"
|
||||
)
|
||||
|
||||
// FormField represents a single form field extracted from HTML
|
||||
type FormField struct {
|
||||
Label string `json:"label"`
|
||||
FieldName string `json:"field_name"`
|
||||
FieldType FieldType `json:"field_type"`
|
||||
Value string `json:"value"`
|
||||
Placeholder string `json:"placeholder,omitempty"`
|
||||
Required bool `json:"required"`
|
||||
Selector string `json:"selector"`
|
||||
}
|
||||
|
||||
// FormExtractionResult represents the structured response from form extraction
|
||||
type FormExtractionResult struct {
|
||||
Fields []FormField `json:"fields"`
|
||||
ApplyButton string `json:"apply_button"`
|
||||
AdjustedCV string `json:"adjusted_cv"`
|
||||
}
|
||||
|
||||
// JobAnalysisResult represents the structured response from job analysis
|
||||
type JobAnalysisResult struct {
|
||||
Recommendation string `json:"recommendation"`
|
||||
|
|
@ -199,3 +246,117 @@ Extract from this HTML Content:
|
|||
|
||||
return jobDescription, nil
|
||||
}
|
||||
|
||||
// ExtractFormFields extracts form fields from HTML and populates them based on CV data.
|
||||
// It also adjusts the CV based on job type (frontend/backend/fullstack).
|
||||
func (s *OpenRouterService) CreateCV(htmlForm string, cvMarkdown string, jobDescription string) (*FormExtractionResult, error) {
|
||||
client, err := openroutergo.
|
||||
NewClient().
|
||||
WithAPIKey(s.apiKey).
|
||||
Create()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create client: %v", err)
|
||||
}
|
||||
|
||||
prompt := fmt.Sprintf(`You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
|
||||
1. Extract ALL form fields and populate them with appropriate values from the CV
|
||||
2. Adjust the CV based on the job type (frontend/backend/fullstack)
|
||||
|
||||
HTML FORM:
|
||||
%s
|
||||
|
||||
JOB DESCRIPTION:
|
||||
%s
|
||||
|
||||
ORIGINAL CV (Markdown):
|
||||
%s
|
||||
|
||||
CV ADJUSTMENT RULES:
|
||||
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
|
||||
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
|
||||
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
|
||||
- If FULLSTACK: Keep both frontend and backend content.
|
||||
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
|
||||
- Maintain the same markdown structure and formatting as the original CV.
|
||||
- Do NOT invent new skills or experience - only filter existing content.
|
||||
|
||||
CRITICAL RULES:
|
||||
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
|
||||
2. Extract ALL input fields, textareas, selects, and buttons from the form
|
||||
3. For each field, determine the appropriate value from the ADJUSTED CV
|
||||
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
|
||||
5. Identify the apply/submit button text
|
||||
6. Use the exact JSON schema below
|
||||
|
||||
JSON Schema:
|
||||
{
|
||||
"fields": [
|
||||
{
|
||||
"label": "Field label or name attribute",
|
||||
"field_name": "name or id attribute of the input",
|
||||
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
|
||||
"value": "Value to populate based on CV data",
|
||||
"placeholder": "Placeholder text if any",
|
||||
"required": true|false,
|
||||
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
|
||||
}
|
||||
],
|
||||
"apply_button": "Unique CSS selector for the form's submit button",
|
||||
"adjusted_cv": "The full adjusted CV in markdown format, tailored to the job type"
|
||||
}
|
||||
|
||||
FIELD MAPPING GUIDELINES:
|
||||
- Name fields: Extract full name, first name, last name from CV
|
||||
- Email: Use email from CV contact info
|
||||
- Phone: Use phone number from CV
|
||||
- LinkedIn/Portfolio/Website: Use URLs from CV
|
||||
- Experience/Years: Calculate from CV work history
|
||||
- Current company/title: Use most recent from CV
|
||||
- Skills: List relevant skills from ADJUSTED CV
|
||||
- Education: Use education details from CV
|
||||
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
|
||||
- Salary expectations: Leave empty unless specified in CV
|
||||
- Location/Address: Use from CV contact info
|
||||
- Resume/CV upload: Leave value empty (file upload)
|
||||
|
||||
Start your response with { and end with }`, htmlForm, jobDescription, cvMarkdown)
|
||||
|
||||
_, resp, err := client.
|
||||
NewChatCompletion().
|
||||
WithModel(s.model).
|
||||
WithSystemMessage("You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.").
|
||||
WithUserMessage(prompt).
|
||||
Execute()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute completion: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("no response choices received from API")
|
||||
}
|
||||
|
||||
jsonContent := resp.Choices[0].Message.Content
|
||||
|
||||
log.Printf("Raw API response for form extraction: %s", jsonContent)
|
||||
|
||||
if len(jsonContent) > 0 && jsonContent[0] == '<' {
|
||||
return nil, fmt.Errorf("API returned HTML/XML instead of JSON. Response: %s", jsonContent[:min(200, len(jsonContent))])
|
||||
}
|
||||
|
||||
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
|
||||
|
||||
var result FormExtractionResult
|
||||
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
|
||||
if syntaxErr, ok := err.(*json.SyntaxError); ok {
|
||||
start := max(int(syntaxErr.Offset)-50, 0)
|
||||
end := int(syntaxErr.Offset) + 50
|
||||
if end > len(jsonContent) {
|
||||
end = len(jsonContent)
|
||||
}
|
||||
log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end])
|
||||
}
|
||||
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(500, len(jsonContent))])
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/domain
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/openai
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000
|
||||
github.com/sashabaranov/go-openai v1.41.2
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../domain
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/openrouter
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/eduardolat/openroutergo v0.1.0
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
)
|
||||
|
||||
require github.com/orsinium-labs/enum v1.4.0 // indirect
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../domain
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/utils
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
5
internal/utils/go.mod
Normal file
5
internal/utils/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
module github.com/jobs-scraper/internal/utils
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
1146
libs/browser-automation/package-lock.json
generated
Normal file
1146
libs/browser-automation/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
25
libs/browser-automation/package.json
Normal file
25
libs/browser-automation/package.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "@jobs-scraper/browser-automation",
|
||||
"version": "1.0.0",
|
||||
"description": "Browser automation library using Puppeteer - ported from Go chromedp implementation",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "ts-node src/index.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": ["puppeteer", "browser", "automation", "scraping"],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"dotenv": "^17.2.3",
|
||||
"puppeteer-core": "^24.25.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
389
libs/browser-automation/src/browser.ts
Normal file
389
libs/browser-automation/src/browser.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
import puppeteer, {
|
||||
Browser as PuppeteerBrowser,
|
||||
Page,
|
||||
ElementHandle,
|
||||
} from "puppeteer-core";
|
||||
import { ChildProcess } from "child_process";
|
||||
import {
|
||||
launchChromeWithDebugging,
|
||||
waitForChromeReady,
|
||||
getChromeWebSocketUrl,
|
||||
} from "./utils/chrome-launcher";
|
||||
import { randomDelay } from "./utils/helpers";
|
||||
|
||||
export interface BrowserOptions {
|
||||
headless?: boolean;
|
||||
port?: number;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
export class Browser {
|
||||
private browser: PuppeteerBrowser | null = null;
|
||||
private page: Page | null = null;
|
||||
private chromeProcess: ChildProcess | null = null;
|
||||
private port: number;
|
||||
private userAgent: string;
|
||||
|
||||
constructor(options: BrowserOptions = {}) {
|
||||
this.port = options.port || 9222;
|
||||
this.userAgent =
|
||||
options.userAgent ||
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the browser by launching Chrome and connecting via Puppeteer
|
||||
*/
|
||||
async init(headless: boolean = false): Promise<void> {
|
||||
this.chromeProcess = launchChromeWithDebugging(this.port, headless);
|
||||
await waitForChromeReady(this.port);
|
||||
|
||||
const wsUrl = await getChromeWebSocketUrl(this.port);
|
||||
|
||||
this.browser = await puppeteer.connect({
|
||||
browserWSEndpoint: wsUrl,
|
||||
defaultViewport: null,
|
||||
});
|
||||
|
||||
console.log("Connected to browser successfully");
|
||||
|
||||
const pages = await this.browser.pages();
|
||||
if (pages.length === 0) {
|
||||
this.page = await this.browser.newPage();
|
||||
} else {
|
||||
this.page = pages[0];
|
||||
}
|
||||
|
||||
await this.page.setUserAgent(this.userAgent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the browser and kill Chrome process
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
if (this.browser) {
|
||||
await this.browser.close();
|
||||
this.browser = null;
|
||||
}
|
||||
|
||||
if (this.chromeProcess) {
|
||||
this.chromeProcess.kill("SIGINT");
|
||||
this.chromeProcess = null;
|
||||
}
|
||||
|
||||
console.log("Browser closed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new tab
|
||||
*/
|
||||
async newTab(): Promise<Page> {
|
||||
if (!this.browser) {
|
||||
throw new Error("Browser not initialized");
|
||||
}
|
||||
return await this.browser.newPage();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current page
|
||||
*/
|
||||
getPage(): Page {
|
||||
if (!this.page) {
|
||||
throw new Error("Page not initialized");
|
||||
}
|
||||
return this.page;
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a URL
|
||||
*/
|
||||
async navigate(url: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await page.goto(url, {
|
||||
waitUntil: ["networkidle2", "domcontentloaded"],
|
||||
timeout: 60000,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an element to be visible
|
||||
*/
|
||||
async waitVisible(
|
||||
selector: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ElementHandle<Element> | null> {
|
||||
const page = this.getPage();
|
||||
return await page.waitForSelector(selector, {
|
||||
visible: true,
|
||||
timeout,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an element to be ready in the DOM (not necessarily visible)
|
||||
*/
|
||||
async waitReady(
|
||||
selector: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ElementHandle<Element> | null> {
|
||||
const page = this.getPage();
|
||||
return await page.waitForSelector(selector, {
|
||||
timeout,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all elements matching a selector
|
||||
*/
|
||||
async getNodes(selector: string): Promise<ElementHandle<Element>[]> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
return await page.$$(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the outer HTML of the first element matching the selector
|
||||
*/
|
||||
async getOuterHTML(selector: string): Promise<string> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
const html = await page.$eval(selector, (el) => el.outerHTML);
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get text content from an element
|
||||
*/
|
||||
async getText(selector: string): Promise<string> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
const text = await page.$eval(selector, (el) => el.textContent || "");
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an attribute value from an element
|
||||
*/
|
||||
async getAttribute(selector: string, attribute: string): Promise<string> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
const value = await page.$eval(
|
||||
selector,
|
||||
(el, attr) => el.getAttribute(attr) || "",
|
||||
attribute
|
||||
);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate JavaScript in the browser context
|
||||
*/
|
||||
async evaluate<T>(expression: string): Promise<T> {
|
||||
const page = this.getPage();
|
||||
return (await page.evaluate(expression)) as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current page URL
|
||||
*/
|
||||
async getCurrentLocation(): Promise<string> {
|
||||
const page = this.getPage();
|
||||
return page.url();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full HTML of the page
|
||||
*/
|
||||
async getHTML(): Promise<string> {
|
||||
const page = this.getPage();
|
||||
return await page.content();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for network to be idle
|
||||
*/
|
||||
async waitForNetworkIdle(
|
||||
idleTime: number = 500,
|
||||
timeout: number = 30000
|
||||
): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await page.waitForNetworkIdle({ idleTime, timeout });
|
||||
}
|
||||
|
||||
/**
|
||||
* Type text into an input field (simulates keystrokes)
|
||||
*/
|
||||
async type(selector: string, value: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.type(selector, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Click on an element
|
||||
*/
|
||||
async click(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.click(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Select an option from a <select> element by value
|
||||
*/
|
||||
async select(selector: string, value: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.select(selector, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a checkbox to checked or unchecked state
|
||||
*/
|
||||
async setCheckbox(selector: string, checked: boolean): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
|
||||
const isChecked = await page.$eval(
|
||||
selector,
|
||||
(el) => (el as HTMLInputElement).checked
|
||||
);
|
||||
|
||||
if (isChecked !== checked) {
|
||||
await page.click(selector);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Click a radio button to select it
|
||||
*/
|
||||
async setRadio(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.click(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to a file input
|
||||
*/
|
||||
async uploadFile(selector: string, filePath: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitReady(selector);
|
||||
|
||||
const input = (await page.$(selector)) as ElementHandle<HTMLInputElement> | null;
|
||||
if (!input) {
|
||||
throw new Error(`File input not found: ${selector}`);
|
||||
}
|
||||
|
||||
await input.uploadFile(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of an input field directly (useful for hidden, date, color, etc.)
|
||||
*/
|
||||
async setValue(selector: string, value: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitReady(selector);
|
||||
|
||||
await page.$eval(
|
||||
selector,
|
||||
(el, val) => {
|
||||
(el as HTMLInputElement).value = val;
|
||||
el.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
el.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
},
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the value of an input field
|
||||
*/
|
||||
async clear(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
|
||||
// Triple-click to select all text, then delete
|
||||
await page.click(selector, { clickCount: 3 });
|
||||
await page.keyboard.press("Backspace");
|
||||
}
|
||||
|
||||
/**
|
||||
* Focus on an element
|
||||
*/
|
||||
async focus(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.focus(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a screenshot
|
||||
*/
|
||||
async screenshot(path?: string): Promise<Buffer> {
|
||||
const page = this.getPage();
|
||||
const buffer = await page.screenshot({ path });
|
||||
return buffer as Buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Random delay between min and max milliseconds
|
||||
*/
|
||||
async randomDelay(minMs: number, maxMs: number): Promise<void> {
|
||||
await randomDelay(minMs, maxMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scroll to an element
|
||||
*/
|
||||
async scrollToElement(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.$eval(selector, (el) => {
|
||||
el.scrollIntoView({ behavior: "smooth", block: "center" });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an element exists
|
||||
*/
|
||||
async exists(selector: string): Promise<boolean> {
|
||||
const page = this.getPage();
|
||||
const element = await page.$(selector);
|
||||
return element !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the count of elements matching a selector
|
||||
*/
|
||||
async count(selector: string): Promise<number> {
|
||||
const page = this.getPage();
|
||||
const elements = await page.$$(selector);
|
||||
return elements.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Press a keyboard key
|
||||
*/
|
||||
async pressKey(key: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await page.keyboard.press(key as any);
|
||||
}
|
||||
|
||||
/**
|
||||
* Triple-click to select all text in an input
|
||||
*/
|
||||
async selectAllText(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.click(selector, { clickCount: 3 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear and type new value (common pattern)
|
||||
*/
|
||||
async clearAndType(selector: string, value: string): Promise<void> {
|
||||
await this.clear(selector);
|
||||
await this.type(selector, value);
|
||||
}
|
||||
}
|
||||
18
libs/browser-automation/src/index.ts
Normal file
18
libs/browser-automation/src/index.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
export { Browser, BrowserOptions } from "./browser";
|
||||
export {
|
||||
launchChromeWithDebugging,
|
||||
waitForChromeReady,
|
||||
getChromeWebSocketUrl,
|
||||
ChromeVersionResponse,
|
||||
} from "./utils/chrome-launcher";
|
||||
export { randomDelay, sleep } from "./utils/helpers";
|
||||
export {
|
||||
OpenRouterService,
|
||||
OpenRouterServiceOptions,
|
||||
FieldType,
|
||||
FormField,
|
||||
FormExtractionResult,
|
||||
JobAnalysisResult,
|
||||
JobDescription,
|
||||
getMockFormData,
|
||||
} from "./openrouter";
|
||||
12
libs/browser-automation/src/openrouter/index.ts
Normal file
12
libs/browser-automation/src/openrouter/index.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
export { OpenRouterService, OpenRouterServiceOptions } from "./openrouter";
|
||||
export {
|
||||
FieldType,
|
||||
FormField,
|
||||
FormExtractionResult,
|
||||
JobAnalysisResult,
|
||||
JobDescription,
|
||||
OpenRouterMessage,
|
||||
OpenRouterRequest,
|
||||
OpenRouterResponse,
|
||||
} from "./types";
|
||||
export { getMockFormData } from "./mock-data";
|
||||
102
libs/browser-automation/src/openrouter/mock-data.ts
Normal file
102
libs/browser-automation/src/openrouter/mock-data.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import { FormExtractionResult } from "./types";
|
||||
|
||||
/**
|
||||
* Returns mock form extraction result for testing
|
||||
*/
|
||||
export function getMockFormData(): FormExtractionResult {
|
||||
return {
|
||||
fields: [
|
||||
{
|
||||
label: "Resume",
|
||||
field_name: "resume",
|
||||
field_type: "file",
|
||||
value: "",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[data-ui='resume']",
|
||||
},
|
||||
{
|
||||
label: "Cover letter",
|
||||
field_name: "cover_letter",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"Dear KOMOJU Hiring Team,\n\nI am excited to apply for the Fullstack Developer position at KOMOJU. With over 3 years of experience building web applications and leading development projects, I am confident I can contribute to your Fraud Prevention Team's mission of protecting merchants and customers across borders.\n\nMy experience includes leading the development of online shopping platforms, mentoring junior developers, and coordinating complex projects. I have hands-on experience with modern web technologies including React, Node.js, and microservices architecture. I am a motivated self-starter who can work independently while collaborating effectively with cross-functional teams.\n\nI am eager to bring my technical skills and passion for product-focused development to KOMOJU's innovative payment gateway solutions.\n\nThank you for your consideration.\n\nBest regards,\nZiad Elshimy",
|
||||
placeholder: "",
|
||||
required: false,
|
||||
selector: "#cover_letter",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"Do you have at least 2 years of personal or professional experience with Ruby?",
|
||||
field_name: "QA_10609776",
|
||||
field_type: "radio",
|
||||
value: "false",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609776']",
|
||||
},
|
||||
{
|
||||
label: "Could you tell us more about your Ruby experience?",
|
||||
field_name: "QA_10609777",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"I do not have professional Ruby experience, but I have extensive experience with JavaScript (React, Node.js), TypeScript, and Go. I am a fast learner with strong fundamentals in software development best practices, data structures, and algorithms. I am excited about the opportunity to learn Ruby and contribute to KOMOJU's fraud prevention systems.",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "#QA_10609777",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"Have you ever investigated and resolved a production issue that wasn't reproducible in a development or staging environment?",
|
||||
field_name: "QA_10609885",
|
||||
field_type: "radio",
|
||||
value: "true",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609885']",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"If the answer for above question is yes, can you tell us how did you approach it?",
|
||||
field_name: "QA_10609886",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"Yes, I have experience debugging production issues. My approach includes: 1) Analyzing production logs and error reports to identify patterns, 2) Using monitoring tools to track system behavior and performance metrics, 3) Creating targeted tests to reproduce the issue in isolated environments, 4) Collaborating with team members to validate hypotheses, and 5) Implementing and deploying fixes with proper testing and monitoring. I understand the importance of being systematic and thorough when diagnosing complex production issues.",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "#QA_10609886",
|
||||
},
|
||||
{
|
||||
label: "Do you currently reside in Japan?",
|
||||
field_name: "QA_10609774",
|
||||
field_type: "radio",
|
||||
value: "false",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609774']",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"This role requires at least 5 hours of overlap with Japan Standard Time (JST) business hours. Are you able to accommodate this?",
|
||||
field_name: "QA_10609887",
|
||||
field_type: "radio",
|
||||
value: "true",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609887']",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"Are you willing to relocate to Japan? If so, what are your motivations for doing so?",
|
||||
field_name: "QA_10609888",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"Yes, I am willing to relocate to Japan. My motivations include: 1) Joining KOMOJU's innovative team working on cutting-edge payment technology, 2) Experiencing Japan's world-class technology culture and work ethic, 3) Contributing to a product that powers payments for major platforms like Steam and TikTok, 4) Growing my career in an international environment with diverse perspectives, and 5) Embracing the opportunity to learn Japanese culture and language while working on globally impactful projects.",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "#QA_10609888",
|
||||
},
|
||||
],
|
||||
apply_button: `button[data-ui="apply-button"]`,
|
||||
};
|
||||
}
|
||||
256
libs/browser-automation/src/openrouter/openrouter.ts
Normal file
256
libs/browser-automation/src/openrouter/openrouter.ts
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
import {
|
||||
FormExtractionResult,
|
||||
JobAnalysisResult,
|
||||
JobDescription,
|
||||
OpenRouterRequest,
|
||||
OpenRouterResponse,
|
||||
} from "./types";
|
||||
|
||||
export interface OpenRouterServiceOptions {
|
||||
model: string;
|
||||
apiKey: string;
|
||||
}
|
||||
|
||||
export class OpenRouterService {
|
||||
private model: string;
|
||||
private apiKey: string;
|
||||
private baseUrl = "https://openrouter.ai/api/v1/chat/completions";
|
||||
|
||||
constructor(options: OpenRouterServiceOptions) {
|
||||
this.model = options.model;
|
||||
this.apiKey = options.apiKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean markdown code blocks from response
|
||||
*/
|
||||
private cleanMarkdownCodeBlocks(content: string): string {
|
||||
content = content.trim();
|
||||
|
||||
if (content.startsWith("```json")) {
|
||||
content = content.slice(7);
|
||||
} else if (content.startsWith("```")) {
|
||||
content = content.slice(3);
|
||||
}
|
||||
|
||||
if (content.endsWith("```")) {
|
||||
content = content.slice(0, -3);
|
||||
}
|
||||
|
||||
return content.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a request to OpenRouter API
|
||||
*/
|
||||
private async makeRequest(
|
||||
systemMessage: string,
|
||||
userMessage: string
|
||||
): Promise<string> {
|
||||
const request: OpenRouterRequest = {
|
||||
model: this.model,
|
||||
messages: [
|
||||
{ role: "system", content: systemMessage },
|
||||
{ role: "user", content: userMessage },
|
||||
],
|
||||
};
|
||||
|
||||
const response = await fetch(this.baseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(
|
||||
`OpenRouter API error: ${response.status} - ${errorText}`
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as OpenRouterResponse;
|
||||
|
||||
if (!data.choices || data.choices.length === 0) {
|
||||
throw new Error("No response choices received from API");
|
||||
}
|
||||
|
||||
const content = data.choices[0].message.content;
|
||||
|
||||
if (content.startsWith("<")) {
|
||||
throw new Error(
|
||||
`API returned HTML/XML instead of JSON: ${content.slice(0, 200)}`
|
||||
);
|
||||
}
|
||||
|
||||
return this.cleanMarkdownCodeBlocks(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a job description against a CV
|
||||
*/
|
||||
async analyzeJobDescription(
|
||||
cv: string,
|
||||
jobDesc: JobDescription
|
||||
): Promise<JobAnalysisResult> {
|
||||
const userMessage = `Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
|
||||
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
|
||||
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
|
||||
|
||||
2) The job shouldn't require any language skills, preferbly only english.
|
||||
|
||||
3) The job should be remote, or provide relocation to the country.
|
||||
|
||||
CV:
|
||||
${cv}
|
||||
|
||||
Job Description:
|
||||
${jobDesc.description}
|
||||
|
||||
Job Criteria (key-value):
|
||||
${JSON.stringify(jobDesc.criteria, null, 2)}
|
||||
|
||||
CRITICAL OUTPUT REQUIREMENTS:
|
||||
- Return ONLY raw JSON - NO markdown formatting whatsoever
|
||||
- NO backticks, NO code blocks, NO json prefix
|
||||
- NO additional text before or after the JSON
|
||||
- Start your response directly with { and end with }
|
||||
- Use this exact schema and key names:
|
||||
{
|
||||
"recommendation": "apply" | "do_not_apply",
|
||||
"confidence_score": number, // integer 0-100
|
||||
"matching_skills": [string],
|
||||
"missing_skills": [string],
|
||||
"experience_match": "excellent" | "good" | "fair" | "poor",
|
||||
"summary": string,
|
||||
"improvement_suggestions": [string]
|
||||
}`;
|
||||
|
||||
const systemMessage =
|
||||
"You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
|
||||
|
||||
const jsonContent = await this.makeRequest(systemMessage, userMessage);
|
||||
|
||||
console.log("Raw API response:", jsonContent);
|
||||
|
||||
const result = JSON.parse(jsonContent) as JobAnalysisResult;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a job description from HTML content
|
||||
*/
|
||||
async createJobDescription(htmlContent: string): Promise<JobDescription> {
|
||||
const userMessage = `You are a job posting data extractor. Analyze the HTML content and extract data according to this JSON structure:
|
||||
{"description":"job description text","criteria":{"title":"Job Title","company":"Company","location":"Location","salary":"Salary","skills":"Skills","experience":"Experience","job_type":"Job Type","remote":"Remote"}}
|
||||
|
||||
CRITICAL RULES:
|
||||
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
|
||||
2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON)
|
||||
3. All criteria values MUST be strings (never use booleans, numbers, or arrays)
|
||||
4. Properly escape all quotes inside strings using \\"
|
||||
5. Use only standard ASCII quotes ("), never smart quotes (" " ' ')
|
||||
7. If no job description is found, use empty string for "description": ""
|
||||
8. All criteria fields must be present with empty string "" if not found
|
||||
|
||||
Example output:
|
||||
{"description":"Develop software applications...","criteria":{"title":"Backend Engineer","company":"Tech Corp","location":"Remote","salary":"$120k","skills":"Go, Docker","experience":"3+ years","job_type":"Full-time","remote":"Yes"}}
|
||||
|
||||
Extract from this HTML Content:
|
||||
${htmlContent}`;
|
||||
|
||||
const systemMessage =
|
||||
"You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
|
||||
|
||||
const jsonContent = await this.makeRequest(systemMessage, userMessage);
|
||||
|
||||
console.log("Raw API response for job description:", jsonContent);
|
||||
|
||||
const result = JSON.parse(jsonContent) as JobDescription;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract form fields from HTML and populate them based on CV data.
|
||||
* Also adjusts the CV based on job type (frontend/backend/fullstack).
|
||||
*/
|
||||
async applyForJob(
|
||||
htmlForm: string,
|
||||
cvMarkdown: string,
|
||||
jobDescription: string
|
||||
): Promise<FormExtractionResult> {
|
||||
const userMessage = `You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
|
||||
1. Extract ALL form fields and populate them with appropriate values from the CV
|
||||
|
||||
HTML FORM:
|
||||
${htmlForm}
|
||||
|
||||
JOB DESCRIPTION:
|
||||
${jobDescription}
|
||||
|
||||
ORIGINAL CV (Markdown):
|
||||
${cvMarkdown}
|
||||
|
||||
CV ADJUSTMENT RULES:
|
||||
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
|
||||
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
|
||||
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
|
||||
- If FULLSTACK: Keep both frontend and backend content.
|
||||
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
|
||||
- Maintain the same markdown structure and formatting as the original CV.
|
||||
- Do NOT invent new skills or experience - only filter existing content.
|
||||
|
||||
CRITICAL RULES:
|
||||
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
|
||||
2. Extract ALL input fields, textareas, selects, and buttons from the form
|
||||
3. For each field, determine the appropriate value from the ADJUSTED CV
|
||||
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
|
||||
5. Identify the apply/submit button text
|
||||
6. Use the exact JSON schema below
|
||||
7. If a field is already populated, ignore it (don't send it in the results)
|
||||
|
||||
JSON Schema:
|
||||
{
|
||||
"fields": [
|
||||
{
|
||||
"label": "Field label or name attribute",
|
||||
"field_name": "name or id attribute of the input",
|
||||
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
|
||||
"value": "Value to populate based on CV data",
|
||||
"placeholder": "Placeholder text if any",
|
||||
"required": true|false,
|
||||
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
|
||||
}
|
||||
],
|
||||
"apply_button": "Unique CSS selector for the form's submit button",
|
||||
}
|
||||
|
||||
FIELD MAPPING GUIDELINES:
|
||||
- Name fields: Extract full name, first name, last name from CV
|
||||
- Email: Use email from CV contact info
|
||||
- Phone: Use phone number from CV
|
||||
- LinkedIn/Portfolio/Website: Use URLs from CV
|
||||
- Experience/Years: Calculate from CV work history
|
||||
- Current company/title: Use most recent from CV
|
||||
- Skills: List relevant skills from ADJUSTED CV
|
||||
- Education: Use education details from CV
|
||||
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
|
||||
- Salary expectations: Leave empty unless specified in CV
|
||||
- Location/Address: Use from CV contact info
|
||||
- Resume/CV upload: Leave value empty (file upload)
|
||||
|
||||
Start your response with { and end with }`;
|
||||
|
||||
const systemMessage =
|
||||
"You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
|
||||
|
||||
const jsonContent = await this.makeRequest(systemMessage, userMessage);
|
||||
|
||||
console.log("Raw API response for form extraction:", jsonContent);
|
||||
|
||||
const result = JSON.parse(jsonContent) as FormExtractionResult;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
104
libs/browser-automation/src/openrouter/types.ts
Normal file
104
libs/browser-automation/src/openrouter/types.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* FieldType represents HTML input field types
|
||||
*/
|
||||
export type FieldType =
|
||||
| "text"
|
||||
| "email"
|
||||
| "tel"
|
||||
| "password"
|
||||
| "number"
|
||||
| "date"
|
||||
| "datetime-local"
|
||||
| "time"
|
||||
| "month"
|
||||
| "week"
|
||||
| "url"
|
||||
| "search"
|
||||
| "color"
|
||||
| "range"
|
||||
| "file"
|
||||
| "hidden"
|
||||
| "checkbox"
|
||||
| "radio"
|
||||
| "select"
|
||||
| "textarea"
|
||||
| "button"
|
||||
| "submit"
|
||||
| "reset";
|
||||
|
||||
/**
|
||||
* FormField represents a single form field extracted from HTML
|
||||
*/
|
||||
export interface FormField {
|
||||
label: string;
|
||||
field_name: string;
|
||||
field_type: FieldType;
|
||||
value: string;
|
||||
placeholder?: string;
|
||||
required: boolean;
|
||||
selector: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* FormExtractionResult represents the structured response from form extraction
|
||||
*/
|
||||
export interface FormExtractionResult {
|
||||
fields: FormField[];
|
||||
apply_button: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* JobAnalysisResult represents the structured response from job analysis
|
||||
*/
|
||||
export interface JobAnalysisResult {
|
||||
recommendation: "apply" | "do_not_apply";
|
||||
confidence_score: number;
|
||||
matching_skills: string[];
|
||||
missing_skills: string[];
|
||||
experience_match: "excellent" | "good" | "fair" | "poor";
|
||||
summary: string;
|
||||
improvement_suggestions: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* JobDescription represents a parsed job description
|
||||
*/
|
||||
export interface JobDescription {
|
||||
description: string;
|
||||
criteria: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter API message format
|
||||
*/
|
||||
export interface OpenRouterMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter API request body
|
||||
*/
|
||||
export interface OpenRouterRequest {
|
||||
model: string;
|
||||
messages: OpenRouterMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter API response
|
||||
*/
|
||||
export interface OpenRouterResponse {
|
||||
id: string;
|
||||
choices: {
|
||||
message: {
|
||||
role: string;
|
||||
content: string;
|
||||
};
|
||||
finish_reason: string;
|
||||
}[];
|
||||
usage?: {
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
};
|
||||
}
|
||||
144
libs/browser-automation/src/utils/chrome-launcher.ts
Normal file
144
libs/browser-automation/src/utils/chrome-launcher.ts
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
import { spawn, ChildProcess } from "child_process";
|
||||
import net from "net";
|
||||
|
||||
export interface ChromeVersionResponse {
|
||||
webSocketDebuggerUrl: string;
|
||||
}
|
||||
|
||||
const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile");
|
||||
|
||||
function getChromePath(): string {
|
||||
const platform = os.platform();
|
||||
|
||||
const paths: Record<NodeJS.Platform, string[]> = {
|
||||
darwin: [
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
],
|
||||
win32: [
|
||||
process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
process.env["PROGRAMFILES(X86)"] +
|
||||
"\\Google\\Chrome\\Application\\chrome.exe",
|
||||
],
|
||||
linux: [
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
],
|
||||
aix: [],
|
||||
freebsd: [],
|
||||
openbsd: [],
|
||||
sunos: [],
|
||||
android: [],
|
||||
haiku: [],
|
||||
cygwin: [],
|
||||
netbsd: [],
|
||||
};
|
||||
|
||||
const platformPaths = paths[platform] || [];
|
||||
|
||||
for (const chromePath of platformPaths) {
|
||||
try {
|
||||
if (fs.existsSync(chromePath)) {
|
||||
return chromePath;
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Could not find Chrome/Chromium installation");
|
||||
}
|
||||
|
||||
export function launchChromeWithDebugging(
|
||||
port: number = 9222,
|
||||
headless: boolean = false
|
||||
): ChildProcess {
|
||||
try {
|
||||
const chromePath = getChromePath();
|
||||
const args = [
|
||||
`--remote-debugging-port=${port}`,
|
||||
`--user-data-dir=${userDataDir}`,
|
||||
"--remote-allow-origins=*",
|
||||
"--incognito",
|
||||
];
|
||||
|
||||
if (headless) {
|
||||
args.push("--headless=new");
|
||||
}
|
||||
|
||||
console.log(`Launching Chrome at: ${chromePath}`);
|
||||
|
||||
const chromeProcess = spawn(chromePath, args, {
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
});
|
||||
|
||||
chromeProcess.unref();
|
||||
|
||||
chromeProcess.on("error", (err) => {
|
||||
console.error("Failed to start Chrome:", err);
|
||||
});
|
||||
|
||||
chromeProcess.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
console.error(`Chrome process exited with code ${code}`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Chrome launched successfully with debugging port ${port}`);
|
||||
return chromeProcess;
|
||||
} catch (error) {
|
||||
console.error("Failed to launch Chrome:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForChromeReady(
|
||||
port: number = 9222,
|
||||
timeout: number = 30000
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
function check() {
|
||||
const client = net.createConnection({ port }, () => {
|
||||
client.end();
|
||||
resolve();
|
||||
});
|
||||
|
||||
client.on("error", () => {
|
||||
if (Date.now() - startTime > timeout) {
|
||||
reject(new Error("Timeout waiting for Chrome to start"));
|
||||
} else {
|
||||
setTimeout(check, 500);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
check();
|
||||
});
|
||||
}
|
||||
|
||||
export async function getChromeWebSocketUrl(
|
||||
port: number = 9222
|
||||
): Promise<string> {
|
||||
const response = await fetch(`http://localhost:${port}/json/version`, {
|
||||
headers: {
|
||||
Origin: "",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to connect to Chrome debugging port");
|
||||
}
|
||||
|
||||
const chromeInstance = (await response.json()) as ChromeVersionResponse;
|
||||
return chromeInstance.webSocketDebuggerUrl;
|
||||
}
|
||||
14
libs/browser-automation/src/utils/helpers.ts
Normal file
14
libs/browser-automation/src/utils/helpers.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/**
|
||||
* Random delay between min and max milliseconds
|
||||
*/
|
||||
export function randomDelay(minMs: number, maxMs: number): Promise<void> {
|
||||
const delay = Math.floor(Math.random() * (maxMs - minMs) + minMs);
|
||||
return new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for a specified number of milliseconds
|
||||
*/
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
34
libs/browser-automation/tsconfig.json
Normal file
34
libs/browser-automation/tsconfig.json
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
]
|
||||
}
|
||||
|
|
@ -4,6 +4,6 @@ go 1.24.0
|
|||
|
||||
toolchain go1.24.7
|
||||
|
||||
require github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
require github.com/jobs-scraper/internal/domain v0.0.0
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
package ports
|
||||
|
||||
import "github.com/jobs-scraper/internal/pkg/domain"
|
||||
import "github.com/jobs-scraper/internal/domain"
|
||||
|
||||
type JobCommands struct {
|
||||
CreateJob JobCommandHandler
|
||||
CreateJob JobCommandHandler
|
||||
ApplyForJob ApplyForJobCommandHandler
|
||||
}
|
||||
|
||||
// CreateJobCommand represents the command to create a job
|
||||
|
|
@ -19,3 +20,7 @@ type CreateJobCommand struct {
|
|||
type JobCommandHandler interface {
|
||||
Handle(cmd CreateJobCommand) error
|
||||
}
|
||||
|
||||
type ApplyForJobCommandHandler interface {
|
||||
Handle(jobId int) error
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package ports
|
|||
|
||||
import (
|
||||
"github.com/jobs-scraper/internal/dto"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
type JobQueries struct {
|
||||
|
|
|
|||
102
libs/rabbitmq-ts/package-lock.json
generated
Normal file
102
libs/rabbitmq-ts/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
{
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"amqplib": "^0.10.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/amqplib": "^0.10.8",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/amqplib": {
|
||||
"version": "0.10.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz",
|
||||
"integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.3.tgz",
|
||||
"integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/amqplib": {
|
||||
"version": "0.10.9",
|
||||
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz",
|
||||
"integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-more-ints": "~1.0.0",
|
||||
"url-parse": "~1.5.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer-more-ints": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz",
|
||||
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/querystringify": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
|
||||
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/requires-port": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
|
||||
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
|
||||
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/url-parse": {
|
||||
"version": "1.5.10",
|
||||
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
|
||||
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"querystringify": "^2.1.1",
|
||||
"requires-port": "^1.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
21
libs/rabbitmq-ts/package.json
Normal file
21
libs/rabbitmq-ts/package.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"amqplib": "^0.10.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/amqplib": "^0.10.8",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
173
libs/rabbitmq-ts/src/index.ts
Normal file
173
libs/rabbitmq-ts/src/index.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
import * as amqp from "amqplib";
|
||||
|
||||
export interface RabbitMQConfig {
|
||||
url: string;
|
||||
queueName: string;
|
||||
exchangeName: string;
|
||||
exchangeType: "direct" | "topic" | "fanout" | "headers";
|
||||
durable: boolean;
|
||||
}
|
||||
|
||||
export interface SearchQuery {
|
||||
keywords: string;
|
||||
location: string;
|
||||
fwt?: string;
|
||||
numPages: number;
|
||||
}
|
||||
|
||||
export class RabbitMQClient {
|
||||
private connection: amqp.ChannelModel | null = null;
|
||||
private channel: amqp.Channel | null = null;
|
||||
private config: RabbitMQConfig;
|
||||
|
||||
constructor(config: RabbitMQConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
try {
|
||||
console.log(`Connecting to RabbitMQ at: ${this.config.url}`);
|
||||
this.connection = await amqp.connect(this.config.url);
|
||||
console.log("Connected to RabbitMQ successfully");
|
||||
|
||||
if (!this.connection) {
|
||||
throw new Error("Failed to establish RabbitMQ connection");
|
||||
}
|
||||
|
||||
this.channel = await this.connection.createChannel();
|
||||
|
||||
if (!this.channel) {
|
||||
throw new Error("Failed to create RabbitMQ channel");
|
||||
}
|
||||
|
||||
// Ensure exchange and queue exist
|
||||
await this.channel.assertExchange(
|
||||
this.config.exchangeName,
|
||||
this.config.exchangeType,
|
||||
{ durable: this.config.durable }
|
||||
);
|
||||
|
||||
await this.channel.assertQueue(this.config.queueName, {
|
||||
durable: this.config.durable,
|
||||
arguments: {
|
||||
"x-dead-letter-exchange": "scraper_dlx",
|
||||
"x-dead-letter-routing-key": this.config.queueName,
|
||||
"x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds
|
||||
"x-max-retries": 3,
|
||||
},
|
||||
});
|
||||
|
||||
await this.channel.bindQueue(
|
||||
this.config.queueName,
|
||||
this.config.exchangeName,
|
||||
this.config.queueName
|
||||
);
|
||||
|
||||
// Set prefetch to process one message at a time
|
||||
await this.channel.prefetch(1);
|
||||
|
||||
// Handle connection events
|
||||
this.connection.on("close", () => {
|
||||
console.log("RabbitMQ connection closed");
|
||||
this.connection = null;
|
||||
this.channel = null;
|
||||
});
|
||||
|
||||
this.connection.on("error", (error: Error) => {
|
||||
console.error("RabbitMQ connection error:", error);
|
||||
this.connection = null;
|
||||
this.channel = null;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to connect to RabbitMQ:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async subscribe<T = SearchQuery>(
|
||||
messageHandler: (message: T) => Promise<void>
|
||||
): Promise<void> {
|
||||
if (!this.channel) {
|
||||
throw new Error(
|
||||
"RabbitMQ channel not initialized. Call connect() first."
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Waiting for messages from ${this.config.queueName}. To exit press CTRL+C`
|
||||
);
|
||||
|
||||
await this.channel.consume(
|
||||
this.config.queueName,
|
||||
async (msg: amqp.ConsumeMessage | null) => {
|
||||
if (msg && this.channel) {
|
||||
try {
|
||||
const messageContent = msg.content.toString();
|
||||
console.log(`Received message: ${messageContent}`);
|
||||
|
||||
const message: T = JSON.parse(messageContent);
|
||||
console.log(`Processing message:`, message);
|
||||
|
||||
// Process the message using the provided handler
|
||||
await messageHandler(message);
|
||||
|
||||
// Acknowledge the message on success
|
||||
this.channel.ack(msg);
|
||||
console.log("Message processed successfully");
|
||||
} catch (error) {
|
||||
console.error("Error processing message:", error);
|
||||
// Reject the message and don't requeue it
|
||||
if (this.channel) {
|
||||
this.channel.nack(msg, false, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
if (this.channel) {
|
||||
try {
|
||||
await this.channel.close();
|
||||
} catch {
|
||||
// Channel may already be closing
|
||||
}
|
||||
this.channel = null;
|
||||
}
|
||||
if (this.connection) {
|
||||
try {
|
||||
await this.connection.close();
|
||||
} catch {
|
||||
// Connection may already be closing
|
||||
}
|
||||
this.connection = null;
|
||||
}
|
||||
console.log("RabbitMQ connection closed gracefully");
|
||||
} catch (error) {
|
||||
console.error("Error closing RabbitMQ connection:", error);
|
||||
}
|
||||
}
|
||||
|
||||
isConnected(): boolean {
|
||||
return this.connection !== null && this.channel !== null;
|
||||
}
|
||||
}
|
||||
|
||||
export function createRabbitMQConfig(): RabbitMQConfig {
|
||||
return {
|
||||
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
|
||||
queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor",
|
||||
exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange",
|
||||
exchangeType: "topic",
|
||||
durable: true,
|
||||
};
|
||||
}
|
||||
|
||||
export async function createRabbitMQClient(): Promise<RabbitMQClient> {
|
||||
const config = createRabbitMQConfig();
|
||||
const client = new RabbitMQClient(config);
|
||||
await client.connect();
|
||||
return client;
|
||||
}
|
||||
29
libs/rabbitmq-ts/tsconfig.json
Normal file
29
libs/rabbitmq-ts/tsconfig.json
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
|
@ -5,8 +5,8 @@ go 1.24.0
|
|||
toolchain go1.24.7
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/lib/pq v1.10.9
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
type JobDescriptionRepository struct {
|
||||
|
|
|
|||
|
|
@ -3,11 +3,12 @@ package repo
|
|||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
type JobRepository struct {
|
||||
|
|
@ -104,7 +105,7 @@ type JobFilter struct {
|
|||
}
|
||||
|
||||
func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) {
|
||||
query := "SELECT id, title, company, company_link, location, job_link, job_timestamp FROM jobs"
|
||||
query := "SELECT id, title, company, company_link, location, job_link, job_timestamp, provider FROM jobs"
|
||||
countQuery := "SELECT COUNT(*) FROM jobs"
|
||||
var conditions []string
|
||||
var args []interface{}
|
||||
|
|
@ -164,7 +165,7 @@ func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error)
|
|||
var jobs []domain.Job
|
||||
for rows.Next() {
|
||||
var job domain.Job
|
||||
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime); err != nil {
|
||||
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime, &job.Provider); err != nil {
|
||||
return nil, 0, fmt.Errorf("error scanning job row: %v", err)
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
|
|
@ -180,7 +181,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
|||
var job domain.Job
|
||||
|
||||
sqlStatement := `
|
||||
SELECT id, title, company, company_link, location, job_link, job_timestamp
|
||||
SELECT id, title, company, company_link, location, job_link, job_timestamp, provider
|
||||
FROM jobs
|
||||
WHERE id = $1
|
||||
`
|
||||
|
|
@ -193,6 +194,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
|||
&job.Location,
|
||||
&job.JobLink,
|
||||
&job.JobPostTime,
|
||||
&job.Provider,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
|
|
@ -206,6 +208,49 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
|||
return &job, nil
|
||||
}
|
||||
|
||||
func (r *JobRepository) GetJobWithDescription(id int) (*domain.JobWithDescription, error) {
|
||||
var job domain.JobWithDescription
|
||||
var criteriaJSON []byte
|
||||
|
||||
sqlStatement := `
|
||||
SELECT j.id, j.title, j.company, j.company_link, j.location, j.job_link, j.job_timestamp, j.provider,
|
||||
COALESCE(jd.description, ''), COALESCE(jd.job_criteria, '{}'::jsonb)
|
||||
FROM jobs j
|
||||
LEFT JOIN job_descriptions jd ON j.id = jd.job_id
|
||||
WHERE j.id = $1
|
||||
`
|
||||
|
||||
err := r.db.QueryRow(sqlStatement, id).Scan(
|
||||
&job.Job.ID,
|
||||
&job.Job.Title,
|
||||
&job.Job.Company,
|
||||
&job.Job.CompanyLink,
|
||||
&job.Job.Location,
|
||||
&job.Job.JobLink,
|
||||
&job.Job.JobPostTime,
|
||||
&job.Job.Provider,
|
||||
&job.JobDescription.Description,
|
||||
&criteriaJSON,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("job with ID %d not found", id)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error querying job: %v", err)
|
||||
}
|
||||
|
||||
// Parse criteria JSON
|
||||
if err := json.Unmarshal(criteriaJSON, &job.JobDescription.Criteria); err != nil {
|
||||
job.JobDescription.Criteria = make(map[string]string)
|
||||
}
|
||||
|
||||
job.JobDescription.JobID = job.Job.ID
|
||||
|
||||
return &job, nil
|
||||
}
|
||||
|
||||
func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) {
|
||||
bf := bytes.Buffer{}
|
||||
values := make([]interface{}, 0, len(models)*7)
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/api/internal/app"
|
||||
httpHandler "github.com/jobs-scraper/services/api/pkg/http"
|
||||
|
|
@ -64,12 +64,12 @@ func main() {
|
|||
|
||||
router := mux.NewRouter()
|
||||
|
||||
// Swagger endpoint
|
||||
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
|
||||
|
||||
router.Use(httpHandler.CORSMiddleware)
|
||||
router.Use(httpHandler.LogsMiddleware)
|
||||
|
||||
// Swagger endpoint
|
||||
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
|
||||
|
||||
app := app.NewApplication(router, db, rmq)
|
||||
|
||||
// Create job analysis result repository
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ go 1.24.0
|
|||
require (
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/gorilla/schema v1.4.1
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/pkg/utils v0.0.0
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/utils v0.0.0
|
||||
github.com/jobs-scraper/libs/ports v0.0.0
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000
|
||||
|
|
@ -39,11 +39,11 @@ require (
|
|||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils
|
||||
replace github.com/jobs-scraper/internal/utils => ../../internal/utils
|
||||
|
||||
replace github.com/jobs-scraper/libs/ports => ../../libs/ports
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"database/sql"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/libs/server"
|
||||
|
|
@ -30,7 +30,8 @@ func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient
|
|||
Router: router,
|
||||
DB: db,
|
||||
JobCommands: &ports.JobCommands{
|
||||
CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq),
|
||||
CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq),
|
||||
ApplyForJob: jobCommands.NewApplyForJobHandler(jobRepo, rmq),
|
||||
},
|
||||
JobQueries: &ports.JobQueries{
|
||||
GetJobs: jobQueries.NewGetJobsHandler(jobRepo),
|
||||
|
|
|
|||
44
services/api/internal/commands/job/apply-for-job.go
Normal file
44
services/api/internal/commands/job/apply-for-job.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package job
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/jobs-scraper/internal/dto"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
||||
type ApplyForJob struct {
|
||||
jobRepo *repo.JobRepository
|
||||
rmq *rabbitmq.RabbitMQClient
|
||||
}
|
||||
|
||||
func NewApplyForJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *ApplyForJob {
|
||||
return &ApplyForJob{
|
||||
jobRepo: jobRepo,
|
||||
rmq: rmq,
|
||||
}
|
||||
}
|
||||
|
||||
func (aj *ApplyForJob) Handle(jobId int) error {
|
||||
job, err := aj.jobRepo.GetJobWithDescription(jobId)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
jobDTO := dto.JobWithDescriptionFromDomain(*job)
|
||||
jsonData, err := json.Marshal(jobDTO)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = aj.rmq.Publish(rabbitmq.JobApplierQueue, rabbitmq.ScraperExchange, jsonData)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -5,8 +5,8 @@ import (
|
|||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
|
@ -33,25 +33,21 @@ func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error {
|
|||
|
||||
jsonData, err := json.Marshal(jobRequest)
|
||||
if err != nil {
|
||||
log.Printf("Error marshaling JSON: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
switch cmd.Provider {
|
||||
case domain.LinkedIn:
|
||||
if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
|
||||
log.Printf("Error publishing LinkedIn message: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Published LinkedIn job request: %s", string(jsonData))
|
||||
case domain.Glassdoor:
|
||||
if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
|
||||
log.Printf("Error publishing Glassdoor message: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Published Glassdoor job request: %s", string(jsonData))
|
||||
default:
|
||||
log.Printf("Unsupported job provider: %v", cmd.Provider)
|
||||
return errors.New("unsupported job provider")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import (
|
|||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/schema"
|
||||
"github.com/jobs-scraper/internal/pkg/utils"
|
||||
"github.com/jobs-scraper/internal/utils"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
|
@ -43,6 +43,7 @@ func (h *JobHandler) RegisterRoutes(router *mux.Router) {
|
|||
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
|
||||
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
|
||||
jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET")
|
||||
jobs.HandleFunc("/apply/{id}", utils.Make(h.ApplyForJob)).Methods("POST")
|
||||
}
|
||||
|
||||
// CreateJob handles the creation of a new job
|
||||
|
|
@ -73,6 +74,44 @@ func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Apply applies for a job by id
|
||||
// @Summary applies for a job
|
||||
// @Description Applies for a job with the provided id
|
||||
// @Tags jobs
|
||||
// @Param id query string false "Job ID"
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Success 201 {object} map[string]string "Successfully applied for job"
|
||||
// @Failure 400 {object} map[string]string "Invalid request data"
|
||||
// @Failure 500 {object} map[string]string "Internal server error"
|
||||
// @Router /jobs/apply/{id} [post]
|
||||
func (h *JobHandler) ApplyForJob(w http.ResponseWriter, r *http.Request) error {
|
||||
jobId := r.URL.Query().Get("id")
|
||||
|
||||
if jobId == "" {
|
||||
slog.Error("Job ID is required")
|
||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("job ID is required"))
|
||||
}
|
||||
|
||||
jobIdInt, err := strconv.Atoi(jobId)
|
||||
|
||||
if err != nil {
|
||||
slog.Error("Invalid Job ID", "error", err)
|
||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
|
||||
}
|
||||
|
||||
err = h.jobCommands.ApplyForJob.Handle(jobIdInt)
|
||||
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
return utils.NewAPIError(http.StatusInternalServerError, fmt.Errorf("Internal server error"))
|
||||
}
|
||||
|
||||
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJobs handles the retrieval of all jobs
|
||||
// @Summary Gets all jobs
|
||||
// @Description Gets all jobs with the provided specifications
|
||||
|
|
|
|||
|
|
@ -13,11 +13,17 @@ import (
|
|||
func CORSMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Set CORS headers
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With")
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
origin = "*"
|
||||
}
|
||||
|
||||
w.Header().Set("Access-Control-Allow-Origin", origin)
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With, X-API-Key")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
w.Header().Set("Access-Control-Max-Age", "86400")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range")
|
||||
|
||||
// Handle preflight OPTIONS requests
|
||||
if r.Method == "OPTIONS" {
|
||||
|
|
|
|||
|
|
@ -77,31 +77,25 @@ const docTemplate = `{
|
|||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"200": {
|
||||
"description": "Successfully retrieved jobs",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -241,6 +235,58 @@ const docTemplate = `{
|
|||
}
|
||||
}
|
||||
},
|
||||
"/jobs/apply/{id}": {
|
||||
"post": {
|
||||
"description": "Applies for a job with the provided id",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"jobs"
|
||||
],
|
||||
"summary": "applies for a job",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Job ID",
|
||||
"name": "id",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Successfully applied for job",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/jobs/{id}/analysis": {
|
||||
"get": {
|
||||
"description": "Retrieves the analysis result for a specific job",
|
||||
|
|
|
|||
|
|
@ -70,31 +70,25 @@
|
|||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"200": {
|
||||
"description": "Successfully retrieved jobs",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -234,6 +228,58 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"/jobs/apply/{id}": {
|
||||
"post": {
|
||||
"description": "Applies for a job with the provided id",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"jobs"
|
||||
],
|
||||
"summary": "applies for a job",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Job ID",
|
||||
"name": "id",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Successfully applied for job",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/jobs/{id}/analysis": {
|
||||
"get": {
|
||||
"description": "Retrieves the analysis result for a specific job",
|
||||
|
|
|
|||
|
|
@ -107,23 +107,20 @@ paths:
|
|||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"201":
|
||||
"200":
|
||||
description: Successfully retrieved jobs
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request data
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Internal server error
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
additionalProperties: true
|
||||
type: object
|
||||
summary: Gets all jobs
|
||||
tags:
|
||||
|
|
@ -253,4 +250,38 @@ paths:
|
|||
summary: Get top matching jobs
|
||||
tags:
|
||||
- jobs
|
||||
/jobs/apply/{id}:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: Applies for a job with the provided id
|
||||
parameters:
|
||||
- description: Job ID
|
||||
in: query
|
||||
name: id
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"201":
|
||||
description: Successfully applied for job
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request data
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
"500":
|
||||
description: Internal server error
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
summary: applies for a job
|
||||
tags:
|
||||
- jobs
|
||||
swagger: "2.0"
|
||||
|
|
|
|||
17
services/job-applier-ts/.env
Normal file
17
services/job-applier-ts/.env
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=your_db_user
|
||||
DB_PASSWORD=your_db_password
|
||||
DB_NAME=linkedin_jobs
|
||||
DB_SSLMODE=disable
|
||||
|
||||
SERVER_PORT=8080
|
||||
SERVER_HOST=localhost
|
||||
|
||||
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
|
||||
OPENAI_BASE_URL=https://hubai.loe.gg/v1
|
||||
OPENAI_MODEL=gpt-4o-mini
|
||||
|
||||
CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
|
||||
OPENROUTER_API_KEY=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
|
||||
GEMINI_API_KEY=AIzaSyBkGcsPdqhIrjg-wbLOuhAYmjOCt3A_zxM
|
||||
BIN
services/job-applier-ts/CV.pdf
Normal file
BIN
services/job-applier-ts/CV.pdf
Normal file
Binary file not shown.
65
services/job-applier-ts/cv.txt
Normal file
65
services/job-applier-ts/cv.txt
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
# Ziad Elshimy
|
||||
## Frontend Developer
|
||||
|
||||
Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects.
|
||||
|
||||
`ziadshimy7@gmail.com`
|
||||
`01223381370`
|
||||
`Alexandria`
|
||||
[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6)
|
||||
[GitHub](https://github.com/ziadshimy7)
|
||||
|
||||
---
|
||||
|
||||
## Work Experience
|
||||
|
||||
### kari - Fullstack Developer
|
||||
**Jun 2022 - current**
|
||||
- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates.
|
||||
- Led the development of multiple internal projects.
|
||||
- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills.
|
||||
- Led task planning and coordinated project timelines, reducing overall project duration by 20%.
|
||||
|
||||
### Callibri - Frontend Developer
|
||||
**Jan 2022 - Mar 2022**
|
||||
- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement.
|
||||
- Collaborated closely with senior developers to manage a complex design project, increasing efficiency.
|
||||
|
||||
### EJADA - Frontend Developer
|
||||
**Mar 2021 - Dec 2021**
|
||||
- Troubleshooted the website's problems and stay up to date on technology.
|
||||
|
||||
---
|
||||
|
||||
## Education
|
||||
|
||||
### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General
|
||||
**Jan 2017 - Dec 2021**
|
||||
|
||||
---
|
||||
|
||||
## Skills
|
||||
- HTML5
|
||||
- Cascading Style Sheets (CSS)
|
||||
- SCSS
|
||||
- Tailwind css
|
||||
- JavaScript
|
||||
- TypeScript
|
||||
- React.js
|
||||
- Redux.js
|
||||
- Next.js
|
||||
- Node.js
|
||||
- SSR
|
||||
- Webpack
|
||||
- Vite
|
||||
- Go (Golang)
|
||||
- Microservices
|
||||
- docker
|
||||
- docker-compose
|
||||
|
||||
---
|
||||
|
||||
## Certifications
|
||||
- CCNA
|
||||
- The Complete JavaScript Course 2023- Udemy
|
||||
- React - The Complete Guide - Udemy
|
||||
287
services/job-applier-ts/package-lock.json
generated
Normal file
287
services/job-applier-ts/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
{
|
||||
"name": "job-applier",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "job-applier",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
|
||||
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
|
||||
"dotenv": "^17.2.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"../../libs/browser-automation": {
|
||||
"name": "@jobs-scraper/browser-automation",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"dotenv": "^17.2.3",
|
||||
"puppeteer-core": "^24.25.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"../../libs/rabbitmq-ts": {
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"amqplib": "^0.10.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/amqplib": "^0.10.8",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@cspotcode/source-map-support": {
|
||||
"version": "0.8.1",
|
||||
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
|
||||
"integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jridgewell/trace-mapping": "0.3.9"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@jobs-scraper/browser-automation": {
|
||||
"resolved": "../../libs/browser-automation",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@jobs-scraper/rabbitmq-ts": {
|
||||
"resolved": "../../libs/rabbitmq-ts",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@jridgewell/resolve-uri": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
|
||||
"integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@jridgewell/sourcemap-codec": {
|
||||
"version": "1.5.5",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
|
||||
"integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@jridgewell/trace-mapping": {
|
||||
"version": "0.3.9",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
|
||||
"integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jridgewell/resolve-uri": "^3.0.3",
|
||||
"@jridgewell/sourcemap-codec": "^1.4.10"
|
||||
}
|
||||
},
|
||||
"node_modules/@tsconfig/node10": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz",
|
||||
"integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@tsconfig/node12": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
|
||||
"integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@tsconfig/node14": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
|
||||
"integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@tsconfig/node16": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
|
||||
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "24.10.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz",
|
||||
"integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/acorn": {
|
||||
"version": "8.15.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
|
||||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/acorn-walk": {
|
||||
"version": "8.3.4",
|
||||
"resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
|
||||
"integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"acorn": "^8.11.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
|
||||
"integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/create-require": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
|
||||
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "17.2.3",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
|
||||
"integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/make-error": {
|
||||
"version": "1.3.6",
|
||||
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
|
||||
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ts-node": {
|
||||
"version": "10.9.2",
|
||||
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
|
||||
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@cspotcode/source-map-support": "^0.8.0",
|
||||
"@tsconfig/node10": "^1.0.7",
|
||||
"@tsconfig/node12": "^1.0.7",
|
||||
"@tsconfig/node14": "^1.0.0",
|
||||
"@tsconfig/node16": "^1.0.2",
|
||||
"acorn": "^8.4.1",
|
||||
"acorn-walk": "^8.1.1",
|
||||
"arg": "^4.1.0",
|
||||
"create-require": "^1.1.0",
|
||||
"diff": "^4.0.1",
|
||||
"make-error": "^1.1.1",
|
||||
"v8-compile-cache-lib": "^3.0.1",
|
||||
"yn": "3.1.1"
|
||||
},
|
||||
"bin": {
|
||||
"ts-node": "dist/bin.js",
|
||||
"ts-node-cwd": "dist/bin-cwd.js",
|
||||
"ts-node-esm": "dist/bin-esm.js",
|
||||
"ts-node-script": "dist/bin-script.js",
|
||||
"ts-node-transpile-only": "dist/bin-transpile.js",
|
||||
"ts-script": "dist/bin-script-deprecated.js"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@swc/core": ">=1.2.50",
|
||||
"@swc/wasm": ">=1.2.50",
|
||||
"@types/node": "*",
|
||||
"typescript": ">=2.7"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@swc/core": {
|
||||
"optional": true
|
||||
},
|
||||
"@swc/wasm": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
|
||||
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/v8-compile-cache-lib": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
|
||||
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/yn": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
|
||||
"integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
25
services/job-applier-ts/package.json
Normal file
25
services/job-applier-ts/package.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "job-applier",
|
||||
"version": "1.0.0",
|
||||
"description": "Job application automation service using Puppeteer",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "ts-node src/index.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": ["job", "automation", "puppeteer", "application"],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
|
||||
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
|
||||
"dotenv": "^17.2.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
347
services/job-applier-ts/src/applier/work.ts
Normal file
347
services/job-applier-ts/src/applier/work.ts
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
import {
|
||||
Browser,
|
||||
randomDelay,
|
||||
OpenRouterService,
|
||||
FormField,
|
||||
getMockFormData,
|
||||
FormExtractionResult,
|
||||
} from "@jobs-scraper/browser-automation";
|
||||
|
||||
/**
|
||||
* Job represents a job posting
|
||||
*/
|
||||
export interface Job {
|
||||
id: number;
|
||||
title: string;
|
||||
company: string;
|
||||
companyLink: string;
|
||||
location: string;
|
||||
jobLink: string;
|
||||
provider: number;
|
||||
jobPostTime: string;
|
||||
}
|
||||
|
||||
export interface JobWithDescription {
|
||||
job: Job;
|
||||
jobDescription: JobDescription;
|
||||
}
|
||||
|
||||
export interface JobDescription {
|
||||
description: string;
|
||||
criteria: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read CV from file
|
||||
*/
|
||||
export function readCV(cvPath: string = "cv.txt"): string {
|
||||
const absolutePath = path.isAbsolute(cvPath)
|
||||
? cvPath
|
||||
: path.join(process.cwd(), cvPath);
|
||||
|
||||
return fs.readFileSync(absolutePath, "utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read CV from file
|
||||
*/
|
||||
export function getCVPath(cvPath: string = "CV.pdf"): string {
|
||||
return path.isAbsolute(cvPath) ? cvPath : path.join(process.cwd(), cvPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle text, email, tel, url, search, password inputs
|
||||
*/
|
||||
async function handleTextInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.clear(field.selector);
|
||||
await browser.type(field.selector, "");
|
||||
await browser.type(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle textarea inputs
|
||||
*/
|
||||
async function handleTextarea(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.clear(field.selector);
|
||||
await browser.type(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle number and range inputs
|
||||
*/
|
||||
async function handleNumberInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle date, datetime-local, time, month, week inputs
|
||||
*/
|
||||
async function handleDateInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle select dropdown inputs
|
||||
*/
|
||||
async function handleSelect(browser: Browser, field: FormField): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.select(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle radio button inputs
|
||||
*/
|
||||
async function handleRadio(browser: Browser, field: FormField): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setRadio(field.selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle checkbox inputs
|
||||
*/
|
||||
async function handleCheckbox(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
const checked =
|
||||
field.value === "true" || field.value === "1" || field.value === "yes";
|
||||
await browser.setCheckbox(field.selector, checked);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file input fields
|
||||
*/
|
||||
async function handleFileUpload(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.uploadFile(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle color picker inputs
|
||||
*/
|
||||
async function handleColorInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle hidden input fields
|
||||
*/
|
||||
async function handleHiddenInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
export interface WorkOptions {
|
||||
useMockData?: boolean;
|
||||
openRouterApiKey?: string;
|
||||
openRouterModel?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main work function that fills out a job application form
|
||||
*/
|
||||
export async function work(
|
||||
browser: Browser,
|
||||
job: JobWithDescription,
|
||||
options: WorkOptions = {}
|
||||
): Promise<void> {
|
||||
const { openRouterApiKey = "", openRouterModel = "", useMockData } = options;
|
||||
|
||||
try {
|
||||
await browser.init();
|
||||
|
||||
await browser.navigate(`${job.job.jobLink}apply`);
|
||||
|
||||
await browser.evaluate<boolean>(
|
||||
`
|
||||
(() => {
|
||||
const acceptBtn = document.querySelector('[data-ui="cookie-consent-accept"]');
|
||||
if (acceptBtn) {
|
||||
acceptBtn.click();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
})()
|
||||
`
|
||||
);
|
||||
|
||||
let result: FormExtractionResult;
|
||||
|
||||
const formHtml = await browser.getOuterHTML("form");
|
||||
|
||||
// Read the CV
|
||||
const cv = readCV();
|
||||
|
||||
const cvPDFPath = getCVPath();
|
||||
|
||||
console.log({ cvPDFPath });
|
||||
|
||||
// let resumeButtonClicked: boolean = false;
|
||||
|
||||
// while (!resumeButtonClicked) {
|
||||
// resumeButtonClicked = await browser.evaluate<boolean>(
|
||||
// `
|
||||
// (() => {
|
||||
// const buttons = document.querySelectorAll('button');
|
||||
// const resumeBtn = Array.from(buttons).find(btn =>
|
||||
// btn.innerText.toLowerCase().includes('resume') ||
|
||||
// btn.innerText.toLowerCase().includes('import cv')
|
||||
// );
|
||||
// if (resumeBtn) {
|
||||
// resumeBtn.click();
|
||||
// return true;
|
||||
// }
|
||||
// return false;
|
||||
// })()
|
||||
// `
|
||||
// );
|
||||
// }
|
||||
|
||||
// console.log({ resumeButtonClicked });
|
||||
// await browser.uploadFile("input#file-upload", cvPDFPath);
|
||||
await randomDelay(5000, 7000);
|
||||
// await browser.uploadFile("input#resume-upload-input", cvPDFPath);
|
||||
|
||||
if (!useMockData) {
|
||||
const openRouterService = new OpenRouterService({
|
||||
apiKey: openRouterApiKey,
|
||||
model: openRouterModel,
|
||||
});
|
||||
console.log("Calling OpenRouter service...");
|
||||
result = await openRouterService.applyForJob(formHtml, cv, "");
|
||||
|
||||
console.log("Form extraction result:", result);
|
||||
} else {
|
||||
result = getMockFormData();
|
||||
}
|
||||
|
||||
console.log({ result });
|
||||
|
||||
if (result.fields.length === 0) {
|
||||
console.log("No fields extracted, using mock data");
|
||||
} else {
|
||||
for (const field of result.fields) {
|
||||
await randomDelay(1000, 2000);
|
||||
await processField(browser, field);
|
||||
}
|
||||
}
|
||||
|
||||
await randomDelay(160000, 180000);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single form field
|
||||
*/
|
||||
async function processField(browser: Browser, field: FormField): Promise<void> {
|
||||
if (field.selector === "") {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("Processing field:", field);
|
||||
|
||||
try {
|
||||
switch (field.field_type) {
|
||||
case "text":
|
||||
case "email":
|
||||
case "tel":
|
||||
case "url":
|
||||
case "search":
|
||||
case "password":
|
||||
await handleTextInput(browser, field);
|
||||
break;
|
||||
case "textarea":
|
||||
await handleTextarea(browser, field);
|
||||
break;
|
||||
case "number":
|
||||
case "range":
|
||||
await handleNumberInput(browser, field);
|
||||
break;
|
||||
case "date":
|
||||
case "datetime-local":
|
||||
case "time":
|
||||
case "month":
|
||||
case "week":
|
||||
await handleDateInput(browser, field);
|
||||
break;
|
||||
case "select":
|
||||
await handleSelect(browser, field);
|
||||
break;
|
||||
case "radio":
|
||||
await handleRadio(browser, field);
|
||||
break;
|
||||
case "checkbox":
|
||||
await handleCheckbox(browser, field);
|
||||
break;
|
||||
case "file":
|
||||
await handleFileUpload(browser, field);
|
||||
break;
|
||||
case "color":
|
||||
await handleColorInput(browser, field);
|
||||
break;
|
||||
case "hidden":
|
||||
await handleHiddenInput(browser, field);
|
||||
break;
|
||||
case "button":
|
||||
case "submit":
|
||||
case "reset":
|
||||
// Skip buttons - they're handled separately
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error processing field ${field.field_name}:`, error);
|
||||
// Continue with other fields
|
||||
}
|
||||
|
||||
await randomDelay(1500, 2000);
|
||||
}
|
||||
91
services/job-applier-ts/src/index.ts
Normal file
91
services/job-applier-ts/src/index.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import * as dotenv from "dotenv";
|
||||
import { RabbitMQClient, RabbitMQConfig } from "@jobs-scraper/rabbitmq-ts";
|
||||
import { work, JobWithDescription } from "./applier/work";
|
||||
import { Browser } from "@jobs-scraper/browser-automation";
|
||||
|
||||
if (dotenv.config({ path: ".local.env" }).error) {
|
||||
console.log("No .local.env file found, trying .env");
|
||||
if (dotenv.config().error) {
|
||||
console.log("No .env file found, using system environment variables");
|
||||
}
|
||||
}
|
||||
|
||||
const JOB_APPLIER_QUEUE = "job.applier";
|
||||
const SCRAPER_EXCHANGE = "scraper_exchange";
|
||||
|
||||
let browser = new Browser({ port: 9223 });
|
||||
let rabbitMQClient: RabbitMQClient | null = null;
|
||||
|
||||
function createRabbitMQConfig(): RabbitMQConfig {
|
||||
return {
|
||||
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
|
||||
queueName: JOB_APPLIER_QUEUE,
|
||||
exchangeName: SCRAPER_EXCHANGE,
|
||||
exchangeType: "topic",
|
||||
durable: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function processJob(job: JobWithDescription): Promise<void> {
|
||||
console.log(
|
||||
`Processing job application for: ${job.job.title} at ${job.job.company}`
|
||||
);
|
||||
|
||||
await work(browser, job, {
|
||||
useMockData: true,
|
||||
openRouterApiKey: process.env.OPENROUTER_API_KEY,
|
||||
openRouterModel: process.env.OPENROUTER_MODEL,
|
||||
});
|
||||
|
||||
console.log("Job application completed successfully!");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("Starting Job Applier service...");
|
||||
|
||||
try {
|
||||
await browser.init();
|
||||
|
||||
const config = createRabbitMQConfig();
|
||||
rabbitMQClient = new RabbitMQClient(config);
|
||||
await rabbitMQClient.connect();
|
||||
|
||||
await rabbitMQClient.subscribe<JobWithDescription>(processJob);
|
||||
} catch (error) {
|
||||
console.error("Error starting service:", error);
|
||||
await cleanup();
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
try {
|
||||
if (rabbitMQClient) {
|
||||
await rabbitMQClient.close();
|
||||
rabbitMQClient = null;
|
||||
}
|
||||
await browser.close();
|
||||
console.log("Cleanup completed");
|
||||
} catch (error) {
|
||||
console.error("Error during cleanup:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle graceful shutdown
|
||||
process.on("SIGINT", async () => {
|
||||
console.log("Received SIGINT signal");
|
||||
await cleanup();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", async () => {
|
||||
console.log("Received SIGTERM signal");
|
||||
await cleanup();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
main().catch(async (error) => {
|
||||
console.error("Fatal error:", error);
|
||||
await cleanup();
|
||||
process.exit(1);
|
||||
});
|
||||
34
services/job-applier-ts/tsconfig.json
Normal file
34
services/job-applier-ts/tsconfig.json
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
]
|
||||
}
|
||||
|
|
@ -1,19 +1,15 @@
|
|||
package analyzer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/openrouter"
|
||||
// "github.com/jobs-scraper/internal/pkg/openai"
|
||||
// "github.com/jobs-scraper/internal/pkg/openrouter"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/openrouter"
|
||||
)
|
||||
|
||||
func GetJobDescription(htmlContent string) (*domain.JobDescription, error) {
|
||||
openRouterApiKey := os.Getenv("OPENROUTER_API_KEY")
|
||||
openRouterModel := os.Getenv("OPENROUTER_MODEL")
|
||||
fmt.Println("key", openRouterApiKey)
|
||||
|
||||
client := openrouter.NewOpenRouterService(openRouterModel, openRouterApiKey)
|
||||
result, err := client.CreateJobDescription(htmlContent)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ go 1.24.0
|
|||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/levmv/sked v0.2.1
|
||||
)
|
||||
|
|
@ -18,4 +18,4 @@ require (
|
|||
golang.org/x/net v0.44.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
|
|
|||
|
|
@ -8,8 +8,8 @@ import (
|
|||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/browser"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/browser"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/scraper-google/analyzer"
|
||||
si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
|
||||
|
|
@ -49,7 +49,6 @@ func main() {
|
|||
sched := sked.New(ctx)
|
||||
googleLinkStream := make(chan domain.GoogleLink, 100)
|
||||
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
|
||||
// pages := []int{50, 60, 70, 80, 90, 100}
|
||||
|
||||
log.Println("Starting scraper with scheduler...")
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"database/sql"
|
||||
"log"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,28 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func BuildUrl(query string, page int) string {
|
||||
var url strings.Builder
|
||||
url.WriteString("https://www.google.com/search?q=")
|
||||
url.WriteString(query)
|
||||
url.WriteString("&start=")
|
||||
url.WriteString(strconv.Itoa(page))
|
||||
return url.String()
|
||||
params := url.Values{}
|
||||
params.Add("q", query)
|
||||
params.Add("start", strconv.Itoa(page))
|
||||
|
||||
now := time.Now()
|
||||
twoWeeksAgo := now.AddDate(0, 0, -14)
|
||||
|
||||
startDate := twoWeeksAgo.Format("01/02/2006")
|
||||
endDate := now.Format("01/02/2006")
|
||||
|
||||
tbs := "cdr:1,cd_min:" + startDate + ",cd_max:" + endDate
|
||||
params.Add("tbs", tbs)
|
||||
|
||||
// Optional language bias
|
||||
// params.Add("hl", "en")
|
||||
// params.Add("lr", "lang_en")
|
||||
|
||||
return "https://www.google.com/search?" + params.Encode()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,8 +9,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/jobs-scraper/internal/pkg/browser"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/browser"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
func GetHTMLRaw(ctx context.Context, b *browser.Browser, googleLink domain.GoogleLink) (string, string, error) {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/browser"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/browser"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/services/scraper-google/utils"
|
||||
)
|
||||
|
||||
|
|
@ -21,9 +23,9 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha
|
|||
|
||||
browser.RandomDelay(1000, 3000)
|
||||
|
||||
// query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid"
|
||||
// query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en`
|
||||
query := `site:workable.com (Software OR Backend OR Full-Stack) Engineer lang:en`
|
||||
// quer := `site: ashbyhq.com (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en`
|
||||
// query := `site:lever.co (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en`
|
||||
query := `site:workable.com (Software OR Backend OR Full-Stack OR Frontend) Engineer lang:en`
|
||||
// url := utils.BuildUrl(query, page)
|
||||
err = tab.Navigate("https://linkedin.com")
|
||||
|
||||
|
|
@ -32,8 +34,9 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha
|
|||
}
|
||||
|
||||
browser.RandomDelay(2000, 2500)
|
||||
|
||||
fmt.Println(utils.BuildUrl(query, page))
|
||||
err = tab.Navigate(utils.BuildUrl(query, page))
|
||||
err = tab.WaitForNetworkIdle(10 * time.Second)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
|
|
|
|||
|
|
@ -39,10 +39,11 @@ async function cleanup() {
|
|||
}
|
||||
|
||||
async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
|
||||
chrome = launchChromeWithDebugging();
|
||||
let port = 9223;
|
||||
chrome = launchChromeWithDebugging(port);
|
||||
await waitForChromeReady();
|
||||
|
||||
const response = await fetch("http://localhost:9222/json/version", {
|
||||
const response = await fetch(`http://localhost:${port}/json/version`, {
|
||||
headers: {
|
||||
Origin: "",
|
||||
},
|
||||
|
|
|
|||
|
|
@ -70,11 +70,11 @@ function getChromePath(): string {
|
|||
throw new Error("Could not find Chrome/Chromium installation");
|
||||
}
|
||||
|
||||
export function launchChromeWithDebugging() {
|
||||
export function launchChromeWithDebugging(port: number) {
|
||||
try {
|
||||
const chromePath = getChromePath();
|
||||
const args = [
|
||||
"--remote-debugging-port=9222",
|
||||
`--remote-debugging-port=${port}`,
|
||||
`--user-data-dir=${userDataDir}`,
|
||||
"--remote-allow-origins=*",
|
||||
"--incognito",
|
||||
|
|
|
|||
|
|
@ -6,9 +6,9 @@ toolchain go1.24.7
|
|||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/utils v0.0.0
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0
|
||||
github.com/jobs-scraper/internal/utils v0.0.0
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
)
|
||||
|
|
@ -23,10 +23,10 @@ require (
|
|||
golang.org/x/net v0.44.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
||||
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils
|
||||
replace github.com/jobs-scraper/internal/utils => ../../internal/utils
|
||||
|
|
|
|||
|
|
@ -10,9 +10,9 @@ import (
|
|||
|
||||
"time"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/scraper/pipeline"
|
||||
"github.com/joho/godotenv"
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import (
|
|||
|
||||
"time"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"context"
|
||||
"log"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
func GetJobs(context context.Context, scraperService *Scraper, searchQuery domain.SearchQuery) <-chan domain.Job {
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/utils"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/utils"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
|
|
|
|||
Loading…
Reference in a new issue