diff --git a/.env b/.env index ad2311a..1eb4359 100644 --- a/.env +++ b/.env @@ -8,5 +8,9 @@ DB_SSLMODE=disable SERVER_PORT=8080 SERVER_HOST=localhost + +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini CV_AI_MODEL=your_ai_model OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index ab7ec0d..f98e31a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,8 @@ "type": "go", "request": "launch", "mode": "debug", - "program": "${workspaceFolder}/scraper/main.go", + "program": "${workspaceFolder}/services/scraper-linkedin/main.go", + "cwd": "${workspaceFolder}/services/scraper-linkedin", "env": { "GO_ENV": "development" }, @@ -22,7 +23,8 @@ "type": "go", "request": "launch", "mode": "debug", - "program": "${workspaceFolder}/cmd/cron-analyzer/main.go", + "program": "${workspaceFolder}/apps/cron-analyzer/cmd/cli/main.go", + "cwd": "${workspaceFolder}/apps/cron-analyzer", "env": { "GO_ENV": "development" }, @@ -31,11 +33,12 @@ "console": "integratedTerminal" }, { - "name": "Launch CV Service", + "name": "Launch OpenAi CV analyze services", "type": "go", "request": "launch", "mode": "debug", - "program": "${workspaceFolder}/cv/main.go", + "program": "${workspaceFolder}/apps/cron-analyzer/cmd/openai-demo/main.go", + "cwd": "${workspaceFolder}/apps/cron-analyzer", "env": { "GO_ENV": "development" }, @@ -48,7 +51,8 @@ "type": "go", "request": "launch", "mode": "debug", - "program": "${workspaceFolder}/api/main.go", + "program": "${workspaceFolder}/services/api/cmd/server/main.go", + "cwd": "${workspaceFolder}/services/api", "env": { "GO_ENV": "development" }, @@ -56,6 +60,18 @@ "showLog": true, "console": "integratedTerminal" }, + { + "name": "Launch Glassdoor Scraper", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/services/scraper-glassdoor/index.ts", + "cwd": "${workspaceFolder}/services/scraper-glassdoor", + "runtimeArgs": ["-r", "ts-node/register"], + "env": { + "NODE_ENV": "development" + }, + "console": "integratedTerminal" + }, { "name": "Launch Current File", "type": "go", diff --git a/Makefile b/Makefile index cda852d..5b280e1 100644 --- a/Makefile +++ b/Makefile @@ -74,6 +74,16 @@ build-cv-analyzer: ## Build the CV analyzer app @echo "Building CV analyzer app..." @cd $(CV_ANALYZER_DIR) && go build -o bin/cv-analyzer cmd/cli/main.go +.PHONY: run-openai-demo +run-openai-demo: ## Run the OpenAI service demo + @echo "Running OpenAI service demo..." + @cd $(CRON_ANALYZER_DIR) && go run cmd/openai-demo/main.go + +.PHONY: build-openai-demo +build-openai-demo: ## Build the OpenAI service demo + @echo "Building OpenAI service demo..." + @cd $(CRON_ANALYZER_DIR) && go build -o bin/openai-demo cmd/openai-demo/main.go + # Legacy aliases for backward compatibility .PHONY: run run: run-api ## Alias for run-api diff --git a/README.md b/README.md deleted file mode 100644 index bf97c40..0000000 --- a/README.md +++ /dev/null @@ -1,190 +0,0 @@ -# Jobs Scraper Monorepo - -A modern monorepo architecture for job scraping and analysis services. - -## ๐Ÿ—๏ธ Architecture - -``` -jobs-scraper/ -โ”œโ”€โ”€ go.mod # Root go.mod with module name -โ”œโ”€โ”€ Makefile # Common build/test tasks -โ”œโ”€โ”€ .github/workflows/ # CI/CD pipelines -โ”œโ”€โ”€ tools/ # Build tools and utilities -โ”œโ”€โ”€ internal/ # Private shared packages -โ”‚ โ”œโ”€โ”€ pkg/ -โ”‚ โ”‚ โ”œโ”€โ”€ domain/ # Core domain models -โ”‚ โ”‚ โ”œโ”€โ”€ infrastructure/ # Database, RabbitMQ, HTTP -โ”‚ โ”‚ โ””โ”€โ”€ utils/ # Common utilities -โ”‚ โ””โ”€โ”€ migrations/ # Database migrations -โ”œโ”€โ”€ libs/ # Shared libraries -โ”‚ โ”œโ”€โ”€ ports/ # Command interfaces -โ”‚ โ”œโ”€โ”€ repo/ # Data repositories -โ”‚ โ””โ”€โ”€ server/ # Server utilities -โ”œโ”€โ”€ services/ # Deployable services -โ”‚ โ”œโ”€โ”€ api/ # HTTP API service -โ”‚ โ”œโ”€โ”€ scraper-linkedin/ # LinkedIn scraper service -โ”‚ โ””โ”€โ”€ scraper-glassdoor/ # Glassdoor scraper service -โ”œโ”€โ”€ apps/ # Client applications -โ”‚ โ”œโ”€โ”€ cron-analyzer/ # Job analysis cron -โ”‚ โ””โ”€โ”€ cv-analyzer/ # CV analysis utility -โ””โ”€โ”€ scripts/ # Deployment and utility scripts -``` - -## ๐Ÿš€ Services - -### API Service (`services/api/`) -- **Purpose**: HTTP API server with CQRS pattern -- **Tech Stack**: Go, Gorilla Mux, PostgreSQL, RabbitMQ -- **Features**: Job management, Swagger documentation, CORS support - -### LinkedIn Scraper (`services/scraper-linkedin/`) -- **Purpose**: Scrapes job postings from LinkedIn -- **Tech Stack**: Go, Playwright -- **Features**: Automated job scraping, RabbitMQ integration - -### Glassdoor Scraper (`services/scraper-glassdoor/`) -- **Purpose**: Scrapes job postings from Glassdoor -- **Tech Stack**: TypeScript, Node.js -- **Features**: Web scraping, data processing - -## ๐Ÿ“ฑ Applications - -### Cron Analyzer (`apps/cron-analyzer/`) -- **Purpose**: Analyzes jobs with AI on a schedule -- **Tech Stack**: Go, OpenRouter API -- **Features**: Job analysis, status updates - -### CV Analyzer (`apps/cv-analyzer/`) -- **Purpose**: Analyzes CVs against job requirements -- **Tech Stack**: Go, OpenRouter API -- **Features**: CV matching, skill analysis - -## ๐Ÿ› ๏ธ Development - -### Prerequisites -- Go 1.24+ -- Node.js 18+ -- Docker & Docker Compose -- PostgreSQL 15+ -- RabbitMQ 3+ - -### Quick Start - -1. **Clone the repository** - ```bash - git clone - cd jobs-scraper - ``` - -2. **Start infrastructure** - ```bash - make rabbitmq-server - # In another terminal: - docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=password postgres:15 - ``` - -3. **Run services** - ```bash - # API Service - make run-api - - # LinkedIn Scraper - make run-scraper-linkedin - - # Glassdoor Scraper - make run-scraper-glassdoor - - # Cron Analyzer - make run-cron-analyzer - ``` - -### Available Commands - -#### Services -- `make run-api` - Run API service -- `make build-api` - Build API service -- `make run-scraper-linkedin` - Run LinkedIn scraper -- `make build-scraper-linkedin` - Build LinkedIn scraper -- `make run-scraper-glassdoor` - Run Glassdoor scraper -- `make build-scraper-glassdoor` - Build Glassdoor scraper - -#### Applications -- `make run-cron-analyzer` - Run cron analyzer -- `make build-cron-analyzer` - Build cron analyzer -- `make run-cv-analyzer` - Run CV analyzer -- `make build-cv-analyzer` - Build CV analyzer - -#### Infrastructure -- `make rabbitmq-server` - Start RabbitMQ server -- `make swagger` - Generate API documentation -- `make clean` - Clean build artifacts - -#### Legacy Aliases (Backward Compatibility) -- `make run` - Alias for `run-api` -- `make scraper` - Alias for `run-scraper-linkedin` -- `make cron` - Alias for `run-cron-analyzer` - -## ๐Ÿณ Docker - -Each service has its own Dockerfile and can be built independently: - -```bash -# Build API service -cd services/api -docker build -t jobs-scraper-api . - -# Build LinkedIn scraper -cd services/scraper-linkedin -docker build -t jobs-scraper-linkedin . -``` - -## ๐Ÿ”ง Configuration - -Environment variables are managed through `.env` files: - -- `.env` - Production configuration -- `.local.env` - Local development configuration - -## ๐Ÿงช Testing - -Run tests for individual services: - -```bash -# API Service -cd services/api && go test ./... - -# LinkedIn Scraper -cd services/scraper-linkedin && go test ./... - -# Cron Analyzer -cd apps/cron-analyzer && go test ./... -``` - -## ๐Ÿ“Š Monitoring - -- **API Documentation**: http://localhost:8080/swagger/ -- **RabbitMQ Management**: http://localhost:15672 (guest/guest) -- **Health Checks**: Built into each service - -## ๐Ÿš€ Deployment - -The monorepo includes GitHub Actions workflows for: -- Automated testing -- Docker image building -- Multi-service deployment - -## ๐Ÿค Contributing - -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Add tests -5. Submit a pull request - -## ๐Ÿ“„ License - -[Add your license information here] - -## ๐Ÿ†˜ Support - -For support and questions, please [create an issue](issues) or contact the development team. diff --git a/apps/cron-analyzer/.env b/apps/cron-analyzer/.env new file mode 100644 index 0000000..4618e4b --- /dev/null +++ b/apps/cron-analyzer/.env @@ -0,0 +1,17 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 + +# OpenAI Configuration +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini \ No newline at end of file diff --git a/apps/cron-analyzer/bin/cron-analyzer b/apps/cron-analyzer/bin/cron-analyzer deleted file mode 100755 index a6d70bc..0000000 Binary files a/apps/cron-analyzer/bin/cron-analyzer and /dev/null differ diff --git a/apps/cron-analyzer/cmd/cli/.env b/apps/cron-analyzer/cmd/cli/.env new file mode 100644 index 0000000..a040ca1 --- /dev/null +++ b/apps/cron-analyzer/cmd/cli/.env @@ -0,0 +1,17 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=jobs_scraper +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 + +# OpenAI Configuration +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini \ No newline at end of file diff --git a/apps/cron-analyzer/cmd/cli/main.go b/apps/cron-analyzer/cmd/cli/main.go index 96840db..41124c8 100644 --- a/apps/cron-analyzer/cmd/cli/main.go +++ b/apps/cron-analyzer/cmd/cli/main.go @@ -9,10 +9,9 @@ import ( ) func main() { - // Try to load .local.env first, then fallback to .env - if err := godotenv.Load(".local.env"); err != nil { - log.Println("No .local.env file found, trying .env") - if err := godotenv.Load(".env"); err != nil { + if err := godotenv.Load("cmd/cli/.local.env"); err != nil { + log.Printf("No .local.env file found: %v, trying .env", err) + if err := godotenv.Load("cmd/cli/.env"); err != nil { log.Println("No .env file found, using system environment variables") } } @@ -21,7 +20,7 @@ func main() { db, err := infrastructure.NewConnection(dbConfig) if err != nil { - log.Fatal("Error connecting to db") + log.Fatalf("Error connecting to db: %v", err) } err = db.Ping() @@ -31,6 +30,13 @@ func main() { log.Println("Successfully connected to db") + // Run migrations + log.Println("Running database migrations...") + if err := infrastructure.RunMigrations(db); err != nil { + log.Fatalf("Error running migrations: %v", err) + } + log.Println("Migrations completed successfully") + analyzer := internal.NewJobAnalyzer(db) if err := analyzer.AnalyzeJobs(); err != nil { diff --git a/cv.txt b/apps/cron-analyzer/cv.txt similarity index 100% rename from cv.txt rename to apps/cron-analyzer/cv.txt diff --git a/apps/cron-analyzer/internal/analyze-jobs.go b/apps/cron-analyzer/internal/analyze-jobs.go index 897dcd9..d04d3fe 100644 --- a/apps/cron-analyzer/internal/analyze-jobs.go +++ b/apps/cron-analyzer/internal/analyze-jobs.go @@ -6,8 +6,11 @@ import ( "log" "os" + // "path/filepath" + "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/pkg/openai" "github.com/jobs-scraper/internal/pkg/openrouter" "github.com/jobs-scraper/libs/repo" ) @@ -19,16 +22,28 @@ type JobAnalyzer struct { jobAnalysisResultRepo *repo.JobAnalysisResultRepository openRouterService openrouter.OpenRouterService rmq *rabbitmq.RabbitMQClient + openAiService *openai.OpenAIService } func NewJobAnalyzer(db *sql.DB) *JobAnalyzer { apiKey := os.Getenv("OPENROUTER_API_KEY") model := os.Getenv("CV_AI_MODEL") + openAiapiKey := os.Getenv("OPENAI_API_KEY") + if openAiapiKey == "" { + log.Fatal("OPENAI_API_KEY environment variable is required") + } + + // Create OpenAI service + openAiModel := os.Getenv("OPENAI_MODEL") + if openAiModel == "" { + openAiModel = "gpt-4o-mini" // Default to cost-effective model + } jobRepo := repo.NewJobRepository(db) jobDescriptionRepo := repo.NewJobDescriptionRepository(db) jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) openRouterService := openrouter.NewOpenRouterService(model, apiKey) + openAiService := openai.NewOpenAIService(openAiapiKey, openAiModel) rmq, err := rabbitmq.NewRabbitMQClient() if err != nil { @@ -42,6 +57,7 @@ func NewJobAnalyzer(db *sql.DB) *JobAnalyzer { jobAnalysisResultRepo: jobAnalysisResultRepo, openRouterService: openRouterService, rmq: rmq, + openAiService: openAiService, } } @@ -54,8 +70,8 @@ func (ja *JobAnalyzer) AnalyzeJobs() error { log.Printf("Found %d jobs to analyze", len(jobs)) - // Read CV file - cv, err := os.ReadFile("../../cv.txt") + cv, err := os.ReadFile("cv.txt") + if err != nil { return err } @@ -89,7 +105,7 @@ func (ja *JobAnalyzer) analyzeJob(job domain.Job, cv string) error { } // Analyze job with CV - result, err := ja.openRouterService.AnalyzeJobDescription(cv, domain.JobDescription{ + result, err := ja.openAiService.AnalyzeJobDescription(cv, domain.JobDescription{ JobID: job.ID, Description: jobDescription, Criteria: jobCriteria, diff --git a/apps/cv-analyzer/bin/cv-analyzer b/apps/cv-analyzer/bin/cv-analyzer deleted file mode 100755 index 1ebf925..0000000 Binary files a/apps/cv-analyzer/bin/cv-analyzer and /dev/null differ diff --git a/apps/cv-analyzer/cmd/cli/main.go b/apps/cv-analyzer/cmd/cli/main.go deleted file mode 100644 index d01363d..0000000 --- a/apps/cv-analyzer/cmd/cli/main.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "log" - "os" - - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/internal/pkg/infrastructure" - "github.com/jobs-scraper/internal/pkg/openrouter" - "github.com/jobs-scraper/libs/repo" - "github.com/joho/godotenv" -) - -func main() { - // Try to load .local.env first, then fallback to .env - if err := godotenv.Load("../.local.env"); err != nil { - log.Println("No .local.env file found, trying .env") - if err := godotenv.Load("../.env"); err != nil { - log.Println("No .env file found, using system environment variables") - } - } - - dbConfig := infrastructure.LoadConfigFromEnv() - db, err := infrastructure.NewConnection(dbConfig) - if err != nil { - log.Fatal("Error connecting to db") - } - - apiKey := os.Getenv("OPENROUTER_API_KEY") - model := os.Getenv("CV_AI_MODEL") - - err = db.Ping() - if err != nil { - log.Fatal("Error pinging db") - } - - log.Println("Successfully connected to db") - - jobRepo := repo.NewJobRepository(db) - jobDescriptionRepo := repo.NewJobDescriptionRepository(db) - openRouterService := openrouter.NewOpenRouterService(model, apiKey) - - cv, err := os.ReadFile("../../cv.txt") - - if err != nil { - log.Fatalf("Failed to get cv: %v", err) - } - - job, err := jobRepo.GetJobByID(4312925061) - - if err != nil { - log.Fatalf("Failed to get job: %v", err) - } - - jobDescription, jobCriteria, err := jobDescriptionRepo.GetJobDescriptionByJobID(job.ID) - - if err != nil { - log.Fatalf("Failed to get job description: %v", err) - } - - result, err := openRouterService.AnalyzeJobDescription(string(cv), domain.JobDescription{ - JobID: job.ID, - Description: jobDescription, - Criteria: jobCriteria, - }) - - if err != nil { - log.Fatalf("Failed to get job analysis result: %v", err) - } - - log.Println(result) -} diff --git a/apps/cv-analyzer/go.mod b/apps/cv-analyzer/go.mod deleted file mode 100644 index 47cf47f..0000000 --- a/apps/cv-analyzer/go.mod +++ /dev/null @@ -1,15 +0,0 @@ -module github.com/jobs-scraper/apps/cv-analyzer - -go 1.24.0 - -require ( - github.com/jobs-scraper/internal/pkg/domain v0.0.0 - github.com/jobs-scraper/libs/repo v0.0.0-00010101000000-000000000000 - github.com/joho/godotenv v1.5.1 -) - -require github.com/lib/pq v1.10.9 // indirect - -replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain - -replace github.com/jobs-scraper/libs/repo => ../../libs/repo diff --git a/apps/cv-analyzer/go.sum b/apps/cv-analyzer/go.sum deleted file mode 100644 index 3a912e2..0000000 --- a/apps/cv-analyzer/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= diff --git a/go.mod b/go.mod index a84c11b..bde6a6a 100644 --- a/go.mod +++ b/go.mod @@ -3,22 +3,3 @@ module github.com/jobs-scraper go 1.24.0 toolchain go1.24.7 - -require github.com/swaggo/swag v1.16.3 - -require ( - github.com/KyleBanks/depth v1.2.1 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/spec v0.20.6 // indirect - github.com/go-openapi/swag v0.19.15 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - golang.org/x/tools v0.36.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect -) - -require ( - github.com/stretchr/testify v1.10.0 // indirect - golang.org/x/sync v0.17.0 // indirect -) diff --git a/go.sum b/go.sum index 72e6649..e69de29 100644 --- a/go.sum +++ b/go.sum @@ -1,52 +0,0 @@ -github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ= -github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= -github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work b/go.work index 295a216..7f3fea3 100644 --- a/go.work +++ b/go.work @@ -3,9 +3,9 @@ go 1.24.0 use ( . ./apps/cron-analyzer - ./apps/cv-analyzer ./internal/pkg/domain ./internal/pkg/infrastructure + ./internal/pkg/openai ./internal/pkg/openrouter ./internal/pkg/utils ./libs/ports @@ -13,5 +13,4 @@ use ( ./libs/server ./services/api ./services/scraper-linkedin - ./tools ) diff --git a/internal/pkg/openai/README.md b/internal/pkg/openai/README.md new file mode 100644 index 0000000..6a17f65 --- /dev/null +++ b/internal/pkg/openai/README.md @@ -0,0 +1,157 @@ +# OpenAI Service + +This package provides an OpenAI-powered service for job analysis and CV generation using the official OpenAI Go SDK. + +## Features + +- **Job Analysis**: Analyze job descriptions against CVs to determine fit and provide recommendations +- **CV Generation**: Create tailored CVs based on job descriptions +- **Cover Letter Generation**: Generate personalized cover letters + +## Setup + +1. Install the OpenAI Go SDK: +```bash +go get github.com/sashabaranov/go-openai +``` + +2. Set your OpenAI API key as an environment variable: +```bash +export OPENAI_API_KEY="your-api-key-here" +``` + +3. (Optional) Set a custom base URL: +```bash +export OPENAI_BASE_URL="https://your-custom-endpoint.com/v1" +``` + +## Usage + +### Basic Job Analysis + +```go +package main + +import ( + "fmt" + "os" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/openai" +) + +func main() { + // Create service + apiKey := os.Getenv("OPENAI_API_KEY") + service := openai.NewOpenAIService(apiKey, "gpt-4o-mini") + + // Prepare job description + jobDesc := domain.JobDescription{ + JobID: 1, + Description: "Looking for a Go developer...", + Criteria: map[string]string{ + "experience": "5+ years", + "languages": "Go, JavaScript", + }, + } + + // Analyze job + result, err := service.AnalyzeJobDescription(cv, jobDesc) + if err != nil { + panic(err) + } + + fmt.Printf("Should apply: %t\n", result.ShouldApply()) + fmt.Printf("Confidence: %d%%\n", result.ConfidenceScore) +} +``` + +### Using Custom Base URL + +You can configure a custom base URL in two ways: + +**Method 1: Environment Variable** +```bash +export OPENAI_BASE_URL="https://api.openai-proxy.com/v1" +# Service will automatically use this URL +service := openai.NewOpenAIService(apiKey, "gpt-4o-mini") +``` + +**Method 2: Direct Parameter** +```go +// Use a custom base URL directly +service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4o-mini", "https://api.openai-proxy.com/v1") + +// Or use Azure OpenAI +service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4", "https://your-resource.openai.azure.com/") +``` + +### Generate Tailored CV + +```go +tailoredCV, err := service.GenerateCV(originalCV, jobDesc) +if err != nil { + panic(err) +} +fmt.Println(tailoredCV) +``` + +### Generate Cover Letter + +```go +coverLetter, err := service.GenerateCoverLetter(cv, jobDesc, "Tech Company Inc") +if err != nil { + panic(err) +} +fmt.Println(coverLetter) +``` + +## Models + +The service supports all OpenAI models. Common choices: + +- `gpt-4o-mini` - Cost-effective, good performance (default) +- `gpt-4o` - Higher quality, more expensive +- `gpt-3.5-turbo` - Fastest, most cost-effective + +## Environment Variables + +- `OPENAI_API_KEY` - Your OpenAI API key (required) +- `OPENAI_BASE_URL` - Custom base URL for OpenAI API (optional) +- `OPENAI_MODEL` - Default model to use (optional) + +## Response Structure + +### JobAnalysisResult + +```go +type JobAnalysisResult struct { + Recommendation string // "apply" or "do_not_apply" + ConfidenceScore int // 0-100 + MatchingSkills []string + MissingSkills []string + ExperienceMatch string // "excellent", "good", "fair", "poor" + Summary string + ImprovementSuggestions []string +} +``` + +## Error Handling + +The service handles common errors: +- API authentication issues +- Rate limiting +- Invalid JSON responses +- Network timeouts + +Always check for errors when calling service methods. + +## Testing + +Run the example test: +```bash +cd internal/pkg/openai +go test -v +``` + +Make sure to set `OPENAI_API_KEY` environment variable before running tests that make actual API calls. diff --git a/internal/pkg/openai/go.mod b/internal/pkg/openai/go.mod new file mode 100644 index 0000000..5790c22 --- /dev/null +++ b/internal/pkg/openai/go.mod @@ -0,0 +1,10 @@ +module github.com/jobs-scraper/internal/pkg/openai + +go 1.24.0 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000 + github.com/sashabaranov/go-openai v1.41.2 +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../domain diff --git a/internal/pkg/openai/go.sum b/internal/pkg/openai/go.sum new file mode 100644 index 0000000..17713c1 --- /dev/null +++ b/internal/pkg/openai/go.sum @@ -0,0 +1,2 @@ +github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM= +github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= diff --git a/internal/pkg/openai/interface.go b/internal/pkg/openai/interface.go new file mode 100644 index 0000000..7a89640 --- /dev/null +++ b/internal/pkg/openai/interface.go @@ -0,0 +1,27 @@ +package openai + +import "github.com/jobs-scraper/internal/pkg/domain" + +// JobAnalysisService defines the interface for job analysis services +type JobAnalysisService interface { + // AnalyzeJobDescription analyzes a job description against a CV + AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) +} + +// CVService defines the interface for CV-related services +type CVService interface { + // GenerateCV creates a tailored CV based on a job description + GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error) + + // GenerateCoverLetter creates a cover letter based on CV and job description + GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error) +} + +// AIService combines all AI-related services +type AIService interface { + JobAnalysisService + CVService +} + +// Ensure OpenAIService implements AIService +var _ AIService = (*OpenAIService)(nil) diff --git a/internal/pkg/openai/openai.go b/internal/pkg/openai/openai.go new file mode 100644 index 0000000..db0e9f7 --- /dev/null +++ b/internal/pkg/openai/openai.go @@ -0,0 +1,275 @@ +package openai + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "strings" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/sashabaranov/go-openai" +) + +// JobAnalysisResult represents the structured response from job analysis +type JobAnalysisResult struct { + Recommendation string `json:"recommendation"` + ConfidenceScore int `json:"confidence_score"` + MatchingSkills []string `json:"matching_skills"` + MissingSkills []string `json:"missing_skills"` + ExperienceMatch string `json:"experience_match"` + Summary string `json:"summary"` + ImprovementSuggestions []string `json:"improvement_suggestions"` +} + +// ShouldApply returns true if the recommendation is to apply for the job +func (r *JobAnalysisResult) ShouldApply() bool { + return r.Recommendation == "apply" +} + +// IsHighConfidence returns true if the confidence score is 70 or above +func (r *JobAnalysisResult) IsHighConfidence() bool { + return r.ConfidenceScore >= 70 +} + +// OpenAIService provides job analysis using OpenAI's API +type OpenAIService struct { + client *openai.Client + model string +} + +// NewOpenAIService creates a new OpenAI service instance +func NewOpenAIService(apiKey, model string) *OpenAIService { + return NewOpenAIServiceWithBaseURL(apiKey, model, "") +} + +// NewOpenAIServiceWithBaseURL creates a new OpenAI service instance with a custom base URL +func NewOpenAIServiceWithBaseURL(apiKey, model, baseURL string) *OpenAIService { + if model == "" { + model = openai.GPT4oMini // Default to GPT-4o-mini for cost efficiency + } + + // Create client configuration + config := openai.DefaultConfig(apiKey) + + // Use provided baseURL, or check environment variable, or use default + if baseURL != "" { + config.BaseURL = baseURL + } else if envBaseURL := os.Getenv("OPENAI_BASE_URL"); envBaseURL != "" { + config.BaseURL = envBaseURL + } + + client := openai.NewClientWithConfig(config) + + return &OpenAIService{ + client: client, + model: model, + } +} + +// AnalyzeJobDescription analyzes a job description against a CV using OpenAI +func (s *OpenAIService) AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) { + // Build the user message with all the provided data + userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below. + 1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv. + for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills. + + 2) The job shouldn't require any language skills, preferably only english. + + 3) The job should be remote, or provide relocation to the country. + CV: + %s + + Job Description: + %s + + Job Criteria (key-value): + %v + + CRITICAL OUTPUT REQUIREMENTS: + - Return ONLY raw JSON - NO markdown formatting whatsoever + - NO backticks, NO code blocks, NO json prefix + - NO additional text before or after the JSON + - Start your response directly with { and end with } + - Use this exact schema and key names: + { + "recommendation": "apply" | "do_not_apply", + "confidence_score": number, // integer 0-100 + "matching_skills": [string], + "missing_skills": [string], + "experience_match": "excellent" | "good" | "fair" | "poor", + "summary": string, + "improvement_suggestions": [string] + }`, cv, jobDesc.Description, jobDesc.Criteria) + + // Create the chat completion request + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: userMessage, + }, + }, + Temperature: 0.3, // Lower temperature for more consistent responses + MaxTokens: 1000, + } + + // Execute the request + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return nil, fmt.Errorf("failed to create chat completion: %v", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response choices received from OpenAI API") + } + + // Extract JSON from the response + jsonContent := resp.Choices[0].Message.Content + + // Log the raw response for debugging + log.Printf("Raw OpenAI response: %s", jsonContent) + + // Clean up markdown code blocks if present + jsonContent = cleanMarkdownCodeBlocks(jsonContent) + + // Parse the JSON response into our struct + var result JobAnalysisResult + if err := json.Unmarshal([]byte(jsonContent), &result); err != nil { + return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(200, len(jsonContent))]) + } + + return &result, nil +} + +// GenerateCV creates a tailored CV based on a job description +func (s *OpenAIService) GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error) { + userMessage := fmt.Sprintf(`Based on the original CV and job description below, create a tailored CV that highlights relevant experience and skills for this specific position. + + Original CV: + %s + + Job Description: + %s + + Job Criteria: + %v + + Instructions: + 1. Keep all factual information accurate - do not fabricate experience or skills + 2. Reorganize and emphasize relevant experience and skills + 3. Use keywords from the job description where appropriate + 4. Maintain professional formatting + 5. Focus on achievements and quantifiable results + 6. Return the tailored CV in plain text format`, originalCV, jobDesc.Description, jobDesc.Criteria) + + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are an expert CV writer who specializes in tailoring resumes for specific job applications. You help candidates present their existing experience and skills in the most relevant way for each position, without fabricating information.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: userMessage, + }, + }, + Temperature: 0.7, // Higher temperature for more creative writing + MaxTokens: 2000, + } + + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return "", fmt.Errorf("failed to create chat completion: %v", err) + } + + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no response choices received from OpenAI API") + } + + return resp.Choices[0].Message.Content, nil +} + +// GenerateCoverLetter creates a cover letter based on CV and job description +func (s *OpenAIService) GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error) { + userMessage := fmt.Sprintf(`Create a professional cover letter based on the CV and job description below. + + CV: + %s + + Job Description: + %s + + Job Criteria: + %v + + Company Name: %s + + Instructions: + 1. Write a compelling opening paragraph + 2. Highlight relevant experience and skills from the CV + 3. Show enthusiasm for the specific role and company + 4. Keep it concise (3-4 paragraphs) + 5. Use a professional but engaging tone + 6. Include a strong closing paragraph`, cv, jobDesc.Description, jobDesc.Criteria, companyName) + + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are an expert career counselor who writes compelling cover letters that help candidates stand out while maintaining professionalism.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: userMessage, + }, + }, + Temperature: 0.8, // Higher temperature for more creative and engaging writing + MaxTokens: 1500, + } + + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return "", fmt.Errorf("failed to create chat completion: %v", err) + } + + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no response choices received from OpenAI API") + } + + return resp.Choices[0].Message.Content, nil +} + +// min returns the smaller of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// cleanMarkdownCodeBlocks removes markdown code block formatting from JSON response +func cleanMarkdownCodeBlocks(content string) string { + // Remove ```json prefix and ``` suffix if present + content = strings.TrimSpace(content) + + // Remove opening code block markers + if strings.HasPrefix(content, "```json") { + content = strings.TrimPrefix(content, "```json") + } else if strings.HasPrefix(content, "```") { + content = strings.TrimPrefix(content, "```") + } + + // Remove closing code block markers + content = strings.TrimSuffix(content, "```") + + return strings.TrimSpace(content) +} diff --git a/services/api/bin/api b/services/api/bin/api deleted file mode 100755 index 3dd3e37..0000000 Binary files a/services/api/bin/api and /dev/null differ diff --git a/services/api/cmd/server/main.go b/services/api/cmd/server/main.go index 0813f8b..b44955a 100644 --- a/services/api/cmd/server/main.go +++ b/services/api/cmd/server/main.go @@ -10,11 +10,11 @@ import ( "time" "github.com/gorilla/mux" - "github.com/jobs-scraper/services/api/internal/app" - httpHandler "github.com/jobs-scraper/services/api/pkg/http" "github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/services/api/internal/app" + httpHandler "github.com/jobs-scraper/services/api/pkg/http" _ "github.com/jobs-scraper/services/api/pkg/swagger" "github.com/joho/godotenv" httpSwagger "github.com/swaggo/http-swagger" @@ -26,9 +26,9 @@ import ( // @BasePath / func main() { // Try to load .local.env first, then fallback to .env - if err := godotenv.Load("../.local.env"); err != nil { + if err := godotenv.Load(".local.env"); err != nil { log.Println("No .local.env file found, trying .env") - if err := godotenv.Load("../.env"); err != nil { + if err := godotenv.Load(".env"); err != nil { log.Println("No .env file found, using system environment variables") } } diff --git a/services/scraper-glassdoor/package-lock.json b/services/scraper-glassdoor/package-lock.json index 885ffdc..84bb99b 100644 --- a/services/scraper-glassdoor/package-lock.json +++ b/services/scraper-glassdoor/package-lock.json @@ -107,19 +107,19 @@ "dev": true }, "node_modules/@types/amqplib": { - "version": "0.10.7", - "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.7.tgz", - "integrity": "sha512-IVj3avf9AQd2nXCx0PGk/OYq7VmHiyNxWFSb5HhU9ATh+i+gHWvVcljFTcTWQ/dyHJCTrzCixde+r/asL2ErDA==", + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz", + "integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==", "dependencies": { "@types/node": "*" } }, "node_modules/@types/node": { - "version": "24.8.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-24.8.1.tgz", - "integrity": "sha512-alv65KGRadQVfVcG69MuB4IzdYVpRwMG/mq8KWOaoOdyY617P5ivaDiMCGOFDWD2sAn5Q0mR3mRtUOgm99hL9Q==", + "version": "24.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz", + "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==", "dependencies": { - "undici-types": "~7.14.0" + "undici-types": "~7.16.0" } }, "node_modules/@types/yauzl": { @@ -228,9 +228,9 @@ } }, "node_modules/bare-events": { - "version": "2.8.0", - "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.0.tgz", - "integrity": "sha512-AOhh6Bg5QmFIXdViHbMc2tLDsBIRxdkIaIddPslJF9Z5De3APBScuqGP2uThXnIpqFrgoxMNC6km7uXNIMLHXA==", + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.1.tgz", + "integrity": "sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==", "peerDependencies": { "bare-abort-controller": "*" }, @@ -241,9 +241,9 @@ } }, "node_modules/bare-fs": { - "version": "4.4.11", - "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.4.11.tgz", - "integrity": "sha512-Bejmm9zRMvMTRoHS+2adgmXw1ANZnCNx+B5dgZpGwlP1E3x6Yuxea8RToddHUbWtVV0iUMWqsgZr8+jcgUI2SA==", + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.0.tgz", + "integrity": "sha512-GljgCjeupKZJNetTqxKaQArLK10vpmK28or0+RwWjEl5Rk+/xG3wkpmkv+WrcBm3q1BwHKlnhXzR8O37kcvkXQ==", "optional": true, "dependencies": { "bare-events": "^2.5.4", @@ -304,9 +304,9 @@ } }, "node_modules/bare-url": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.0.tgz", - "integrity": "sha512-c+RCqMSZbkz97Mw1LWR0gcOqwK82oyYKfLoHJ8k13ybi1+I80ffdDzUy0TdAburdrR/kI0/VuN8YgEnJqX+Nyw==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz", + "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", "optional": true, "dependencies": { "bare-path": "^3.0.0" @@ -334,9 +334,9 @@ "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" }, "node_modules/chromium-bidi": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-9.1.0.tgz", - "integrity": "sha512-rlUzQ4WzIAWdIbY/viPShhZU2n21CxDUgazXVbw4Hu1MwaeUSEksSeM6DqPgpRjCLXRk702AVRxJxoOz0dw4OA==", + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-10.5.1.tgz", + "integrity": "sha512-rlj6OyhKhVTnk4aENcUme3Jl9h+cq4oXu4AzBcvr8RMmT6BR4a3zSNT9dbIfXr9/BS6ibzRyDhowuw4n2GgzsQ==", "dependencies": { "mitt": "^3.0.1", "zod": "^3.24.1" @@ -418,9 +418,9 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.1508733", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1508733.tgz", - "integrity": "sha512-QJ1R5gtck6nDcdM+nlsaJXcelPEI7ZxSMw1ujHpO1c4+9l+Nue5qlebi9xO1Z2MGr92bFOQTW7/rrheh5hHxDg==" + "version": "0.0.1521046", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1521046.tgz", + "integrity": "sha512-vhE6eymDQSKWUXwwA37NtTTVEzjtGVfDr3pRbsWEQ5onH/Snp2c+2xZHWJJawG/0hCCJLRGt4xVtEVUVILol4w==" }, "node_modules/diff": { "version": "4.0.2", @@ -742,16 +742,16 @@ } }, "node_modules/puppeteer-core": { - "version": "24.25.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.25.0.tgz", - "integrity": "sha512-8Xs6q3Ut+C8y7sAaqjIhzv1QykGWG4gc2mEZ2mYE7siZFuRp4xQVehOf8uQKSQAkeL7jXUs3mknEeiqnRqUKvQ==", + "version": "24.27.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.27.0.tgz", + "integrity": "sha512-yubwj2XXmTM3wRIpbhO5nCjbByPgpFHlgrsD4IK+gMPqO7/a5FfnoSXDKjmqi8A2M1Ewusz0rTI/r+IN0GU0MA==", "dependencies": { "@puppeteer/browsers": "2.10.12", - "chromium-bidi": "9.1.0", + "chromium-bidi": "10.5.1", "debug": "^4.4.3", - "devtools-protocol": "0.0.1508733", + "devtools-protocol": "0.0.1521046", "typed-query-selector": "^2.12.0", - "webdriver-bidi-protocol": "0.3.7", + "webdriver-bidi-protocol": "0.3.8", "ws": "^8.18.3" }, "engines": { @@ -963,9 +963,9 @@ } }, "node_modules/undici-types": { - "version": "7.14.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.14.0.tgz", - "integrity": "sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==" + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==" }, "node_modules/url-parse": { "version": "1.5.10", @@ -983,9 +983,9 @@ "dev": true }, "node_modules/webdriver-bidi-protocol": { - "version": "0.3.7", - "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.7.tgz", - "integrity": "sha512-wIx5Gu/LLTeexxilpk8WxU2cpGAKlfbWRO5h+my6EMD1k5PYqM1qQO1MHUFf4f3KRnhBvpbZU7VkizAgeSEf7g==" + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.8.tgz", + "integrity": "sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ==" }, "node_modules/wrap-ansi": { "version": "7.0.0", diff --git a/services/scraper-linkedin/bin/scraper b/services/scraper-linkedin/bin/scraper deleted file mode 100755 index 537c910..0000000 Binary files a/services/scraper-linkedin/bin/scraper and /dev/null differ diff --git a/services/scraper-linkedin/main.go b/services/scraper-linkedin/main.go index 0f34658..902c10b 100644 --- a/services/scraper-linkedin/main.go +++ b/services/scraper-linkedin/main.go @@ -10,11 +10,11 @@ import ( "time" - "github.com/jobs-scraper/services/scraper/pipeline" "github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/services/scraper/pipeline" "github.com/joho/godotenv" ) diff --git a/services/scraper-linkedin/scraper b/services/scraper-linkedin/scraper deleted file mode 100755 index 139f854..0000000 Binary files a/services/scraper-linkedin/scraper and /dev/null differ diff --git a/tools/go.mod b/tools/go.mod deleted file mode 100644 index 73ef739..0000000 --- a/tools/go.mod +++ /dev/null @@ -1,7 +0,0 @@ -module github.com/jobs-scraper/tools - -go 1.24.0 - -require ( - github.com/swaggo/swag v1.16.3 -)