bugblitz77/jobs-monorepo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

infrastructure fixes

Browse source
This commit is contained in:
Elshimy Ziad Magdy Taha 2025-11-03 00:29:33 +05:00
parent 06f5f9cf02
commit 5060643a3f
29 changed files with 610 additions and 411 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.env
View file

@ -8,5 +8,9 @@ DB_SSLMODE=disable
SERVER_PORT=8080 SERVER_PORT=8080
SERVER_HOST=localhost SERVER_HOST=localhost
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini
CV_AI_MODEL=your_ai_model CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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 OPENROUTER_API_KEY=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

26
.vscode/launch.json vendored
View file

@ -9,7 +9,8 @@
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "debug", "mode": "debug",
"program": "${workspaceFolder}/scraper/main.go", "program": "${workspaceFolder}/services/scraper-linkedin/main.go",
"cwd": "${workspaceFolder}/services/scraper-linkedin",
"env": { "env": {
"GO_ENV": "development" "GO_ENV": "development"
}, },
@ -22,7 +23,8 @@
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "debug", "mode": "debug",
"program": "${workspaceFolder}/cmd/cron-analyzer/main.go", "program": "${workspaceFolder}/apps/cron-analyzer/cmd/cli/main.go",
"cwd": "${workspaceFolder}/apps/cron-analyzer",
"env": { "env": {
"GO_ENV": "development" "GO_ENV": "development"
}, },
@ -31,11 +33,12 @@
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
{ {
"name": "Launch CV Service", "name": "Launch OpenAi CV analyze services",
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "debug", "mode": "debug",
"program": "${workspaceFolder}/cv/main.go", "program": "${workspaceFolder}/apps/cron-analyzer/cmd/openai-demo/main.go",
"cwd": "${workspaceFolder}/apps/cron-analyzer",
"env": { "env": {
"GO_ENV": "development" "GO_ENV": "development"
}, },
@ -48,7 +51,8 @@
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "debug", "mode": "debug",
"program": "${workspaceFolder}/api/main.go", "program": "${workspaceFolder}/services/api/cmd/server/main.go",
"cwd": "${workspaceFolder}/services/api",
"env": { "env": {
"GO_ENV": "development" "GO_ENV": "development"
}, },
@ -56,6 +60,18 @@
"showLog": true, "showLog": true,
"console": "integratedTerminal" "console": "integratedTerminal"
}, },
{
"name": "Launch Glassdoor Scraper",
"type": "node",
"request": "launch",
"program": "${workspaceFolder}/services/scraper-glassdoor/index.ts",
"cwd": "${workspaceFolder}/services/scraper-glassdoor",
"runtimeArgs": ["-r", "ts-node/register"],
"env": {
"NODE_ENV": "development"
},
"console": "integratedTerminal"
},
{ {
"name": "Launch Current File", "name": "Launch Current File",
"type": "go", "type": "go",

10
Makefile
View file

@ -74,6 +74,16 @@ build-cv-analyzer: ## Build the CV analyzer app
@echo "Building CV analyzer app..." @echo "Building CV analyzer app..."
@cd $(CV_ANALYZER_DIR) && go build -o bin/cv-analyzer cmd/cli/main.go @cd $(CV_ANALYZER_DIR) && go build -o bin/cv-analyzer cmd/cli/main.go
.PHONY: run-openai-demo
run-openai-demo: ## Run the OpenAI service demo
@echo "Running OpenAI service demo..."
@cd $(CRON_ANALYZER_DIR) && go run cmd/openai-demo/main.go
.PHONY: build-openai-demo
build-openai-demo: ## Build the OpenAI service demo
@echo "Building OpenAI service demo..."
@cd $(CRON_ANALYZER_DIR) && go build -o bin/openai-demo cmd/openai-demo/main.go
# Legacy aliases for backward compatibility # Legacy aliases for backward compatibility
.PHONY: run .PHONY: run
run: run-api ## Alias for run-api run: run-api ## Alias for run-api

190
README.md
View file

@ -1,190 +0,0 @@
# Jobs Scraper Monorepo
A modern monorepo architecture for job scraping and analysis services.
## 🏗️ Architecture
```
jobs-scraper/
├── go.mod # Root go.mod with module name
├── Makefile # Common build/test tasks
├── .github/workflows/ # CI/CD pipelines
├── tools/ # Build tools and utilities
├── internal/ # Private shared packages
│ ├── pkg/
│ │ ├── domain/ # Core domain models
│ │ ├── infrastructure/ # Database, RabbitMQ, HTTP
│ │ └── utils/ # Common utilities
│ └── migrations/ # Database migrations
├── libs/ # Shared libraries
│ ├── ports/ # Command interfaces
│ ├── repo/ # Data repositories
│ └── server/ # Server utilities
├── services/ # Deployable services
│ ├── api/ # HTTP API service
│ ├── scraper-linkedin/ # LinkedIn scraper service
│ └── scraper-glassdoor/ # Glassdoor scraper service
├── apps/ # Client applications
│ ├── cron-analyzer/ # Job analysis cron
│ └── cv-analyzer/ # CV analysis utility
└── scripts/ # Deployment and utility scripts
```
## 🚀 Services
### API Service (`services/api/`)
- **Purpose**: HTTP API server with CQRS pattern
- **Tech Stack**: Go, Gorilla Mux, PostgreSQL, RabbitMQ
- **Features**: Job management, Swagger documentation, CORS support
### LinkedIn Scraper (`services/scraper-linkedin/`)
- **Purpose**: Scrapes job postings from LinkedIn
- **Tech Stack**: Go, Playwright
- **Features**: Automated job scraping, RabbitMQ integration
### Glassdoor Scraper (`services/scraper-glassdoor/`)
- **Purpose**: Scrapes job postings from Glassdoor
- **Tech Stack**: TypeScript, Node.js
- **Features**: Web scraping, data processing
## 📱 Applications
### Cron Analyzer (`apps/cron-analyzer/`)
- **Purpose**: Analyzes jobs with AI on a schedule
- **Tech Stack**: Go, OpenRouter API
- **Features**: Job analysis, status updates
### CV Analyzer (`apps/cv-analyzer/`)
- **Purpose**: Analyzes CVs against job requirements
- **Tech Stack**: Go, OpenRouter API
- **Features**: CV matching, skill analysis
## 🛠️ Development
### Prerequisites
- Go 1.24+
- Node.js 18+
- Docker & Docker Compose
- PostgreSQL 15+
- RabbitMQ 3+
### Quick Start
1. **Clone the repository**
```bash
git clone <repository-url>
cd jobs-scraper
```
2. **Start infrastructure**
```bash
make rabbitmq-server
# In another terminal:
docker run -d -p 5432:5432 -e POSTGRES_PASSWORD=password postgres:15
```
3. **Run services**
```bash
# API Service
make run-api
# LinkedIn Scraper
make run-scraper-linkedin
# Glassdoor Scraper
make run-scraper-glassdoor
# Cron Analyzer
make run-cron-analyzer
```
### Available Commands
#### Services
- `make run-api` - Run API service
- `make build-api` - Build API service
- `make run-scraper-linkedin` - Run LinkedIn scraper
- `make build-scraper-linkedin` - Build LinkedIn scraper
- `make run-scraper-glassdoor` - Run Glassdoor scraper
- `make build-scraper-glassdoor` - Build Glassdoor scraper
#### Applications
- `make run-cron-analyzer` - Run cron analyzer
- `make build-cron-analyzer` - Build cron analyzer
- `make run-cv-analyzer` - Run CV analyzer
- `make build-cv-analyzer` - Build CV analyzer
#### Infrastructure
- `make rabbitmq-server` - Start RabbitMQ server
- `make swagger` - Generate API documentation
- `make clean` - Clean build artifacts
#### Legacy Aliases (Backward Compatibility)
- `make run` - Alias for `run-api`
- `make scraper` - Alias for `run-scraper-linkedin`
- `make cron` - Alias for `run-cron-analyzer`
## 🐳 Docker
Each service has its own Dockerfile and can be built independently:
```bash
# Build API service
cd services/api
docker build -t jobs-scraper-api .
# Build LinkedIn scraper
cd services/scraper-linkedin
docker build -t jobs-scraper-linkedin .
```
## 🔧 Configuration
Environment variables are managed through `.env` files:
- `.env` - Production configuration
- `.local.env` - Local development configuration
## 🧪 Testing
Run tests for individual services:
```bash
# API Service
cd services/api && go test ./...
# LinkedIn Scraper
cd services/scraper-linkedin && go test ./...
# Cron Analyzer
cd apps/cron-analyzer && go test ./...
```
## 📊 Monitoring
- **API Documentation**: http://localhost:8080/swagger/
- **RabbitMQ Management**: http://localhost:15672 (guest/guest)
- **Health Checks**: Built into each service
## 🚀 Deployment
The monorepo includes GitHub Actions workflows for:
- Automated testing
- Docker image building
- Multi-service deployment
## 🤝 Contributing
1. Fork the repository
2. Create a feature branch
3. Make your changes
4. Add tests
5. Submit a pull request
## 📄 License
[Add your license information here]
## 🆘 Support
For support and questions, please [create an issue](issues) or contact the development team.

17
apps/cron-analyzer/.env Normal file
View file

@ -0,0 +1,17 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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
# OpenAI Configuration
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini

BIN
apps/cron-analyzer/bin/cron-analyzer
View file

Binary file not shown.

17
apps/cron-analyzer/cmd/cli/.env Normal file
View file

@ -0,0 +1,17 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=password
DB_NAME=jobs_scraper
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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
# OpenAI Configuration
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini

16
apps/cron-analyzer/cmd/cli/main.go
View file

@ -9,10 +9,9 @@ import (
) )
func main() { func main() {
// Try to load .local.env first, then fallback to .env if err := godotenv.Load("cmd/cli/.local.env"); err != nil {
if err := godotenv.Load(".local.env"); err != nil { log.Printf("No .local.env file found: %v, trying .env", err)
log.Println("No .local.env file found, trying .env") if err := godotenv.Load("cmd/cli/.env"); err != nil {
if err := godotenv.Load(".env"); err != nil {
log.Println("No .env file found, using system environment variables") log.Println("No .env file found, using system environment variables")
} }
} }
@ -21,7 +20,7 @@ func main() {
db, err := infrastructure.NewConnection(dbConfig) db, err := infrastructure.NewConnection(dbConfig)
if err != nil { if err != nil {
log.Fatal("Error connecting to db") log.Fatalf("Error connecting to db: %v", err)
} }
err = db.Ping() err = db.Ping()
@ -31,6 +30,13 @@ func main() {
log.Println("Successfully connected to db") log.Println("Successfully connected to db")
// Run migrations
log.Println("Running database migrations...")
if err := infrastructure.RunMigrations(db); err != nil {
log.Fatalf("Error running migrations: %v", err)
}
log.Println("Migrations completed successfully")
analyzer := internal.NewJobAnalyzer(db) analyzer := internal.NewJobAnalyzer(db)
if err := analyzer.AnalyzeJobs(); err != nil { if err := analyzer.AnalyzeJobs(); err != nil {

0
cv.txt → apps/cron-analyzer/cv.txt
View file

22
apps/cron-analyzer/internal/analyze-jobs.go
View file

@ -6,8 +6,11 @@ import (
"log" "log"
"os" "os"
// "path/filepath"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/pkg/openai"
"github.com/jobs-scraper/internal/pkg/openrouter" "github.com/jobs-scraper/internal/pkg/openrouter"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
) )
@ -19,16 +22,28 @@ type JobAnalyzer struct {
jobAnalysisResultRepo *repo.JobAnalysisResultRepository jobAnalysisResultRepo *repo.JobAnalysisResultRepository
openRouterService openrouter.OpenRouterService openRouterService openrouter.OpenRouterService
rmq *rabbitmq.RabbitMQClient rmq *rabbitmq.RabbitMQClient
openAiService *openai.OpenAIService
} }
func NewJobAnalyzer(db *sql.DB) *JobAnalyzer { func NewJobAnalyzer(db *sql.DB) *JobAnalyzer {
apiKey := os.Getenv("OPENROUTER_API_KEY") apiKey := os.Getenv("OPENROUTER_API_KEY")
model := os.Getenv("CV_AI_MODEL") model := os.Getenv("CV_AI_MODEL")
openAiapiKey := os.Getenv("OPENAI_API_KEY")
if openAiapiKey == "" {
log.Fatal("OPENAI_API_KEY environment variable is required")
}
// Create OpenAI service
openAiModel := os.Getenv("OPENAI_MODEL")
if openAiModel == "" {
openAiModel = "gpt-4o-mini" // Default to cost-effective model
}
jobRepo := repo.NewJobRepository(db) jobRepo := repo.NewJobRepository(db)
jobDescriptionRepo := repo.NewJobDescriptionRepository(db) jobDescriptionRepo := repo.NewJobDescriptionRepository(db)
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
openRouterService := openrouter.NewOpenRouterService(model, apiKey) openRouterService := openrouter.NewOpenRouterService(model, apiKey)
openAiService := openai.NewOpenAIService(openAiapiKey, openAiModel)
rmq, err := rabbitmq.NewRabbitMQClient() rmq, err := rabbitmq.NewRabbitMQClient()
if err != nil { if err != nil {
@ -42,6 +57,7 @@ func NewJobAnalyzer(db *sql.DB) *JobAnalyzer {
jobAnalysisResultRepo: jobAnalysisResultRepo, jobAnalysisResultRepo: jobAnalysisResultRepo,
openRouterService: openRouterService, openRouterService: openRouterService,
rmq: rmq, rmq: rmq,
openAiService: openAiService,
} }
} }
@ -54,8 +70,8 @@ func (ja *JobAnalyzer) AnalyzeJobs() error {
log.Printf("Found %d jobs to analyze", len(jobs)) log.Printf("Found %d jobs to analyze", len(jobs))
// Read CV file cv, err := os.ReadFile("cv.txt")
cv, err := os.ReadFile("../../cv.txt")
if err != nil { if err != nil {
return err return err
} }
@ -89,7 +105,7 @@ func (ja *JobAnalyzer) analyzeJob(job domain.Job, cv string) error {
} }
// Analyze job with CV // Analyze job with CV
result, err := ja.openRouterService.AnalyzeJobDescription(cv, domain.JobDescription{ result, err := ja.openAiService.AnalyzeJobDescription(cv, domain.JobDescription{
JobID: job.ID, JobID: job.ID,
Description: jobDescription, Description: jobDescription,
Criteria: jobCriteria, Criteria: jobCriteria,

BIN
apps/cv-analyzer/bin/cv-analyzer
View file

Binary file not shown.

72
apps/cv-analyzer/cmd/cli/main.go
View file

@ -1,72 +0,0 @@
package main
import (
"log"
"os"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/pkg/openrouter"
"github.com/jobs-scraper/libs/repo"
"github.com/joho/godotenv"
)
func main() {
// Try to load .local.env first, then fallback to .env
if err := godotenv.Load("../.local.env"); err != nil {
log.Println("No .local.env file found, trying .env")
if err := godotenv.Load("../.env"); err != nil {
log.Println("No .env file found, using system environment variables")
}
}
dbConfig := infrastructure.LoadConfigFromEnv()
db, err := infrastructure.NewConnection(dbConfig)
if err != nil {
log.Fatal("Error connecting to db")
}
apiKey := os.Getenv("OPENROUTER_API_KEY")
model := os.Getenv("CV_AI_MODEL")
err = db.Ping()
if err != nil {
log.Fatal("Error pinging db")
}
log.Println("Successfully connected to db")
jobRepo := repo.NewJobRepository(db)
jobDescriptionRepo := repo.NewJobDescriptionRepository(db)
openRouterService := openrouter.NewOpenRouterService(model, apiKey)
cv, err := os.ReadFile("../../cv.txt")
if err != nil {
log.Fatalf("Failed to get cv: %v", err)
}
job, err := jobRepo.GetJobByID(4312925061)
if err != nil {
log.Fatalf("Failed to get job: %v", err)
}
jobDescription, jobCriteria, err := jobDescriptionRepo.GetJobDescriptionByJobID(job.ID)
if err != nil {
log.Fatalf("Failed to get job description: %v", err)
}
result, err := openRouterService.AnalyzeJobDescription(string(cv), domain.JobDescription{
JobID: job.ID,
Description: jobDescription,
Criteria: jobCriteria,
})
if err != nil {
log.Fatalf("Failed to get job analysis result: %v", err)
}
log.Println(result)
}

15
apps/cv-analyzer/go.mod
View file

@ -1,15 +0,0 @@
module github.com/jobs-scraper/apps/cv-analyzer
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0-00010101000000-000000000000
github.com/joho/godotenv v1.5.1
)
require github.com/lib/pq v1.10.9 // indirect
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/libs/repo => ../../libs/repo

2
apps/cv-analyzer/go.sum
View file

@ -1,2 +0,0 @@
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=

19
go.mod
View file

@ -3,22 +3,3 @@ module github.com/jobs-scraper
go 1.24.0 go 1.24.0
toolchain go1.24.7 toolchain go1.24.7
require github.com/swaggo/swag v1.16.3
require (
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/spec v0.20.6 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
golang.org/x/tools v0.36.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
require (
github.com/stretchr/testify v1.10.0 // indirect
golang.org/x/sync v0.17.0 // indirect
)

52
go.sum
View file

@ -1,52 +0,0 @@
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ=
github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

3
go.work
View file

@ -3,9 +3,9 @@ go 1.24.0
use ( use (
. .
./apps/cron-analyzer ./apps/cron-analyzer
./apps/cv-analyzer
./internal/pkg/domain ./internal/pkg/domain
./internal/pkg/infrastructure ./internal/pkg/infrastructure
./internal/pkg/openai
./internal/pkg/openrouter ./internal/pkg/openrouter
./internal/pkg/utils ./internal/pkg/utils
./libs/ports ./libs/ports
@ -13,5 +13,4 @@ use (
./libs/server ./libs/server
./services/api ./services/api
./services/scraper-linkedin ./services/scraper-linkedin
./tools
) )

157
internal/pkg/openai/README.md Normal file
View file

@ -0,0 +1,157 @@
# OpenAI Service
This package provides an OpenAI-powered service for job analysis and CV generation using the official OpenAI Go SDK.
## Features
- **Job Analysis**: Analyze job descriptions against CVs to determine fit and provide recommendations
- **CV Generation**: Create tailored CVs based on job descriptions
- **Cover Letter Generation**: Generate personalized cover letters
## Setup
1. Install the OpenAI Go SDK:
```bash
go get github.com/sashabaranov/go-openai
```
2. Set your OpenAI API key as an environment variable:
```bash
export OPENAI_API_KEY="your-api-key-here"
```
3. (Optional) Set a custom base URL:
```bash
export OPENAI_BASE_URL="https://your-custom-endpoint.com/v1"
```
## Usage
### Basic Job Analysis
```go
package main
import (
"fmt"
"os"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/openai"
)
func main() {
// Create service
apiKey := os.Getenv("OPENAI_API_KEY")
service := openai.NewOpenAIService(apiKey, "gpt-4o-mini")
// Prepare job description
jobDesc := domain.JobDescription{
JobID: 1,
Description: "Looking for a Go developer...",
Criteria: map[string]string{
"experience": "5+ years",
"languages": "Go, JavaScript",
},
}
// Analyze job
result, err := service.AnalyzeJobDescription(cv, jobDesc)
if err != nil {
panic(err)
}
fmt.Printf("Should apply: %t\n", result.ShouldApply())
fmt.Printf("Confidence: %d%%\n", result.ConfidenceScore)
}
```
### Using Custom Base URL
You can configure a custom base URL in two ways:
**Method 1: Environment Variable**
```bash
export OPENAI_BASE_URL="https://api.openai-proxy.com/v1"
# Service will automatically use this URL
service := openai.NewOpenAIService(apiKey, "gpt-4o-mini")
```
**Method 2: Direct Parameter**
```go
// Use a custom base URL directly
service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4o-mini", "https://api.openai-proxy.com/v1")
// Or use Azure OpenAI
service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4", "https://your-resource.openai.azure.com/")
```
### Generate Tailored CV
```go
tailoredCV, err := service.GenerateCV(originalCV, jobDesc)
if err != nil {
panic(err)
}
fmt.Println(tailoredCV)
```
### Generate Cover Letter
```go
coverLetter, err := service.GenerateCoverLetter(cv, jobDesc, "Tech Company Inc")
if err != nil {
panic(err)
}
fmt.Println(coverLetter)
```
## Models
The service supports all OpenAI models. Common choices:
- `gpt-4o-mini` - Cost-effective, good performance (default)
- `gpt-4o` - Higher quality, more expensive
- `gpt-3.5-turbo` - Fastest, most cost-effective
## Environment Variables
- `OPENAI_API_KEY` - Your OpenAI API key (required)
- `OPENAI_BASE_URL` - Custom base URL for OpenAI API (optional)
- `OPENAI_MODEL` - Default model to use (optional)
## Response Structure
### JobAnalysisResult
```go
type JobAnalysisResult struct {
Recommendation string // "apply" or "do_not_apply"
ConfidenceScore int // 0-100
MatchingSkills []string
MissingSkills []string
ExperienceMatch string // "excellent", "good", "fair", "poor"
Summary string
ImprovementSuggestions []string
}
```
## Error Handling
The service handles common errors:
- API authentication issues
- Rate limiting
- Invalid JSON responses
- Network timeouts
Always check for errors when calling service methods.
## Testing
Run the example test:
```bash
cd internal/pkg/openai
go test -v
```
Make sure to set `OPENAI_API_KEY` environment variable before running tests that make actual API calls.

10
internal/pkg/openai/go.mod Normal file
View file

@ -0,0 +1,10 @@
module github.com/jobs-scraper/internal/pkg/openai
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.41.2
)
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

2
internal/pkg/openai/go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM=
github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=

27
internal/pkg/openai/interface.go Normal file
View file

@ -0,0 +1,27 @@
package openai
import "github.com/jobs-scraper/internal/pkg/domain"
// JobAnalysisService defines the interface for job analysis services
type JobAnalysisService interface {
// AnalyzeJobDescription analyzes a job description against a CV
AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error)
}
// CVService defines the interface for CV-related services
type CVService interface {
// GenerateCV creates a tailored CV based on a job description
GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error)
// GenerateCoverLetter creates a cover letter based on CV and job description
GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error)
}
// AIService combines all AI-related services
type AIService interface {
JobAnalysisService
CVService
}
// Ensure OpenAIService implements AIService
var _ AIService = (*OpenAIService)(nil)

275
internal/pkg/openai/openai.go Normal file
View file

@ -0,0 +1,275 @@
package openai
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/sashabaranov/go-openai"
)
// JobAnalysisResult represents the structured response from job analysis
type JobAnalysisResult struct {
Recommendation string `json:"recommendation"`
ConfidenceScore int `json:"confidence_score"`
MatchingSkills []string `json:"matching_skills"`
MissingSkills []string `json:"missing_skills"`
ExperienceMatch string `json:"experience_match"`
Summary string `json:"summary"`
ImprovementSuggestions []string `json:"improvement_suggestions"`
}
// ShouldApply returns true if the recommendation is to apply for the job
func (r *JobAnalysisResult) ShouldApply() bool {
return r.Recommendation == "apply"
}
// IsHighConfidence returns true if the confidence score is 70 or above
func (r *JobAnalysisResult) IsHighConfidence() bool {
return r.ConfidenceScore >= 70
}
// OpenAIService provides job analysis using OpenAI's API
type OpenAIService struct {
client *openai.Client
model string
}
// NewOpenAIService creates a new OpenAI service instance
func NewOpenAIService(apiKey, model string) *OpenAIService {
return NewOpenAIServiceWithBaseURL(apiKey, model, "")
}
// NewOpenAIServiceWithBaseURL creates a new OpenAI service instance with a custom base URL
func NewOpenAIServiceWithBaseURL(apiKey, model, baseURL string) *OpenAIService {
if model == "" {
model = openai.GPT4oMini // Default to GPT-4o-mini for cost efficiency
}
// Create client configuration
config := openai.DefaultConfig(apiKey)
// Use provided baseURL, or check environment variable, or use default
if baseURL != "" {
config.BaseURL = baseURL
} else if envBaseURL := os.Getenv("OPENAI_BASE_URL"); envBaseURL != "" {
config.BaseURL = envBaseURL
}
client := openai.NewClientWithConfig(config)
return &OpenAIService{
client: client,
model: model,
}
}
// AnalyzeJobDescription analyzes a job description against a CV using OpenAI
func (s *OpenAIService) AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) {
// Build the user message with all the provided data
userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
2) The job shouldn't require any language skills, preferably only english.
3) The job should be remote, or provide relocation to the country.
CV:
%s
Job Description:
%s
Job Criteria (key-value):
%v
CRITICAL OUTPUT REQUIREMENTS:
- Return ONLY raw JSON - NO markdown formatting whatsoever
- NO backticks, NO code blocks, NO json prefix
- NO additional text before or after the JSON
- Start your response directly with { and end with }
- Use this exact schema and key names:
{
"recommendation": "apply" | "do_not_apply",
"confidence_score": number, // integer 0-100
"matching_skills": [string],
"missing_skills": [string],
"experience_match": "excellent" | "good" | "fair" | "poor",
"summary": string,
"improvement_suggestions": [string]
}`, cv, jobDesc.Description, jobDesc.Criteria)
// Create the chat completion request
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.",
},
{
Role: openai.ChatMessageRoleUser,
Content: userMessage,
},
},
Temperature: 0.3, // Lower temperature for more consistent responses
MaxTokens: 1000,
}
// Execute the request
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return nil, fmt.Errorf("failed to create chat completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from OpenAI API")
}
// Extract JSON from the response
jsonContent := resp.Choices[0].Message.Content
// Log the raw response for debugging
log.Printf("Raw OpenAI response: %s", jsonContent)
// Clean up markdown code blocks if present
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
// Parse the JSON response into our struct
var result JobAnalysisResult
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(200, len(jsonContent))])
}
return &result, nil
}
// GenerateCV creates a tailored CV based on a job description
func (s *OpenAIService) GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error) {
userMessage := fmt.Sprintf(`Based on the original CV and job description below, create a tailored CV that highlights relevant experience and skills for this specific position.
Original CV:
%s
Job Description:
%s
Job Criteria:
%v
Instructions:
1. Keep all factual information accurate - do not fabricate experience or skills
2. Reorganize and emphasize relevant experience and skills
3. Use keywords from the job description where appropriate
4. Maintain professional formatting
5. Focus on achievements and quantifiable results
6. Return the tailored CV in plain text format`, originalCV, jobDesc.Description, jobDesc.Criteria)
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are an expert CV writer who specializes in tailoring resumes for specific job applications. You help candidates present their existing experience and skills in the most relevant way for each position, without fabricating information.",
},
{
Role: openai.ChatMessageRoleUser,
Content: userMessage,
},
},
Temperature: 0.7, // Higher temperature for more creative writing
MaxTokens: 2000,
}
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return "", fmt.Errorf("failed to create chat completion: %v", err)
}
if len(resp.Choices) == 0 {
return "", fmt.Errorf("no response choices received from OpenAI API")
}
return resp.Choices[0].Message.Content, nil
}
// GenerateCoverLetter creates a cover letter based on CV and job description
func (s *OpenAIService) GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error) {
userMessage := fmt.Sprintf(`Create a professional cover letter based on the CV and job description below.
CV:
%s
Job Description:
%s
Job Criteria:
%v
Company Name: %s
Instructions:
1. Write a compelling opening paragraph
2. Highlight relevant experience and skills from the CV
3. Show enthusiasm for the specific role and company
4. Keep it concise (3-4 paragraphs)
5. Use a professional but engaging tone
6. Include a strong closing paragraph`, cv, jobDesc.Description, jobDesc.Criteria, companyName)
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are an expert career counselor who writes compelling cover letters that help candidates stand out while maintaining professionalism.",
},
{
Role: openai.ChatMessageRoleUser,
Content: userMessage,
},
},
Temperature: 0.8, // Higher temperature for more creative and engaging writing
MaxTokens: 1500,
}
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return "", fmt.Errorf("failed to create chat completion: %v", err)
}
if len(resp.Choices) == 0 {
return "", fmt.Errorf("no response choices received from OpenAI API")
}
return resp.Choices[0].Message.Content, nil
}
// min returns the smaller of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
// cleanMarkdownCodeBlocks removes markdown code block formatting from JSON response
func cleanMarkdownCodeBlocks(content string) string {
// Remove ```json prefix and ``` suffix if present
content = strings.TrimSpace(content)
// Remove opening code block markers
if strings.HasPrefix(content, "```json") {
content = strings.TrimPrefix(content, "```json")
} else if strings.HasPrefix(content, "```") {
content = strings.TrimPrefix(content, "```")
}
// Remove closing code block markers
content = strings.TrimSuffix(content, "```")
return strings.TrimSpace(content)
}

BIN
services/api/bin/api
View file

Binary file not shown.

8
services/api/cmd/server/main.go
View file

@ -10,11 +10,11 @@ import (
"time" "time"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/jobs-scraper/services/api/internal/app"
httpHandler "github.com/jobs-scraper/services/api/pkg/http"
"github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/api/internal/app"
httpHandler "github.com/jobs-scraper/services/api/pkg/http"
_ "github.com/jobs-scraper/services/api/pkg/swagger" _ "github.com/jobs-scraper/services/api/pkg/swagger"
"github.com/joho/godotenv" "github.com/joho/godotenv"
httpSwagger "github.com/swaggo/http-swagger" httpSwagger "github.com/swaggo/http-swagger"
@ -26,9 +26,9 @@ import (
// @BasePath / // @BasePath /
func main() { func main() {
// Try to load .local.env first, then fallback to .env // Try to load .local.env first, then fallback to .env
if err := godotenv.Load("../.local.env"); err != nil { if err := godotenv.Load(".local.env"); err != nil {
log.Println("No .local.env file found, trying .env") log.Println("No .local.env file found, trying .env")
if err := godotenv.Load("../.env"); err != nil { if err := godotenv.Load(".env"); err != nil {
log.Println("No .env file found, using system environment variables") log.Println("No .env file found, using system environment variables")
} }
} }

68
services/scraper-glassdoor/package-lock.json generated
View file

@ -107,19 +107,19 @@
"dev": true "dev": true
}, },
"node_modules/@types/amqplib": { "node_modules/@types/amqplib": {
"version": "0.10.7", "version": "0.10.8",
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.7.tgz", "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz",
"integrity": "sha512-IVj3avf9AQd2nXCx0PGk/OYq7VmHiyNxWFSb5HhU9ATh+i+gHWvVcljFTcTWQ/dyHJCTrzCixde+r/asL2ErDA==", "integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==",
"dependencies": { "dependencies": {
"@types/node": "*" "@types/node": "*"
} }
}, },
"node_modules/@types/node": { "node_modules/@types/node": {
"version": "24.8.1", "version": "24.9.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.8.1.tgz", "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz",
"integrity": "sha512-alv65KGRadQVfVcG69MuB4IzdYVpRwMG/mq8KWOaoOdyY617P5ivaDiMCGOFDWD2sAn5Q0mR3mRtUOgm99hL9Q==", "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==",
"dependencies": { "dependencies": {
"undici-types": "~7.14.0" "undici-types": "~7.16.0"
} }
}, },
"node_modules/@types/yauzl": { "node_modules/@types/yauzl": {
@ -228,9 +228,9 @@
} }
}, },
"node_modules/bare-events": { "node_modules/bare-events": {
"version": "2.8.0", "version": "2.8.1",
"resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.0.tgz", "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.1.tgz",
"integrity": "sha512-AOhh6Bg5QmFIXdViHbMc2tLDsBIRxdkIaIddPslJF9Z5De3APBScuqGP2uThXnIpqFrgoxMNC6km7uXNIMLHXA==", "integrity": "sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==",
"peerDependencies": { "peerDependencies": {
"bare-abort-controller": "*" "bare-abort-controller": "*"
}, },
@ -241,9 +241,9 @@
} }
}, },
"node_modules/bare-fs": { "node_modules/bare-fs": {
"version": "4.4.11", "version": "4.5.0",
"resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.4.11.tgz", "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.0.tgz",
"integrity": "sha512-Bejmm9zRMvMTRoHS+2adgmXw1ANZnCNx+B5dgZpGwlP1E3x6Yuxea8RToddHUbWtVV0iUMWqsgZr8+jcgUI2SA==", "integrity": "sha512-GljgCjeupKZJNetTqxKaQArLK10vpmK28or0+RwWjEl5Rk+/xG3wkpmkv+WrcBm3q1BwHKlnhXzR8O37kcvkXQ==",
"optional": true, "optional": true,
"dependencies": { "dependencies": {
"bare-events": "^2.5.4", "bare-events": "^2.5.4",
@ -304,9 +304,9 @@
} }
}, },
"node_modules/bare-url": { "node_modules/bare-url": {
"version": "2.3.0", "version": "2.3.2",
"resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.0.tgz", "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz",
"integrity": "sha512-c+RCqMSZbkz97Mw1LWR0gcOqwK82oyYKfLoHJ8k13ybi1+I80ffdDzUy0TdAburdrR/kI0/VuN8YgEnJqX+Nyw==", "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==",
"optional": true, "optional": true,
"dependencies": { "dependencies": {
"bare-path": "^3.0.0" "bare-path": "^3.0.0"
@ -334,9 +334,9 @@
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg=="
}, },
"node_modules/chromium-bidi": { "node_modules/chromium-bidi": {
"version": "9.1.0", "version": "10.5.1",
"resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-9.1.0.tgz", "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-10.5.1.tgz",
"integrity": "sha512-rlUzQ4WzIAWdIbY/viPShhZU2n21CxDUgazXVbw4Hu1MwaeUSEksSeM6DqPgpRjCLXRk702AVRxJxoOz0dw4OA==", "integrity": "sha512-rlj6OyhKhVTnk4aENcUme3Jl9h+cq4oXu4AzBcvr8RMmT6BR4a3zSNT9dbIfXr9/BS6ibzRyDhowuw4n2GgzsQ==",
"dependencies": { "dependencies": {
"mitt": "^3.0.1", "mitt": "^3.0.1",
"zod": "^3.24.1" "zod": "^3.24.1"
@ -418,9 +418,9 @@
} }
}, },
"node_modules/devtools-protocol": { "node_modules/devtools-protocol": {
"version": "0.0.1508733", "version": "0.0.1521046",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1508733.tgz", "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1521046.tgz",
"integrity": "sha512-QJ1R5gtck6nDcdM+nlsaJXcelPEI7ZxSMw1ujHpO1c4+9l+Nue5qlebi9xO1Z2MGr92bFOQTW7/rrheh5hHxDg==" "integrity": "sha512-vhE6eymDQSKWUXwwA37NtTTVEzjtGVfDr3pRbsWEQ5onH/Snp2c+2xZHWJJawG/0hCCJLRGt4xVtEVUVILol4w=="
}, },
"node_modules/diff": { "node_modules/diff": {
"version": "4.0.2", "version": "4.0.2",
@ -742,16 +742,16 @@
} }
}, },
"node_modules/puppeteer-core": { "node_modules/puppeteer-core": {
"version": "24.25.0", "version": "24.27.0",
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.25.0.tgz", "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.27.0.tgz",
"integrity": "sha512-8Xs6q3Ut+C8y7sAaqjIhzv1QykGWG4gc2mEZ2mYE7siZFuRp4xQVehOf8uQKSQAkeL7jXUs3mknEeiqnRqUKvQ==", "integrity": "sha512-yubwj2XXmTM3wRIpbhO5nCjbByPgpFHlgrsD4IK+gMPqO7/a5FfnoSXDKjmqi8A2M1Ewusz0rTI/r+IN0GU0MA==",
"dependencies": { "dependencies": {
"@puppeteer/browsers": "2.10.12", "@puppeteer/browsers": "2.10.12",
"chromium-bidi": "9.1.0", "chromium-bidi": "10.5.1",
"debug": "^4.4.3", "debug": "^4.4.3",
"devtools-protocol": "0.0.1508733", "devtools-protocol": "0.0.1521046",
"typed-query-selector": "^2.12.0", "typed-query-selector": "^2.12.0",
"webdriver-bidi-protocol": "0.3.7", "webdriver-bidi-protocol": "0.3.8",
"ws": "^8.18.3" "ws": "^8.18.3"
}, },
"engines": { "engines": {
@ -963,9 +963,9 @@
} }
}, },
"node_modules/undici-types": { "node_modules/undici-types": {
"version": "7.14.0", "version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.14.0.tgz", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==" "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="
}, },
"node_modules/url-parse": { "node_modules/url-parse": {
"version": "1.5.10", "version": "1.5.10",
@ -983,9 +983,9 @@
"dev": true "dev": true
}, },
"node_modules/webdriver-bidi-protocol": { "node_modules/webdriver-bidi-protocol": {
"version": "0.3.7", "version": "0.3.8",
"resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.7.tgz", "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.8.tgz",
"integrity": "sha512-wIx5Gu/LLTeexxilpk8WxU2cpGAKlfbWRO5h+my6EMD1k5PYqM1qQO1MHUFf4f3KRnhBvpbZU7VkizAgeSEf7g==" "integrity": "sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ=="
}, },
"node_modules/wrap-ansi": { "node_modules/wrap-ansi": {
"version": "7.0.0", "version": "7.0.0",

BIN
services/scraper-linkedin/bin/scraper
View file

Binary file not shown.

2
services/scraper-linkedin/main.go
View file

@ -10,11 +10,11 @@ import (
"time" "time"
"github.com/jobs-scraper/services/scraper/pipeline"
"github.com/jobs-scraper/internal/pkg/domain" "github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure" "github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo" "github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper/pipeline"
"github.com/joho/godotenv" "github.com/joho/godotenv"
) )

BIN
services/scraper-linkedin/scraper
View file

Binary file not shown.

7
tools/go.mod
View file

@ -1,7 +0,0 @@
module github.com/jobs-scraper/tools
go 1.24.0
require (
github.com/swaggo/swag v1.16.3
)