init
This commit is contained in:
Elshimy Ziad Magdy Taha
commit
c341cb8ff8
28 changed files with 1938 additions and 0 deletions
16
.env
Normal file
16
.env
Normal file
|
|
@ -0,0 +1,16 @@
|
||||||
|
DB_HOST=localhost
|
||||||
|
DB_PORT=5432
|
||||||
|
DB_USER=postgres
|
||||||
|
DB_PASSWORD=password
|
||||||
|
DB_NAME=linkedin_jobs
|
||||||
|
DB_SSLMODE=disable
|
||||||
|
|
||||||
|
SERVER_PORT=8080
|
||||||
|
SERVER_HOST=localhost
|
||||||
|
|
||||||
|
# LinkedIn parsing settings
|
||||||
|
LINKEDIN_BASE_URL=https://www.linkedin.com
|
||||||
|
REQUEST_TIMEOUT=30s
|
||||||
|
RATE_LIMIT_DELAY=2s
|
||||||
|
CV_AI_MODEL=deepseek/deepseek-chat-v3.1:free
|
||||||
|
OPENROUTER_API_KEY=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
|
||||||
66
.vscode/launch.json
vendored
Normal file
66
.vscode/launch.json
vendored
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
{
|
||||||
|
// Use IntelliSense to learn about possible attributes.
|
||||||
|
// Hover to view descriptions of existing attributes.
|
||||||
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Launch Scraper",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "debug",
|
||||||
|
"program": "${workspaceFolder}/scraper/main.go",
|
||||||
|
"env": {
|
||||||
|
"GO_ENV": "development"
|
||||||
|
},
|
||||||
|
"args": [],
|
||||||
|
"showLog": true,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Launch CV Service",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "debug",
|
||||||
|
"program": "${workspaceFolder}/cv/main.go",
|
||||||
|
"env": {
|
||||||
|
"GO_ENV": "development"
|
||||||
|
},
|
||||||
|
"args": [],
|
||||||
|
"showLog": true,
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Launch Current File",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "debug",
|
||||||
|
"program": "${file}",
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Launch Package",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "debug",
|
||||||
|
"program": "${fileDirname}",
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test Current Package",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "test",
|
||||||
|
"program": "${fileDirname}",
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Test Current File",
|
||||||
|
"type": "go",
|
||||||
|
"request": "launch",
|
||||||
|
"mode": "test",
|
||||||
|
"program": "${file}",
|
||||||
|
"console": "integratedTerminal"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
3
.vscode/mcp.json
vendored
Normal file
3
.vscode/mcp.json
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
{
|
||||||
|
"servers": {}
|
||||||
|
}
|
||||||
333
README.md
Normal file
333
README.md
Normal file
|
|
@ -0,0 +1,333 @@
|
||||||
|
# LinkedIn Jobs Scraper
|
||||||
|
|
||||||
|
A robust Go application that scrapes LinkedIn job postings with **retry logic**, **exponential backoff**, and **concurrent processing** using a streaming pipeline architecture.
|
||||||
|
|
||||||
|
## 🏗️ Architecture Overview
|
||||||
|
|
||||||
|
The application uses a **streaming pipeline** with **intelligent retry mechanisms** that processes jobs concurrently while handling network failures gracefully.
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────┐
|
||||||
|
│ Job Scraping │
|
||||||
|
│ (Sequential + Retry Logic) │
|
||||||
|
│ Page 1 → Page 2 → Page 3 → Page N │ ──┐
|
||||||
|
│ ↓ Retry on failure │ │
|
||||||
|
│ [Exponential Backoff] │ │ Jobs streamed to jobChan
|
||||||
|
└─────────────────────────────────────┘ │ immediately as found
|
||||||
|
│
|
||||||
|
┌─────────────────────────────────────┐ │
|
||||||
|
│ Job Description Processing │ │
|
||||||
|
│ (Concurrent Workers + Retries) │ ←─┘
|
||||||
|
│ Worker 1 Worker 2 Worker 3 ... N │
|
||||||
|
│ ↓ Retry on failure │
|
||||||
|
│ [Exponential Backoff] │
|
||||||
|
└─────────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────┐
|
||||||
|
│ Database Storage │
|
||||||
|
│ Jobs → Job Descriptions → Match │
|
||||||
|
└─────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🚀 Key Features
|
||||||
|
|
||||||
|
### 🔄 Intelligent Retry System
|
||||||
|
- **Exponential Backoff**: 1s → 2s → 4s → 8s delays between retries
|
||||||
|
- **Smart Error Handling**: Retries server errors (5xx), skips client errors (4xx)
|
||||||
|
- **Configurable Limits**: Max retries, delays, and timeouts
|
||||||
|
- **Context Cancellation**: Respects cancellation during retries
|
||||||
|
- **Request Timeout**: 30-second timeout per HTTP request
|
||||||
|
|
||||||
|
### 🚀 Streaming Pipeline Flow
|
||||||
|
1. **Step 1**: Jobs scraped page-by-page with retry logic
|
||||||
|
2. **Step 2**: Workers process job descriptions concurrently with retries
|
||||||
|
3. **Step 3**: Save jobs to database to get auto-generated IDs
|
||||||
|
4. **Step 4**: Get jobs back from database with IDs
|
||||||
|
5. **Step 5**: Map job descriptions to database IDs and save
|
||||||
|
|
||||||
|
### ⚡ Performance Optimizations
|
||||||
|
- **Zero Idle Time**: Workers start immediately when first jobs arrive
|
||||||
|
- **Concurrent Processing**: 5 configurable workers processing descriptions
|
||||||
|
- **Rate Limiting**: 2-second delays between requests to avoid detection
|
||||||
|
- **Resilient Network Handling**: Automatic retry with exponential backoff
|
||||||
|
- **Context-Aware**: Proper cancellation support throughout pipeline
|
||||||
|
|
||||||
|
### 🛡️ Anti-Detection Measures
|
||||||
|
- **Sequential Page Scraping**: Reduces bot detection risk
|
||||||
|
- **Rate Limiting**: Built-in delays between all requests
|
||||||
|
- **User-Agent Headers**: Mimics real browser requests
|
||||||
|
- **Retry Logic**: Handles temporary blocks gracefully
|
||||||
|
- **Configurable Search**: Customizable job search parameters
|
||||||
|
|
||||||
|
## 📁 Project Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
jobs-scraper/
|
||||||
|
├── main.go # Application entry point
|
||||||
|
├── infrastructure/
|
||||||
|
│ └── db.go # Database connection & migrations
|
||||||
|
├── internal/
|
||||||
|
│ ├── models/
|
||||||
|
│ │ └── Job.go # Job data structure
|
||||||
|
│ ├── pipeline/
|
||||||
|
│ │ └── job_pipeline.go # Core streaming pipeline
|
||||||
|
│ ├── repo/
|
||||||
|
│ │ ├── job.go # Job repository
|
||||||
|
│ │ └── job-description.go # Job description repository
|
||||||
|
│ ├── services/
|
||||||
|
│ │ ├── scraper.go # Scraping service wrapper
|
||||||
|
│ │ └── gemini.go # AI job analysis (optional)
|
||||||
|
│ └── scraper.go # Core scraping logic
|
||||||
|
├── migrations/ # Database schema migrations
|
||||||
|
└── cv/ # CV matching (future feature)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔧 Pipeline Components
|
||||||
|
|
||||||
|
### 1. Core Scraper (`internal/scraper.go`)
|
||||||
|
```go
|
||||||
|
// HTTP requests with exponential backoff retry logic
|
||||||
|
func (s *Scraper) RetryableHTTPRequest(ctx context.Context, url string) (*http.Response, error)
|
||||||
|
|
||||||
|
// Context-aware job scraping with retry support
|
||||||
|
func (s *Scraper) ScrapeJobsWithContext(ctx context.Context, page int, params SearchQuery) ([]models.Job, error)
|
||||||
|
```
|
||||||
|
- **Retry Logic**: Up to 3 attempts with exponential backoff (1s → 2s → 4s → 8s)
|
||||||
|
- **Smart Error Handling**: Retries 5xx errors, fails fast on 4xx errors
|
||||||
|
- **Configurable Parameters**: Search keywords, location, work type (remote/hybrid)
|
||||||
|
- **Request Timeout**: 30-second timeout per request
|
||||||
|
|
||||||
|
### 2. Scraper Service (`internal/services/scraper.go`)
|
||||||
|
```go
|
||||||
|
// Streams jobs with configurable search parameters
|
||||||
|
func (s *Scraper) ScrapeLinkedInJobsStreaming(ctx context.Context, numPages int, jobChan chan<- models.Job, params SearchQuery) error
|
||||||
|
```
|
||||||
|
- **Sequential Processing**: Pages scraped one by one to avoid detection
|
||||||
|
- **Rate Limited**: 2-second delays between requests
|
||||||
|
- **Immediate Streaming**: Jobs sent to channel as soon as found
|
||||||
|
- **Configurable Search**: Custom keywords, location, work type filters
|
||||||
|
|
||||||
|
### 3. Pipeline Orchestrator (`internal/pipeline/job_pipeline.go`)
|
||||||
|
```go
|
||||||
|
// Coordinates the entire streaming pipeline with retry-enabled scraping
|
||||||
|
func (p *JobPipeline) ProcessJobsStreaming(ctx context.Context, numPages int, jobRepo *JobRepository, jobDescRepo *JobDescriptionRepository, params SearchQuery) error
|
||||||
|
```
|
||||||
|
|
||||||
|
**Pipeline Steps:**
|
||||||
|
1. **Job Scraping Goroutine**: Scrapes pages and streams to `jobChan`
|
||||||
|
2. **Worker Goroutines**: 5 concurrent workers processing job descriptions
|
||||||
|
3. **Channel Coordinator**: Waits for workers and closes result channel
|
||||||
|
4. **Result Collection**: Main thread collects all results via channel ranging
|
||||||
|
5. **Database Operations**: Sequential saves with proper ID mapping
|
||||||
|
|
||||||
|
### 4. Job Description Workers
|
||||||
|
```go
|
||||||
|
// Each worker processes jobs with retry logic
|
||||||
|
func (p *JobPipeline) jobDescriptionWorker(ctx context.Context, jobChan <-chan models.Job, resultChan chan<- JobDescriptionResult)
|
||||||
|
```
|
||||||
|
- **Retry-Enabled**: Uses `ScrapeJobDescriptionWithContext` with retry logic
|
||||||
|
- **Rate Limited**: 2-second delays per worker to avoid overwhelming servers
|
||||||
|
- **Concurrent Processing**: Multiple workers process descriptions simultaneously
|
||||||
|
- **Graceful Failure**: Failed scrapes don't stop other workers
|
||||||
|
- **Context Cancellation**: Respects cancellation signals
|
||||||
|
|
||||||
|
|
||||||
|
## ⚙️ Setup & Installation
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
- Go 1.21+
|
||||||
|
- PostgreSQL 12+
|
||||||
|
- LinkedIn access (for scraping)
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
Create a `.env` file:
|
||||||
|
```env
|
||||||
|
DB_HOST=localhost
|
||||||
|
DB_PORT=5432
|
||||||
|
DB_USER=postgres
|
||||||
|
DB_PASSWORD=your_password
|
||||||
|
DB_NAME=linkedin_jobs
|
||||||
|
DB_SSLMODE=disable
|
||||||
|
|
||||||
|
# Optional: For AI job analysis
|
||||||
|
GEMINI_API_KEY=your_gemini_api_key
|
||||||
|
```
|
||||||
|
|
||||||
|
### Installation
|
||||||
|
```bash
|
||||||
|
# Clone repository
|
||||||
|
git clone <repository-url>
|
||||||
|
cd jobs-scraper
|
||||||
|
|
||||||
|
# Install dependencies
|
||||||
|
go mod tidy
|
||||||
|
|
||||||
|
# Run database migrations
|
||||||
|
go run main.go
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🚀 Usage
|
||||||
|
|
||||||
|
### Basic Usage
|
||||||
|
```bash
|
||||||
|
go run main.go
|
||||||
|
```
|
||||||
|
|
||||||
|
The application will:
|
||||||
|
1. Connect to PostgreSQL database
|
||||||
|
2. Run pending migrations
|
||||||
|
3. Initialize the streaming pipeline
|
||||||
|
4. Scrape 10 pages of LinkedIn jobs (configurable)
|
||||||
|
5. Process job descriptions concurrently
|
||||||
|
6. Store results in database
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
#### Scraper Configuration
|
||||||
|
```go
|
||||||
|
scraper := internal.NewScraper(internal.Config{
|
||||||
|
Distance: "25", // Search radius in miles
|
||||||
|
SortBy: "R", // Sort by relevance
|
||||||
|
MaxRetries: 3, // Maximum retry attempts
|
||||||
|
BaseDelay: 1 * time.Second, // Base delay for exponential backoff
|
||||||
|
MaxDelay: 30 * time.Second, // Maximum delay between retries
|
||||||
|
RequestTimeout: 30 * time.Second, // HTTP request timeout
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Search Parameters
|
||||||
|
```go
|
||||||
|
searchParams := internal.SearchQuery{
|
||||||
|
Keywords: "Frontend Developer", // Job search keywords
|
||||||
|
Location: "Japan", // Job location
|
||||||
|
FWT: "2,3", // Work type: 2=remote, 3=hybrid
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Pipeline Configuration
|
||||||
|
```go
|
||||||
|
// Pipeline with 5 workers and 1-second rate limit
|
||||||
|
jobPipeline := pipeline.NewJobPipeline(&scraperService, 5, 1*time.Second)
|
||||||
|
|
||||||
|
// Process 10 pages with search parameters
|
||||||
|
err = jobPipeline.ProcessJobsStreaming(ctx, 10, jobRepo, jobDescRepo, searchParams)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 📊 Performance Metrics
|
||||||
|
|
||||||
|
### Before Optimization (Sequential + No Retries)
|
||||||
|
- Scrape 100 jobs: ~2 minutes
|
||||||
|
- Process descriptions: ~5 minutes
|
||||||
|
- **Failures**: High failure rate due to network issues
|
||||||
|
- **Total: ~7+ minutes** (with manual retries)
|
||||||
|
|
||||||
|
### After Optimization (Streaming Pipeline + Retry Logic)
|
||||||
|
- Scrape 100 jobs: ~2 minutes (with automatic retries)
|
||||||
|
- Process descriptions: **~2 minutes (concurrent with retries)**
|
||||||
|
- **Failures**: Near-zero failure rate with exponential backoff
|
||||||
|
- **Total: ~2 minutes** (65% improvement + reliability)
|
||||||
|
|
||||||
|
### Key Improvements
|
||||||
|
1. **Streaming Architecture**: Jobs processed immediately as scraped
|
||||||
|
2. **Concurrent Workers**: 5 workers processing descriptions simultaneously
|
||||||
|
3. **Intelligent Retries**: Automatic retry with exponential backoff
|
||||||
|
4. **Resilient Network Handling**: Graceful handling of temporary failures
|
||||||
|
5. **Smart Error Classification**: Skip permanent errors, retry temporary ones
|
||||||
|
|
||||||
|
## 🔍 Monitoring & Debugging
|
||||||
|
|
||||||
|
The application provides detailed logging with retry information:
|
||||||
|
```
|
||||||
|
Scraping page 1
|
||||||
|
Page 1 complete: sent 25 jobs to channel
|
||||||
|
Processing job: Frontend Developer at Company X
|
||||||
|
Request attempt 1 failed: connection timeout
|
||||||
|
Retrying in 1s... (attempt 1/3)
|
||||||
|
Request attempt 2 succeeded
|
||||||
|
Saved 250 job descriptions to database
|
||||||
|
```
|
||||||
|
|
||||||
|
### Retry Logging
|
||||||
|
```
|
||||||
|
Request attempt 1 failed with status 503
|
||||||
|
Retrying in 1s... (attempt 1/3)
|
||||||
|
Request attempt 2 failed with status 502
|
||||||
|
Retrying in 2s... (attempt 2/3)
|
||||||
|
Request attempt 3 succeeded
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🛠️ Troubleshooting
|
||||||
|
|
||||||
|
### Common Issues
|
||||||
|
|
||||||
|
**Network/Retry Issues**
|
||||||
|
- Check retry configuration in scraper config
|
||||||
|
- Monitor retry logs for patterns
|
||||||
|
- Adjust `MaxRetries`, `BaseDelay`, or `MaxDelay` if needed
|
||||||
|
- Verify network connectivity and DNS resolution
|
||||||
|
|
||||||
|
**Rate Limiting/Blocking**
|
||||||
|
- Increase delays: modify rate limiter from 2s to 5s+
|
||||||
|
- Reduce concurrent workers from 5 to 2-3
|
||||||
|
- Check if IP is temporarily blocked
|
||||||
|
- Verify User-Agent header is set correctly
|
||||||
|
|
||||||
|
**Database Connection Issues**
|
||||||
|
- Verify PostgreSQL is running: `pg_ctl status`
|
||||||
|
- Check connection string in `.env` file
|
||||||
|
- Ensure database exists and migrations ran
|
||||||
|
- Check database logs for connection errors
|
||||||
|
|
||||||
|
**Memory/Performance Issues**
|
||||||
|
- Monitor goroutine count for leaks
|
||||||
|
- Reduce number of pages processed per run
|
||||||
|
- Check channel buffer sizes
|
||||||
|
- Monitor database connection pool usage
|
||||||
|
|
||||||
|
### Debugging Tips
|
||||||
|
|
||||||
|
**Enable Verbose Logging**
|
||||||
|
- Watch retry attempts and delays
|
||||||
|
- Monitor worker processing rates
|
||||||
|
- Check database operation timing
|
||||||
|
- Verify channel coordination
|
||||||
|
|
||||||
|
**Test Configuration**
|
||||||
|
```go
|
||||||
|
// Conservative settings for testing
|
||||||
|
scraper := internal.NewScraper(internal.Config{
|
||||||
|
MaxRetries: 5, // More retries
|
||||||
|
BaseDelay: 2 * time.Second, // Longer delays
|
||||||
|
MaxDelay: 60 * time.Second, // Higher max delay
|
||||||
|
RequestTimeout: 60 * time.Second, // Longer timeout
|
||||||
|
})
|
||||||
|
|
||||||
|
// Fewer workers for testing
|
||||||
|
jobPipeline := pipeline.NewJobPipeline(&scraperService, 2, 3*time.Second)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔮 Future Enhancements
|
||||||
|
|
||||||
|
- [ ] **CV Matching**: Compare scraped jobs against CV requirements
|
||||||
|
- [ ] **AI Analysis**: Enhanced job analysis using Gemini AI
|
||||||
|
- [ ] **Web Interface**: Dashboard for monitoring and results
|
||||||
|
- [ ] **Multiple Sources**: Support for other job boards
|
||||||
|
- [ ] **Real-time Updates**: Continuous scraping with webhooks
|
||||||
|
- [ ] **Advanced Filtering**: Location, salary, experience filters
|
||||||
|
|
||||||
|
## 📝 Contributing
|
||||||
|
|
||||||
|
1. Fork the repository
|
||||||
|
2. Create feature branch (`git checkout -b feature/amazing-feature`)
|
||||||
|
3. Commit changes (`git commit -m 'Add amazing feature'`)
|
||||||
|
4. Push to branch (`git push origin feature/amazing-feature`)
|
||||||
|
5. Open Pull Request
|
||||||
|
|
||||||
|
## 📄 License
|
||||||
|
|
||||||
|
This project is licensed under the MIT License - see the LICENSE file for details.
|
||||||
|
|
||||||
|
## ⚠️ Disclaimer
|
||||||
|
|
||||||
|
This tool is for educational purposes. Please respect LinkedIn's Terms of Service and robots.txt. Use responsibly and consider rate limiting to avoid being blocked.
|
||||||
65
cv.txt
Normal file
65
cv.txt
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
# Ziad Elshimy
|
||||||
|
## Frontend Developer
|
||||||
|
|
||||||
|
Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects.
|
||||||
|
|
||||||
|
`ziadshimy7@gmail.com`
|
||||||
|
`01223381370`
|
||||||
|
`Alexandria`
|
||||||
|
[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6)
|
||||||
|
[GitHub](https://github.com/ziadshimy7)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Work Experience
|
||||||
|
|
||||||
|
### kari - Fullstack Developer
|
||||||
|
**Jun 2022 - current**
|
||||||
|
- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates.
|
||||||
|
- Led the development of multiple internal projects.
|
||||||
|
- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills.
|
||||||
|
- Led task planning and coordinated project timelines, reducing overall project duration by 20%.
|
||||||
|
|
||||||
|
### Callibri - Frontend Developer
|
||||||
|
**Jan 2022 - Mar 2022**
|
||||||
|
- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement.
|
||||||
|
- Collaborated closely with senior developers to manage a complex design project, increasing efficiency.
|
||||||
|
|
||||||
|
### EJADA - Frontend Developer
|
||||||
|
**Mar 2021 - Dec 2021**
|
||||||
|
- Troubleshooted the website's problems and stay up to date on technology.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Education
|
||||||
|
|
||||||
|
### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General
|
||||||
|
**Jan 2017 - Dec 2021**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Skills
|
||||||
|
- HTML5
|
||||||
|
- Cascading Style Sheets (CSS)
|
||||||
|
- SCSS
|
||||||
|
- Tailwind css
|
||||||
|
- JavaScript
|
||||||
|
- TypeScript
|
||||||
|
- React.js
|
||||||
|
- Redux.js
|
||||||
|
- Next.js
|
||||||
|
- Node.js
|
||||||
|
- SSR
|
||||||
|
- Webpack
|
||||||
|
- Vite
|
||||||
|
- Go (Golang)
|
||||||
|
- Microservices
|
||||||
|
- docker
|
||||||
|
- docker-compose
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Certifications
|
||||||
|
- CCNA
|
||||||
|
- The Complete JavaScript Course 2023- Udemy
|
||||||
|
- React - The Complete Guide - Udemy
|
||||||
76
cv/main.go
Normal file
76
cv/main.go
Normal file
|
|
@ -0,0 +1,76 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/jobs-scraper/infrastructure"
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
"github.com/jobs-scraper/internal/repo"
|
||||||
|
"github.com/jobs-scraper/internal/services"
|
||||||
|
"github.com/joho/godotenv"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := godotenv.Load(); err != nil {
|
||||||
|
log.Println("No .env file found, using system environment variables")
|
||||||
|
}
|
||||||
|
|
||||||
|
dbConfig := infrastructure.LoadConfigFromEnv()
|
||||||
|
db, err := infrastructure.NewConnection(dbConfig)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error connecting to db")
|
||||||
|
}
|
||||||
|
|
||||||
|
apiKey := os.Getenv("OPENROUTER_API_KEY")
|
||||||
|
model := os.Getenv("CV_AI_MODEL")
|
||||||
|
|
||||||
|
err = db.Ping()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error pinging db")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("Successfully connected to db")
|
||||||
|
|
||||||
|
// Run database migrations
|
||||||
|
if err := infrastructure.RunMigrations(db); err != nil {
|
||||||
|
log.Fatalf("Failed to run migrations: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("Database migrations completed successfully")
|
||||||
|
|
||||||
|
jobRepo := repo.NewJobRepository(db)
|
||||||
|
jobDescriptionRepo := repo.NewJobDescriptionRepository(db)
|
||||||
|
openRouterService := services.NewOpenRouterService(model, apiKey)
|
||||||
|
|
||||||
|
cv, err := os.ReadFile("../cv.txt")
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to get cv: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
job, err := jobRepo.GetJobByID(4306471753)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to get job: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobDescription, jobCriteria, err := jobDescriptionRepo.GetJobDescriptionByJobID(job.ID)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to get job description: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := openRouterService.AnalyzeJobDescription(string(cv), models.JobDescription{
|
||||||
|
JobID: job.ID,
|
||||||
|
Description: jobDescription,
|
||||||
|
Criteria: jobCriteria,
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to get job analysis result: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println(result)
|
||||||
|
// job,err :=
|
||||||
|
}
|
||||||
22
go.mod
Normal file
22
go.mod
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
module github.com/jobs-scraper
|
||||||
|
|
||||||
|
go 1.24.0
|
||||||
|
|
||||||
|
toolchain go1.24.7
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/PuerkitoBio/goquery v1.10.3
|
||||||
|
github.com/eduardolat/openroutergo v0.1.0
|
||||||
|
github.com/golang-migrate/migrate/v4 v4.19.0
|
||||||
|
github.com/joho/godotenv v1.5.1
|
||||||
|
github.com/lib/pq v1.10.9
|
||||||
|
golang.org/x/time v0.5.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||||
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
|
github.com/orsinium-labs/enum v1.4.0 // indirect
|
||||||
|
golang.org/x/net v0.44.0 // indirect
|
||||||
|
)
|
||||||
148
go.sum
Normal file
148
go.sum
Normal file
|
|
@ -0,0 +1,148 @@
|
||||||
|
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
|
||||||
|
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
|
||||||
|
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||||
|
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
|
||||||
|
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||||
|
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
|
||||||
|
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||||
|
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||||
|
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||||
|
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||||
|
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
|
||||||
|
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
|
||||||
|
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
||||||
|
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||||
|
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
|
||||||
|
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
|
||||||
|
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||||
|
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
|
||||||
|
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||||
|
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||||
|
github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI=
|
||||||
|
github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY=
|
||||||
|
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||||
|
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||||
|
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||||
|
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
|
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||||
|
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||||
|
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||||
|
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
|
||||||
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
|
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
|
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||||
|
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||||
|
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||||
|
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||||
|
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
|
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||||
|
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||||
|
github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
|
||||||
|
github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
|
||||||
|
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||||
|
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
|
||||||
|
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
||||||
|
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
|
||||||
|
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||||
|
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
|
||||||
|
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||||
|
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
||||||
|
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||||
|
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
|
||||||
|
github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM=
|
||||||
|
github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc=
|
||||||
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||||
|
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||||
|
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
|
||||||
|
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
|
||||||
|
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||||
|
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||||
|
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||||
|
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||||
|
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||||
|
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||||
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
|
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||||
|
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||||
|
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||||
|
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||||
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||||
|
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||||
|
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||||
|
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||||
|
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||||
|
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||||
|
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||||
|
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||||
|
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||||
|
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||||
|
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||||
|
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||||
|
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||||
|
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||||
|
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
|
||||||
|
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||||
|
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||||
|
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
|
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
|
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||||
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
|
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||||
|
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
|
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||||
|
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||||
|
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||||
|
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||||
|
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||||
|
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||||
|
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||||
|
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||||
|
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
|
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||||
|
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||||
|
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||||
|
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||||
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
|
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
|
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||||
|
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
||||||
|
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||||
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
|
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||||
|
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||||
|
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||||
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||||
|
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
171
infrastructure/db.go
Normal file
171
infrastructure/db.go
Normal file
|
|
@ -0,0 +1,171 @@
|
||||||
|
package infrastructure
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
|
||||||
|
"github.com/golang-migrate/migrate/v4"
|
||||||
|
"github.com/golang-migrate/migrate/v4/database/postgres"
|
||||||
|
_ "github.com/golang-migrate/migrate/v4/source/file"
|
||||||
|
_ "github.com/lib/pq"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config holds database configuration
|
||||||
|
type Config struct {
|
||||||
|
Host string
|
||||||
|
Port string
|
||||||
|
User string
|
||||||
|
Password string
|
||||||
|
DBName string
|
||||||
|
SSLMode string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewConnection creates a new database connection
|
||||||
|
func NewConnection(config Config) (*sql.DB, error) {
|
||||||
|
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
|
||||||
|
config.Host, config.Port, config.User, config.Password, config.DBName, config.SSLMode)
|
||||||
|
|
||||||
|
db, err := sql.Open("postgres", dsn)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to open database connection: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := db.Ping(); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to ping database: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadConfigFromEnv loads database configuration from environment variables
|
||||||
|
func LoadConfigFromEnv() Config {
|
||||||
|
return Config{
|
||||||
|
Host: getEnv("DB_HOST", "localhost"),
|
||||||
|
Port: getEnv("DB_PORT", "5432"),
|
||||||
|
User: getEnv("DB_USER", "postgres"),
|
||||||
|
Password: getEnv("DB_PASSWORD", "password"),
|
||||||
|
DBName: getEnv("DB_NAME", "linkedin_jobs"),
|
||||||
|
SSLMode: getEnv("DB_SSLMODE", "disable"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getEnv(key, defaultValue string) string {
|
||||||
|
if value := os.Getenv(key); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunMigrations runs all pending migrations
|
||||||
|
func RunMigrations(db *sql.DB) error {
|
||||||
|
driver, err := postgres.WithInstance(db, &postgres.Config{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create postgres driver: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the migrations directory path
|
||||||
|
migrationsPath, err := getMigrationsPath()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get migrations path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := migrate.NewWithDatabaseInstance(
|
||||||
|
fmt.Sprintf("file://%s", migrationsPath),
|
||||||
|
"postgres",
|
||||||
|
driver,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create migrate instance: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||||
|
return fmt.Errorf("failed to run migrations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RollbackMigrations rolls back migrations by the specified number of steps
|
||||||
|
func RollbackMigrations(db *sql.DB, steps int) error {
|
||||||
|
driver, err := postgres.WithInstance(db, &postgres.Config{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create postgres driver: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
migrationsPath, err := getMigrationsPath()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get migrations path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := migrate.NewWithDatabaseInstance(
|
||||||
|
fmt.Sprintf("file://%s", migrationsPath),
|
||||||
|
"postgres",
|
||||||
|
driver,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create migrate instance: %w", err)
|
||||||
|
}
|
||||||
|
defer m.Close()
|
||||||
|
|
||||||
|
if err := m.Steps(-steps); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||||
|
return fmt.Errorf("failed to rollback migrations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMigrationVersion returns the current migration version
|
||||||
|
func GetMigrationVersion(db *sql.DB) (uint, bool, error) {
|
||||||
|
driver, err := postgres.WithInstance(db, &postgres.Config{})
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, fmt.Errorf("failed to create postgres driver: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
migrationsPath, err := getMigrationsPath()
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, fmt.Errorf("failed to get migrations path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := migrate.NewWithDatabaseInstance(
|
||||||
|
fmt.Sprintf("file://%s", migrationsPath),
|
||||||
|
"postgres",
|
||||||
|
driver,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, fmt.Errorf("failed to create migrate instance: %w", err)
|
||||||
|
}
|
||||||
|
defer m.Close()
|
||||||
|
|
||||||
|
version, dirty, err := m.Version()
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, migrate.ErrNilVersion) {
|
||||||
|
return 0, false, nil
|
||||||
|
}
|
||||||
|
return 0, false, fmt.Errorf("failed to get migration version: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return version, dirty, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getMigrationsPath() (string, error) {
|
||||||
|
// Get the directory of the current source file
|
||||||
|
_, filename, _, ok := runtime.Caller(0)
|
||||||
|
if !ok {
|
||||||
|
return "", fmt.Errorf("failed to get current file path")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the project root (go up from infrastructure/ to project root)
|
||||||
|
projectRoot := filepath.Dir(filepath.Dir(filename))
|
||||||
|
migrationsPath := filepath.Join(projectRoot, "migrations")
|
||||||
|
|
||||||
|
// Verify the migrations directory exists
|
||||||
|
if _, err := os.Stat(migrationsPath); err != nil {
|
||||||
|
return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return migrationsPath, nil
|
||||||
|
}
|
||||||
17
internal/models/Job.go
Normal file
17
internal/models/Job.go
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
package models
|
||||||
|
|
||||||
|
type Job struct {
|
||||||
|
ID int64
|
||||||
|
Title string
|
||||||
|
Company string
|
||||||
|
CompanyLink string
|
||||||
|
Location string
|
||||||
|
JobLink string
|
||||||
|
}
|
||||||
|
|
||||||
|
type SearchQuery struct {
|
||||||
|
Keywords string `json:"keywords"`
|
||||||
|
Location string `json:"location"`
|
||||||
|
FWT string `json:"f_WT"` // Work type filter (1=onsite, 2=remote, 3=hybrid)
|
||||||
|
GeoId string `json:"geoId"` // Geographic location ID
|
||||||
|
}
|
||||||
7
internal/models/JobDescription.go
Normal file
7
internal/models/JobDescription.go
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
package models
|
||||||
|
|
||||||
|
type JobDescription struct {
|
||||||
|
JobID int64
|
||||||
|
Description string
|
||||||
|
Criteria map[string]string
|
||||||
|
}
|
||||||
6
internal/models/JobWithDescription.go
Normal file
6
internal/models/JobWithDescription.go
Normal file
|
|
@ -0,0 +1,6 @@
|
||||||
|
package models
|
||||||
|
|
||||||
|
type JobWithDescription struct {
|
||||||
|
Job Job
|
||||||
|
JobDescription JobDescription
|
||||||
|
}
|
||||||
107
internal/pipeline/job_pipeline.go
Normal file
107
internal/pipeline/job_pipeline.go
Normal file
|
|
@ -0,0 +1,107 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
// "log"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
"github.com/jobs-scraper/internal/repo"
|
||||||
|
"golang.org/x/time/rate"
|
||||||
|
)
|
||||||
|
|
||||||
|
// JobDescriptionResult represents the result of job description scraping
|
||||||
|
type JobDescriptionResult struct {
|
||||||
|
Job models.Job
|
||||||
|
Description string
|
||||||
|
Criteria map[string]string
|
||||||
|
Error error
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobPipeline manages the job processing pipeline
|
||||||
|
type JobPipeline struct {
|
||||||
|
scraperService *Scraper
|
||||||
|
numWorkers int
|
||||||
|
rateLimit time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewJobPipeline creates a new job processing pipeline
|
||||||
|
func NewJobPipeline(scraperService *Scraper, numWorkers int, rateLimit time.Duration) *JobPipeline {
|
||||||
|
return &JobPipeline{
|
||||||
|
scraperService: scraperService,
|
||||||
|
numWorkers: numWorkers,
|
||||||
|
rateLimit: rateLimit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProcessJobsStreaming processes jobs and job descriptions concurrently
|
||||||
|
func (p *JobPipeline) ProcessJobsStreaming(ctx context.Context, numPages int, jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository, params models.SearchQuery) error {
|
||||||
|
// Create channels for the pipeline
|
||||||
|
allJobs := make([]models.Job, 0, 100)
|
||||||
|
allJobDescriptions := make([]models.JobDescription, 0, 100)
|
||||||
|
// var mu sync.Mutex
|
||||||
|
var jbMu sync.Mutex
|
||||||
|
jobsChan := GetJobs(ctx, p.scraperService)
|
||||||
|
jobWithDescriptionChan := GetJobDescription(ctx, p.scraperService, jobsChan)
|
||||||
|
|
||||||
|
for jobWithDescription := range jobWithDescriptionChan {
|
||||||
|
fmt.Printf("Received job description for job : %d\n", jobWithDescription.Job.ID)
|
||||||
|
jbMu.Lock()
|
||||||
|
allJobs = append(allJobs, jobWithDescription.Job)
|
||||||
|
allJobDescriptions = append(allJobDescriptions, jobWithDescription.JobDescription)
|
||||||
|
jbMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := jobRepo.SaveJobs(allJobs); err != nil {
|
||||||
|
return fmt.Errorf("failed to save jobs to database: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := jobDescRepo.SaveJobDescriptions(allJobDescriptions); err != nil {
|
||||||
|
return fmt.Errorf("failed to save job descriptions: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobDescriptionWorker processes jobs from jobChan and sends results to jobDescriptionChan
|
||||||
|
func (p *JobPipeline) jobDescriptionWorker(ctx context.Context, jobChan <-chan models.Job, resultChan chan<- JobDescriptionResult) {
|
||||||
|
|
||||||
|
limiter := rate.NewLimiter(rate.Every(2*time.Second), 1) // 1 request per second
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case job, ok := <-jobChan:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := limiter.Wait(ctx); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Processing job: %s\n", job.Title)
|
||||||
|
|
||||||
|
description, criteria, err := p.scraperService.ScrapeJobDescriptionWithContext(ctx, job)
|
||||||
|
|
||||||
|
result := JobDescriptionResult{
|
||||||
|
Job: job,
|
||||||
|
Description: description,
|
||||||
|
Criteria: criteria,
|
||||||
|
Error: err,
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case resultChan <- result:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
321
internal/pipeline/scraper.go
Normal file
321
internal/pipeline/scraper.go
Normal file
|
|
@ -0,0 +1,321 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
"github.com/jobs-scraper/internal/utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Timespan string // Time filter for job postings (e.g., "r86400" for last 24 hours)
|
||||||
|
Distance string // Search radius in miles (e.g., "25")
|
||||||
|
SortBy string // Sort results by (R for relevance, DD for date posted)
|
||||||
|
MaxRetries int // Maximum number of retries for failed requests
|
||||||
|
BaseDelay time.Duration // Base delay for exponential backoff
|
||||||
|
MaxDelay time.Duration // Maximum delay between retries
|
||||||
|
RequestTimeout time.Duration // Timeout for individual HTTP requests
|
||||||
|
}
|
||||||
|
|
||||||
|
type Scraper struct {
|
||||||
|
config Config
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewScraper(config Config) *Scraper {
|
||||||
|
// r86400 last 24 hours
|
||||||
|
// r604800 last week
|
||||||
|
// r2592000 last month
|
||||||
|
// r7776000 last 3 months
|
||||||
|
// r31536000 last year
|
||||||
|
if config.Timespan == "" {
|
||||||
|
config.Timespan = "r604800" // Default to last week
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set default retry configuration
|
||||||
|
if config.MaxRetries == 0 {
|
||||||
|
config.MaxRetries = 3
|
||||||
|
}
|
||||||
|
if config.BaseDelay == 0 {
|
||||||
|
config.BaseDelay = 1 * time.Second
|
||||||
|
}
|
||||||
|
if config.MaxDelay == 0 {
|
||||||
|
config.MaxDelay = 30 * time.Second
|
||||||
|
}
|
||||||
|
if config.RequestTimeout == 0 {
|
||||||
|
config.RequestTimeout = 30 * time.Second
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Scraper{
|
||||||
|
config: config,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrapeLinkedInJobsStreaming scrapes jobs page by page and sends them to channel immediately
|
||||||
|
func (s *Scraper) ScrapeLinkedInJobsStreaming(ctx context.Context, numPages int, jobChan chan<- models.Job, params models.SearchQuery) error {
|
||||||
|
// Process pages sequentially to send jobs immediately
|
||||||
|
for i := range numPages {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
default:
|
||||||
|
jobs, err := s.ScrapeJobsWithContext(ctx, i, params)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error scraping page %d: %w", i, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send jobs to channel immediately
|
||||||
|
for _, job := range jobs {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case jobChan <- job:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scraper) ScrapeJobsWithContext(ctx context.Context, page int, params models.SearchQuery) ([]models.Job, error) {
|
||||||
|
jobs := make([]models.Job, 0, 10)
|
||||||
|
url := s.buildSearchURL(params, page)
|
||||||
|
|
||||||
|
res, err := utils.RetryableHTTPRequest(ctx, url)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error fetching URL after retries: %v\n", err)
|
||||||
|
return jobs, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer res.Body.Close()
|
||||||
|
|
||||||
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return jobs, err
|
||||||
|
}
|
||||||
|
|
||||||
|
doc.Find("li > div.base-card").Each(func(i int, s *goquery.Selection) {
|
||||||
|
job := models.Job{}
|
||||||
|
title := s.Find("[class*=_title]").Text()
|
||||||
|
job.Title = strings.TrimSpace(title)
|
||||||
|
job.Company = strings.TrimSpace(s.Find(".hidden-nested-link").Text())
|
||||||
|
job.CompanyLink = strings.TrimSpace(s.Find(".hidden-nested-link").AttrOr("href", ""))
|
||||||
|
job.Location = strings.TrimSpace(s.Find(".job-search-card__location").Text())
|
||||||
|
job.JobLink = strings.TrimSpace(s.Find("a.base-card__full-link").AttrOr("href", ""))
|
||||||
|
jobId, err := extractJobIDFromURL(job.JobLink)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error extracting job ID from URL %s: %v\n", job.JobLink, err)
|
||||||
|
}
|
||||||
|
jobIdInt, err := strconv.ParseInt(jobId, 10, 64)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error extracting job ID from URL %s: %v\n", job.JobLink, err)
|
||||||
|
}
|
||||||
|
job.ID = jobIdInt
|
||||||
|
|
||||||
|
if job.Title != "" && job.Company != "" && job.Location != "" && job.JobLink != "" {
|
||||||
|
jobs = append(jobs, job)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return jobs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scraper) ScrapeJobDescriptionWithContext(ctx context.Context, job models.Job) (string, map[string]string, error) {
|
||||||
|
var jobDescription string
|
||||||
|
url := s.buildJobDescriptionSearchURL(job.JobLink)
|
||||||
|
|
||||||
|
res, err := utils.RetryableHTTPRequest(ctx, url)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Error fetching job description URL after retries: %v\n", err)
|
||||||
|
return "", map[string]string{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
defer res.Body.Close()
|
||||||
|
|
||||||
|
doc, err := goquery.NewDocumentFromReader(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return "", map[string]string{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the job details section directly by ID and extract its text content
|
||||||
|
jobDescription, jobCriteria, err := s.parseJobDescription(doc)
|
||||||
|
if err != nil {
|
||||||
|
return "", map[string]string{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.TrimSpace(jobDescription), jobCriteria, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scraper) buildSearchURL(query models.SearchQuery, page int) string {
|
||||||
|
baseURL := "https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search"
|
||||||
|
params := url.Values{}
|
||||||
|
|
||||||
|
// Required search parameters
|
||||||
|
params.Set("keywords", url.QueryEscape(query.Keywords))
|
||||||
|
params.Set("location", url.QueryEscape(query.Location))
|
||||||
|
|
||||||
|
// Time filter
|
||||||
|
// params.Set("f_TPR", s.config.Timespan)
|
||||||
|
|
||||||
|
// Work type filter
|
||||||
|
if query.FWT != "" {
|
||||||
|
params.Set("f_WT", query.FWT)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pagination
|
||||||
|
params.Set("start", strconv.Itoa(25*page))
|
||||||
|
|
||||||
|
return fmt.Sprintf("%s?%s", baseURL, params.Encode())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scraper) buildJobDescriptionSearchURL(jobLink string) string {
|
||||||
|
// For job descriptions, we should use the direct job link
|
||||||
|
return strings.ReplaceAll(jobLink, "jp.linkedin.com", "linkedin.com")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scraper) parseJobDescription(doc *goquery.Document) (string, map[string]string, error) {
|
||||||
|
var jobDescription string
|
||||||
|
jobCriteria := make(map[string]string)
|
||||||
|
|
||||||
|
// Find the main description section
|
||||||
|
descriptionSection := doc.Find("section.core-section-container.description .core-section-container__content")
|
||||||
|
|
||||||
|
if descriptionSection.Length() == 0 {
|
||||||
|
// Alternative selector if classes are different
|
||||||
|
descriptionSection = doc.Find("section.description .core-section-container__content")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the main job description from show-more-less-html section
|
||||||
|
descriptionHTML := descriptionSection.Find("section.show-more-less-html .show-more-less-html__markup")
|
||||||
|
|
||||||
|
if descriptionHTML.Length() > 0 {
|
||||||
|
jobDescription += parseHTMLContent(descriptionHTML)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse job criteria list
|
||||||
|
criteriaList := descriptionSection.Find("ul.description__job-criteria-list")
|
||||||
|
|
||||||
|
if criteriaList.Length() > 0 {
|
||||||
|
criteriaList.Find("li.description__job-criteria-item").Each(func(i int, li *goquery.Selection) {
|
||||||
|
// Get the criteria header
|
||||||
|
header := strings.TrimSpace(li.Find("h3.description__job-criteria-subheader").Text())
|
||||||
|
|
||||||
|
// Get the criteria value
|
||||||
|
value := strings.TrimSpace(li.Find("span.description__job-criteria-text").Text())
|
||||||
|
|
||||||
|
if header != "" && value != "" {
|
||||||
|
jobCriteria[header] = value
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up the description
|
||||||
|
jobDescription = strings.TrimSpace(jobDescription)
|
||||||
|
|
||||||
|
if jobDescription == "" {
|
||||||
|
return "", jobCriteria, fmt.Errorf("job description not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
return jobDescription, jobCriteria, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseHTMLContent walks through HTML elements in order and preserves formatting
|
||||||
|
func parseHTMLContent(selection *goquery.Selection) string {
|
||||||
|
var result strings.Builder
|
||||||
|
|
||||||
|
// Walk through all child nodes in order
|
||||||
|
selection.Contents().Each(func(i int, s *goquery.Selection) {
|
||||||
|
if goquery.NodeName(s) == "#text" {
|
||||||
|
// Handle text nodes
|
||||||
|
text := strings.TrimSpace(s.Text())
|
||||||
|
if text != "" {
|
||||||
|
result.WriteString(text)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Handle element nodes
|
||||||
|
tagName := goquery.NodeName(s)
|
||||||
|
text := strings.TrimSpace(s.Text())
|
||||||
|
|
||||||
|
if text == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
switch tagName {
|
||||||
|
case "p":
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString("\n\n")
|
||||||
|
case "h1", "h2", "h3", "h4", "h5", "h6":
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString("\n\n")
|
||||||
|
case "strong", "b":
|
||||||
|
result.WriteString("**")
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString("**")
|
||||||
|
case "em", "i":
|
||||||
|
result.WriteString("*")
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString("*")
|
||||||
|
case "li":
|
||||||
|
result.WriteString("• ")
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString("\n")
|
||||||
|
case "ul", "ol":
|
||||||
|
// Process list items recursively
|
||||||
|
s.Find("li").Each(func(j int, li *goquery.Selection) {
|
||||||
|
liText := strings.TrimSpace(li.Text())
|
||||||
|
if liText != "" {
|
||||||
|
result.WriteString("• ")
|
||||||
|
result.WriteString(liText)
|
||||||
|
result.WriteString("\n")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
result.WriteString("\n")
|
||||||
|
case "br":
|
||||||
|
result.WriteString("\n")
|
||||||
|
case "div", "span":
|
||||||
|
// For div and span, recursively parse content
|
||||||
|
if s.Children().Length() > 0 {
|
||||||
|
result.WriteString(parseHTMLContent(s))
|
||||||
|
} else {
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString(" ")
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
// For other tags, just extract text content
|
||||||
|
result.WriteString(text)
|
||||||
|
result.WriteString(" ")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return result.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractJobIDFromURL(jobURL string) (string, error) {
|
||||||
|
parsed, err := url.Parse(jobURL)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("invalid URL: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the last segment of the path
|
||||||
|
lastSegment := path.Base(parsed.Path)
|
||||||
|
|
||||||
|
// Extract job ID from the end of the segment using regex
|
||||||
|
// Look for a sequence of digits at the end of the string
|
||||||
|
re := regexp.MustCompile(`(\d+)$`)
|
||||||
|
matches := re.FindStringSubmatch(lastSegment)
|
||||||
|
|
||||||
|
if len(matches) < 2 {
|
||||||
|
return "", fmt.Errorf("job ID not found in URL path: %s", jobURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
return matches[1], nil
|
||||||
|
}
|
||||||
70
internal/pipeline/test.go
Normal file
70
internal/pipeline/test.go
Normal file
|
|
@ -0,0 +1,70 @@
|
||||||
|
package pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
func GetJobs(context context.Context, scraperService *Scraper) <-chan models.Job {
|
||||||
|
jobChan := make(chan models.Job, 100)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
defer close(jobChan)
|
||||||
|
if err := scraperService.ScrapeLinkedInJobsStreaming(context, 10, jobChan, models.SearchQuery{
|
||||||
|
Keywords: "Frontend Developer",
|
||||||
|
Location: "Japan",
|
||||||
|
FWT: "2,3",
|
||||||
|
}); err != nil {
|
||||||
|
log.Printf("Error scraping jobs: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return jobChan
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetJobDescription(context context.Context, scraperService *Scraper, jobChan <-chan models.Job) <-chan models.JobWithDescription {
|
||||||
|
jobDescriptionChan := make(chan models.JobWithDescription, 100)
|
||||||
|
const numWorkers = 3
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
// Start 3 worker goroutines
|
||||||
|
for range numWorkers {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for job := range jobChan {
|
||||||
|
select {
|
||||||
|
case <-context.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scrape job description
|
||||||
|
if jd, jc, err := scraperService.ScrapeJobDescriptionWithContext(context, job); err != nil {
|
||||||
|
log.Printf("Error scraping jobs: %v", err)
|
||||||
|
} else {
|
||||||
|
jobDescriptionChan <- models.JobWithDescription{
|
||||||
|
Job: job,
|
||||||
|
JobDescription: models.JobDescription{
|
||||||
|
JobID: job.ID,
|
||||||
|
Description: jd,
|
||||||
|
Criteria: jc,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close the output channel when all workers are done
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(jobDescriptionChan)
|
||||||
|
}()
|
||||||
|
|
||||||
|
return jobDescriptionChan
|
||||||
|
}
|
||||||
87
internal/repo/job-description.go
Normal file
87
internal/repo/job-description.go
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
package repo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
type JobDescriptionRepository struct {
|
||||||
|
db *sql.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
type JobDescriptionData struct {
|
||||||
|
JobID int
|
||||||
|
Description string
|
||||||
|
Criteria map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewJobDescriptionRepository(db *sql.DB) *JobDescriptionRepository {
|
||||||
|
return &JobDescriptionRepository{db: db}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *JobDescriptionRepository) SaveJobDescriptions(jobDescriptions []models.JobDescription) error {
|
||||||
|
if len(jobDescriptions) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the VALUES clause dynamically
|
||||||
|
valueStrings := make([]string, 0, len(jobDescriptions))
|
||||||
|
valueArgs := make([]interface{}, 0, len(jobDescriptions)*3)
|
||||||
|
|
||||||
|
for i, jd := range jobDescriptions {
|
||||||
|
// Convert criteria map to JSONB
|
||||||
|
criteriaByte, err := json.Marshal(jd.Criteria)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error marshaling job criteria for job %d: %v", jd.JobID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", i*3+1, i*3+2, i*3+3))
|
||||||
|
valueArgs = append(valueArgs, jd.JobID, jd.Description, criteriaByte)
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlStatement := fmt.Sprintf(`
|
||||||
|
INSERT INTO job_descriptions (job_id, description, job_criteria)
|
||||||
|
VALUES %s
|
||||||
|
ON CONFLICT (job_id) DO UPDATE SET
|
||||||
|
description = EXCLUDED.description,
|
||||||
|
job_criteria = EXCLUDED.job_criteria,
|
||||||
|
updated_at = CURRENT_TIMESTAMP
|
||||||
|
`, strings.Join(valueStrings, ","))
|
||||||
|
|
||||||
|
_, err := r.db.Exec(sqlStatement, valueArgs...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error saving job descriptions: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *JobDescriptionRepository) GetJobDescriptionByJobID(jobID int64) (string, map[string]string, error) {
|
||||||
|
var (
|
||||||
|
description string
|
||||||
|
criteriaByte []byte
|
||||||
|
criteria map[string]string
|
||||||
|
)
|
||||||
|
|
||||||
|
sqlStatement := `SELECT description, job_criteria FROM job_descriptions WHERE job_id = $1`
|
||||||
|
err := r.db.QueryRow(sqlStatement, jobID).Scan(&description, &criteriaByte)
|
||||||
|
if err != nil {
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return "", nil, nil // No description found
|
||||||
|
}
|
||||||
|
return "", nil, fmt.Errorf("error fetching job description: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unmarshal the JSON criteria
|
||||||
|
if criteriaByte != nil {
|
||||||
|
if err := json.Unmarshal(criteriaByte, &criteria); err != nil {
|
||||||
|
return "", nil, fmt.Errorf("error unmarshaling job criteria: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return description, criteria, nil
|
||||||
|
}
|
||||||
119
internal/repo/job.go
Normal file
119
internal/repo/job.go
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
package repo
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
type JobRepository struct {
|
||||||
|
db *sql.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewJobRepository(db *sql.DB) *JobRepository {
|
||||||
|
return &JobRepository{db: db}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *JobRepository) SaveJobs(jobs []models.Job) error {
|
||||||
|
if len(jobs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deduplicate jobs by ID to avoid duplicate errors
|
||||||
|
jobMap := make(map[int64]models.Job)
|
||||||
|
for _, job := range jobs {
|
||||||
|
jobMap[job.ID] = job
|
||||||
|
}
|
||||||
|
|
||||||
|
uniqueJobs := make([]models.Job, 0, len(jobMap))
|
||||||
|
for _, job := range jobMap {
|
||||||
|
uniqueJobs = append(uniqueJobs, job)
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlStatement := `
|
||||||
|
INSERT INTO jobs (id, title, company, company_link, location, job_link)
|
||||||
|
VALUES
|
||||||
|
`
|
||||||
|
|
||||||
|
// Create the value placeholders for all jobs
|
||||||
|
vals := []interface{}{}
|
||||||
|
for i, job := range uniqueJobs {
|
||||||
|
|
||||||
|
n := i * 6
|
||||||
|
|
||||||
|
if i > 0 {
|
||||||
|
sqlStatement += ","
|
||||||
|
}
|
||||||
|
sqlStatement += fmt.Sprintf("($%d, $%d, $%d, $%d, $%d, $%d)", n+1, n+2, n+3, n+4, n+5, n+6)
|
||||||
|
|
||||||
|
vals = append(vals, job.ID, job.Title, job.Company, job.CompanyLink, job.Location, job.JobLink)
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlStatement += `
|
||||||
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
|
title = EXCLUDED.title,
|
||||||
|
company = EXCLUDED.company,
|
||||||
|
company_link = EXCLUDED.company_link,
|
||||||
|
location = EXCLUDED.location,
|
||||||
|
job_link = EXCLUDED.job_link
|
||||||
|
`
|
||||||
|
|
||||||
|
_, err := r.db.Exec(sqlStatement, vals...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error inserting jobs: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *JobRepository) GetAllJobs() ([]models.Job, error) {
|
||||||
|
rows, err := r.db.Query("SELECT id, title, company, company_link, location, job_link FROM jobs")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error querying jobs: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
var jobs []models.Job
|
||||||
|
for rows.Next() {
|
||||||
|
var job models.Job
|
||||||
|
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil {
|
||||||
|
return nil, fmt.Errorf("error scanning job row: %v", err)
|
||||||
|
}
|
||||||
|
jobs = append(jobs, job)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("error iterating over job rows: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return jobs, nil
|
||||||
|
}
|
||||||
|
func (r *JobRepository) GetJobByID(id int) (*models.Job, error) {
|
||||||
|
var job models.Job
|
||||||
|
|
||||||
|
sqlStatement := `
|
||||||
|
SELECT id, title, company, company_link, location, job_link
|
||||||
|
FROM jobs
|
||||||
|
WHERE id = $1
|
||||||
|
`
|
||||||
|
|
||||||
|
err := r.db.QueryRow(sqlStatement, id).Scan(
|
||||||
|
&job.ID,
|
||||||
|
&job.Title,
|
||||||
|
&job.Company,
|
||||||
|
&job.CompanyLink,
|
||||||
|
&job.Location,
|
||||||
|
&job.JobLink,
|
||||||
|
)
|
||||||
|
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return nil, fmt.Errorf("job with ID %d not found", id)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error querying job: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &job, nil
|
||||||
|
}
|
||||||
114
internal/services/openrouter.go
Normal file
114
internal/services/openrouter.go
Normal file
|
|
@ -0,0 +1,114 @@
|
||||||
|
package services
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"github.com/eduardolat/openroutergo"
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
)
|
||||||
|
|
||||||
|
// JobAnalysisResult represents the structured response from job analysis
|
||||||
|
type JobAnalysisResult struct {
|
||||||
|
Recommendation string `json:"recommendation"`
|
||||||
|
ConfidenceScore int `json:"confidence_score"`
|
||||||
|
MatchingSkills []string `json:"matching_skills"`
|
||||||
|
MissingSkills []string `json:"missing_skills"`
|
||||||
|
ExperienceMatch string `json:"experience_match"`
|
||||||
|
Summary string `json:"summary"`
|
||||||
|
ImprovementSuggestions []string `json:"improvement_suggestions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldApply returns true if the recommendation is to apply for the job
|
||||||
|
func (r *JobAnalysisResult) ShouldApply() bool {
|
||||||
|
return r.Recommendation == "apply"
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsHighConfidence returns true if the confidence score is 70 or above
|
||||||
|
func (r *JobAnalysisResult) IsHighConfidence() bool {
|
||||||
|
return r.ConfidenceScore >= 70
|
||||||
|
}
|
||||||
|
|
||||||
|
type OpenRouterService struct {
|
||||||
|
model string
|
||||||
|
apiKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewOpenRouterService(model string, apiKey string) OpenRouterService {
|
||||||
|
return OpenRouterService{
|
||||||
|
model: model,
|
||||||
|
apiKey: apiKey,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *OpenRouterService) AnalyzeJobDescription(cv string, jobDesc models.JobDescription) (*JobAnalysisResult, error) {
|
||||||
|
client, err := openroutergo.
|
||||||
|
NewClient().
|
||||||
|
WithAPIKey(s.apiKey).
|
||||||
|
Create()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Failed to create client: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the user message with all the provided data
|
||||||
|
userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
|
||||||
|
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
|
||||||
|
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
|
||||||
|
|
||||||
|
2) The job shouldn't require any language skills, preferbly only english.
|
||||||
|
|
||||||
|
3) The job should be remote, or provide relocation to the country.
|
||||||
|
CV:
|
||||||
|
%s
|
||||||
|
|
||||||
|
Job Description:
|
||||||
|
%s
|
||||||
|
|
||||||
|
Job Criteria (key-value):
|
||||||
|
%v
|
||||||
|
|
||||||
|
OUTPUT REQUIREMENTS:
|
||||||
|
- Return ONLY a single valid JSON object, no markdown, no backticks, no superflous characters so i can parse it.
|
||||||
|
- Do NOT include any markdown, code fences, backticks, or any additional text.
|
||||||
|
- Use this exact schema and key names:
|
||||||
|
{
|
||||||
|
"recommendation": "apply" | "do_not_apply",
|
||||||
|
"confidence_score": number, // integer 0-100
|
||||||
|
"matching_skills": [string],
|
||||||
|
"missing_skills": [string],
|
||||||
|
"experience_match": "excellent" | "good" | "fair" | "poor",
|
||||||
|
"summary": string,
|
||||||
|
"improvement_suggestions": [string]
|
||||||
|
}`, cv, jobDesc.Description, jobDesc.Criteria)
|
||||||
|
|
||||||
|
// Build and execute your request with a fluent API
|
||||||
|
_, resp, err := client.
|
||||||
|
NewChatCompletion().
|
||||||
|
WithModel(s.model).
|
||||||
|
WithSystemMessage("You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. Always respond in valid JSON format.").
|
||||||
|
WithUserMessage(userMessage).
|
||||||
|
Execute()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to execute completion: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(resp.Choices) == 0 {
|
||||||
|
return nil, fmt.Errorf("no response choices received from API")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract JSON from the response (handle markdown code blocks)
|
||||||
|
jsonContent := resp.Choices[0].Message.Content
|
||||||
|
|
||||||
|
// Parse the JSON response into our struct
|
||||||
|
var result JobAnalysisResult
|
||||||
|
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse JSON response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &result, nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// func (s *OpenRouterService) CreateCV(cv string, jobDesc models.JobDescription) (*JobAnalysisResult, error) {
|
||||||
|
// }
|
||||||
67
internal/utils/retryable-http-request.go
Normal file
67
internal/utils/retryable-http-request.go
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type RetryConfig struct {
|
||||||
|
MaxRetries int
|
||||||
|
BaseDelay time.Duration
|
||||||
|
MaxDelay time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func RetryableHTTPRequest(ctx context.Context, url string) (*http.Response, error) {
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: 30 * time.Second,
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastErr error
|
||||||
|
|
||||||
|
for attempt := 0; attempt <= 3; attempt++ {
|
||||||
|
// Create request with context
|
||||||
|
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
||||||
|
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
lastErr = err
|
||||||
|
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
|
||||||
|
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
|
||||||
|
// Success
|
||||||
|
return resp, nil
|
||||||
|
} else if resp.StatusCode == http.StatusTooManyRequests {
|
||||||
|
// 429 Too Many Requests - retry with backoff
|
||||||
|
resp.Body.Close()
|
||||||
|
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
|
||||||
|
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
|
||||||
|
} else {
|
||||||
|
// All other errors (4xx, 5xx) - don't retry
|
||||||
|
resp.Body.Close()
|
||||||
|
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
// max attempts for now = 3
|
||||||
|
if attempt < 3 {
|
||||||
|
delay := min(time.Duration(math.Pow(2, float64(attempt)))*1*time.Second, 30*time.Second)
|
||||||
|
|
||||||
|
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, 3)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(delay):
|
||||||
|
// Continue to next attempt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
|
||||||
|
}
|
||||||
1
migrations/001_jobs.down.sql
Normal file
1
migrations/001_jobs.down.sql
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
DROP TABLE IF EXISTS jobs;
|
||||||
9
migrations/001_jobs.up.sql
Normal file
9
migrations/001_jobs.up.sql
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
CREATE TABLE IF NOT EXISTS jobs (
|
||||||
|
id BIGSERIAL PRIMARY KEY,
|
||||||
|
title VARCHAR(255) NOT NULL,
|
||||||
|
company VARCHAR(255) NOT NULL,
|
||||||
|
company_link TEXT,
|
||||||
|
location VARCHAR(255),
|
||||||
|
job_link TEXT UNIQUE,
|
||||||
|
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
12
migrations/002_job_descriptions.down.sql
Normal file
12
migrations/002_job_descriptions.down.sql
Normal file
|
|
@ -0,0 +1,12 @@
|
||||||
|
-- Drop the trigger first
|
||||||
|
DROP TRIGGER IF EXISTS trigger_update_job_descriptions_timestamp ON job_descriptions;
|
||||||
|
|
||||||
|
-- Drop the trigger function
|
||||||
|
DROP FUNCTION IF EXISTS update_job_descriptions_updated_at();
|
||||||
|
|
||||||
|
-- Drop the indexes
|
||||||
|
DROP INDEX IF EXISTS idx_job_descriptions_criteria;
|
||||||
|
DROP INDEX IF EXISTS idx_job_descriptions_job_id;
|
||||||
|
|
||||||
|
-- Drop the table (this will automatically drop any remaining dependencies)
|
||||||
|
DROP TABLE IF EXISTS job_descriptions;
|
||||||
29
migrations/002_job_descriptions.up.sql
Normal file
29
migrations/002_job_descriptions.up.sql
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
CREATE TABLE IF NOT EXISTS job_descriptions (
|
||||||
|
id SERIAL PRIMARY KEY,
|
||||||
|
job_id BIGINT NOT NULL,
|
||||||
|
description TEXT NOT NULL,
|
||||||
|
job_criteria JSONB DEFAULT '{}'::jsonb,
|
||||||
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Create an index on job_id for faster lookups
|
||||||
|
CREATE INDEX idx_job_descriptions_job_id ON job_descriptions(job_id);
|
||||||
|
|
||||||
|
-- Create a GIN index on the job_criteria JSONB field for faster searching within the JSON data
|
||||||
|
CREATE INDEX idx_job_descriptions_criteria ON job_descriptions USING GIN (job_criteria);
|
||||||
|
|
||||||
|
-- Add trigger to update updated_at timestamp
|
||||||
|
CREATE OR REPLACE FUNCTION update_job_descriptions_updated_at()
|
||||||
|
RETURNS TRIGGER AS $$
|
||||||
|
BEGIN
|
||||||
|
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||||
|
RETURN NEW;
|
||||||
|
END;
|
||||||
|
$$ language 'plpgsql';
|
||||||
|
|
||||||
|
CREATE TRIGGER trigger_update_job_descriptions_timestamp
|
||||||
|
BEFORE UPDATE ON job_descriptions
|
||||||
|
FOR EACH ROW
|
||||||
|
EXECUTE FUNCTION update_job_descriptions_updated_at();
|
||||||
2
migrations/003_add_unique_constraint_job_id.down.sql
Normal file
2
migrations/003_add_unique_constraint_job_id.down.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
-- Remove unique constraint from job_id column in job_descriptions table
|
||||||
|
ALTER TABLE job_descriptions DROP CONSTRAINT IF EXISTS unique_job_id;
|
||||||
2
migrations/003_add_unique_constraint_job_id.up.sql
Normal file
2
migrations/003_add_unique_constraint_job_id.up.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
||||||
|
-- Add unique constraint to job_id column in job_descriptions table
|
||||||
|
ALTER TABLE job_descriptions ADD CONSTRAINT unique_job_id UNIQUE (job_id);
|
||||||
1
migrations/004_job_link_unique.down.sql
Normal file
1
migrations/004_job_link_unique.down.sql
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
ALTER TABLE jobs DROP CONSTRAINT unique_job_link;
|
||||||
1
migrations/004_job_link_unique.up.sql
Normal file
1
migrations/004_job_link_unique.up.sql
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
ALTER TABLE jobs ADD CONSTRAINT unique_job_link UNIQUE (job_link);
|
||||||
66
scraper/main.go
Normal file
66
scraper/main.go
Normal file
|
|
@ -0,0 +1,66 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jobs-scraper/infrastructure"
|
||||||
|
"github.com/jobs-scraper/internal/models"
|
||||||
|
"github.com/jobs-scraper/internal/pipeline"
|
||||||
|
"github.com/jobs-scraper/internal/repo"
|
||||||
|
"github.com/joho/godotenv"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := godotenv.Load(); err != nil {
|
||||||
|
log.Println("No .env file found, using system environment variables")
|
||||||
|
}
|
||||||
|
|
||||||
|
dbConfig := infrastructure.LoadConfigFromEnv()
|
||||||
|
db, err := infrastructure.NewConnection(dbConfig)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error connecting to db")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = db.Ping()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Error pinging db")
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("Successfully connected to db")
|
||||||
|
|
||||||
|
// Run database migrations
|
||||||
|
if err := infrastructure.RunMigrations(db); err != nil {
|
||||||
|
log.Fatalf("Failed to run migrations: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
scraper := pipeline.NewScraper(pipeline.Config{
|
||||||
|
Distance: "25",
|
||||||
|
SortBy: "R",
|
||||||
|
MaxRetries: 3,
|
||||||
|
BaseDelay: 1 * time.Second,
|
||||||
|
MaxDelay: 30 * time.Second,
|
||||||
|
RequestTimeout: 30 * time.Second,
|
||||||
|
})
|
||||||
|
|
||||||
|
jobRepo := repo.NewJobRepository(db)
|
||||||
|
jobDescriptionRepo := repo.NewJobDescriptionRepository(db)
|
||||||
|
|
||||||
|
jobPipeline := pipeline.NewJobPipeline(scraper, 5, 1*time.Second) // 5 workers, 1 second rate limit
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
searchParams := models.SearchQuery{
|
||||||
|
Keywords: "Frontend Developer",
|
||||||
|
Location: "Japan",
|
||||||
|
FWT: "2,3",
|
||||||
|
}
|
||||||
|
|
||||||
|
err = jobPipeline.ProcessJobsStreaming(ctx, 10, jobRepo, jobDescriptionRepo, searchParams)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("Pipeline processing failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println("Jobs inserted successfully")
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue