Merge branch 'master' into 'main'

feat: add job analysis infrastructure and Glassdoor support

See merge request alricade/jobs-scraper!1
This commit is contained in:
Zeyad Elshimy 2025-12-15 07:19:35 +00:00
commit 9f7fa431f7
118 changed files with 8929 additions and 0 deletions

31
.dockerignore Normal file
View file

@ -0,0 +1,31 @@
# Git
.git
.gitignore
# Documentation
README.md
*.md
# IDE
.vscode
.idea
# Logs
*.log
# OS
.DS_Store
Thumbs.db
# Development files
.local.env
# Playwright
.playwright-mcp
# Node modules (if any)
node_modules/
# Temporary files
*.tmp
*.temp

16
.env Normal file
View file

@ -0,0 +1,16 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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

116
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,116 @@
name: CI/CD Pipeline
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main ]
jobs:
test:
runs-on: ubuntu-latest
services:
postgres:
image: postgres:15
env:
POSTGRES_PASSWORD: postgres
POSTGRES_DB: jobs_scraper_test
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 5432:5432
rabbitmq:
image: rabbitmq:3-management
ports:
- 5672:5672
- 15672:15672
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: '1.24'
- name: Set up Node.js
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Cache Go modules
uses: actions/cache@v3
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Install dependencies
run: |
go mod download
cd services/scraper-glassdoor && npm ci
- name: Run tests - API Service
run: |
cd services/api
go test ./...
- name: Run tests - LinkedIn Scraper
run: |
cd services/scraper-linkedin
go test ./...
- name: Run tests - Cron Analyzer
run: |
cd apps/cron-analyzer
go test ./...
- name: Run tests - CV Analyzer
run: |
cd apps/cv-analyzer
go test ./...
- name: Build all services
run: |
make build-api
make build-scraper-linkedin
make build-scraper-glassdoor
make build-cron-analyzer
make build-cv-analyzer
docker:
needs: test
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push API service
uses: docker/build-push-action@v5
with:
context: ./services/api
push: true
tags: ${{ secrets.DOCKER_USERNAME }}/jobs-scraper-api:latest
- name: Build and push LinkedIn scraper
uses: docker/build-push-action@v5
with:
context: ./services/scraper-linkedin
push: true
tags: ${{ secrets.DOCKER_USERNAME }}/jobs-scraper-linkedin:latest

39
.gitignore vendored Normal file
View file

@ -0,0 +1,39 @@
# Local environment files (contains real secrets)
.local.env
# IDE files
.vscode/settings.json
.idea/
# Go build artifacts
*.exe
*.exe~
*.dll
*.so
*.dylib
*.test
*.out
# go.work
# Dependency directories
vendor/
# OS generated files
.DS_Store
.DS_Store?
._*
.Spotlight-V100
.Trashes
ehthumbs.db
Thumbs.db
# Logs
*.log
logs/
# Temporary files
*.tmp
*.temp
node_modules
dist

23
.gitlab-ci.yml Normal file
View file

@ -0,0 +1,23 @@
stages:
- deploy
deploy_production:
stage: deploy
image: alpine:3.20
environment:
name: production
url: https://jobs-scraper.ai-assistant-bot.xyz
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
when: manual
allow_failure: false
before_script:
- apk add --no-cache openssh-client rsync
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
script:
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"

108
.vscode/launch.json vendored Normal file
View file

@ -0,0 +1,108 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Scraper",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/services/scraper-linkedin/main.go",
"cwd": "${workspaceFolder}/services/scraper-linkedin",
"env": {
"GO_ENV": "development"
},
"args": [],
"showLog": true,
"console": "integratedTerminal"
},
{
"name": "Launch CV analyze services",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/apps/cron-analyzer/cmd/cli/main.go",
"cwd": "${workspaceFolder}/apps/cron-analyzer",
"env": {
"GO_ENV": "development"
},
"args": [],
"showLog": true,
"console": "integratedTerminal"
},
{
"name": "Launch OpenAi CV analyze services",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/apps/cron-analyzer/cmd/openai-demo/main.go",
"cwd": "${workspaceFolder}/apps/cron-analyzer",
"env": {
"GO_ENV": "development"
},
"args": [],
"showLog": true,
"console": "integratedTerminal"
},
{
"name": "Launch API Service",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${workspaceFolder}/services/api/cmd/server/main.go",
"cwd": "${workspaceFolder}/services/api",
"env": {
"GO_ENV": "development"
},
"args": [],
"showLog": true,
"console": "integratedTerminal"
},
{
"name": "Launch Glassdoor Scraper",
"type": "node",
"request": "launch",
"program": "${workspaceFolder}/services/scraper-glassdoor/index.ts",
"cwd": "${workspaceFolder}/services/scraper-glassdoor",
"runtimeArgs": ["-r", "ts-node/register"],
"env": {
"NODE_ENV": "development"
},
"console": "integratedTerminal"
},
{
"name": "Launch Current File",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${file}",
"console": "integratedTerminal"
},
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${fileDirname}",
"console": "integratedTerminal"
},
{
"name": "Test Current Package",
"type": "go",
"request": "launch",
"mode": "test",
"program": "${fileDirname}",
"console": "integratedTerminal"
},
{
"name": "Test Current File",
"type": "go",
"request": "launch",
"mode": "test",
"program": "${file}",
"console": "integratedTerminal"
}
]
}

3
.vscode/mcp.json vendored Normal file
View file

@ -0,0 +1,3 @@
{
"servers": {}
}

151
Makefile Normal file
View file

@ -0,0 +1,151 @@
# Jobs Scraper Monorepo Makefile
# Variables
APP_NAME=jobs-scraper
API_DIR=./services/api
SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin
SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor
SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright
CRON_ANALYZER_DIR=./apps/cron-analyzer
CV_ANALYZER_DIR=./apps/cv-analyzer
DOCS_DIR=./services/api/pkg/swagger
BIN_DIR=./bin
API_MAIN_FILE=$(API_DIR)/cmd/server/main.go
SCRAPER_LINKEDIN_MAIN_FILE=$(SCRAPER_LINKEDIN_DIR)/main.go
CRON_MAIN_FILE=$(CRON_ANALYZER_DIR)/cmd/cli/main.go
CV_MAIN_FILE=$(CV_ANALYZER_DIR)/cmd/cli/main.go
# Default target
.DEFAULT_GOAL := help
# RabbitMQ server
rabbitmq-server:
@echo "Starting RabbitMQ server..."
docker run --rm -p 5672:5672 -p 15672:15672 rabbitmq:3-management
# Services
.PHONY: run-api
run-api: ## Run the API service
@echo "Running $(APP_NAME) API service..."
@cd $(API_DIR) && go run cmd/server/main.go
.PHONY: build-api
build-api: ## Build the API service
@echo "Building API service..."
@cd $(API_DIR) && go build -o bin/api cmd/server/main.go
.PHONY: run-scraper-linkedin
run-scraper-linkedin: ## Run the LinkedIn scraper service
@echo "Running LinkedIn scraper service..."
@cd $(SCRAPER_LINKEDIN_DIR) && go run main.go
.PHONY: build-scraper-linkedin
build-scraper-linkedin: ## Build the LinkedIn scraper service
@echo "Building LinkedIn scraper service..."
@cd $(SCRAPER_LINKEDIN_DIR) && go build -o bin/scraper main.go
.PHONY: run-scraper-glassdoor
run-scraper-glassdoor: ## Run the Glassdoor scraper service
@echo "Running Glassdoor scraper service..."
@cd $(SCRAPER_GLASSDOOR_DIR) && npm run dev
.PHONY: build-scraper-glassdoor
build-scraper-glassdoor: ## Build the Glassdoor scraper service
@echo "Building Glassdoor scraper service..."
@cd $(SCRAPER_GLASSDOOR_DIR) && npm run build
.PHONY: setup-scraper-playwright
setup-scraper-playwright: ## Setup the Playwright scraper service
@echo "Setting up Playwright scraper service..."
@cd $(SCRAPER_PLAYWRIGHT_DIR) && bash setup.sh
.PHONY: run-scraper-playwright
run-scraper-playwright: ## Run the Playwright scraper service
@echo "Running Playwright scraper service..."
@cd $(SCRAPER_PLAYWRIGHT_DIR) && source venv/bin/activate && python playwright_worker.py
.PHONY: docker-scraper-playwright
docker-scraper-playwright: ## Build and run Playwright scraper in Docker
@echo "Building Docker image for Playwright scraper..."
@cd $(SCRAPER_PLAYWRIGHT_DIR) && docker build -t playwright-scraper .
@echo "Running Playwright scraper in Docker..."
@docker run --rm --name playwright-scraper playwright-scraper
# Apps
.PHONY: run-cron-analyzer
run-cron-analyzer: ## Run the job analysis cron service
@echo "Starting Job Analysis Cron Service..."
@cd $(CRON_ANALYZER_DIR) && go run cmd/cli/main.go
.PHONY: build-cron-analyzer
build-cron-analyzer: ## Build the cron analyzer app
@echo "Building cron analyzer app..."
@cd $(CRON_ANALYZER_DIR) && go build -o bin/cron-analyzer cmd/cli/main.go
.PHONY: run-cv-analyzer
run-cv-analyzer: ## Run the CV analyzer app
@echo "Running CV analyzer app..."
@cd $(CV_ANALYZER_DIR) && go run cmd/cli/main.go
.PHONY: build-cv-analyzer
build-cv-analyzer: ## Build the CV analyzer app
@echo "Building CV analyzer app..."
@cd $(CV_ANALYZER_DIR) && go build -o bin/cv-analyzer cmd/cli/main.go
.PHONY: run-openai-demo
run-openai-demo: ## Run the OpenAI service demo
@echo "Running OpenAI service demo..."
@cd $(CRON_ANALYZER_DIR) && go run cmd/openai-demo/main.go
.PHONY: build-openai-demo
build-openai-demo: ## Build the OpenAI service demo
@echo "Building OpenAI service demo..."
@cd $(CRON_ANALYZER_DIR) && go build -o bin/openai-demo cmd/openai-demo/main.go
# Legacy aliases for backward compatibility
.PHONY: run
run: run-api ## Alias for run-api
.PHONY: scraper
scraper: run-scraper-linkedin ## Alias for run-scraper-linkedin
.PHONY: cron
cron: run-cron-analyzer ## Alias for run-cron-analyzer
.PHONY: migrate
migrate: ## Run database migrations
@echo "Running database migrations..."
@go run $(MIGRATE_MAIN_FILE)
.PHONY: clean
clean: ## Clean build artifacts
@echo "Cleaning build artifacts..."
@rm -rf $(BIN_DIR)
@rm -f $(SCRAPER_DIR)/scraper
@echo "Clean complete"
.PHONY: help
help: ## Show this help message
@echo "Jobs Scraper Monorepo - Available commands:"
@echo ""
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}'
# Swagger documentation
.PHONY: swagger
swagger: ## Generate Swagger documentation
@echo "Generating Swagger documentation..."
@go run github.com/swaggo/swag/cmd/swag@latest init -g $(API_MAIN_FILE) -o $(DOCS_DIR)
@echo "Swagger documentation generated in $(DOCS_DIR)/"
# Docker operations (if needed in the future)
.PHONY: docker-build
docker-build: ## Build Docker image
@echo "Building Docker image..."
@docker build -t $(APP_NAME) .
@echo "Docker image built: $(APP_NAME)"
.PHONY: docker-run
docker-run: ## Run Docker container
@echo "Running Docker container..."
@docker run -p 8080:8080 $(APP_NAME)

17
apps/cron-analyzer/.env Normal file
View file

@ -0,0 +1,17 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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
# OpenAI Configuration
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini

View file

@ -0,0 +1,17 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=password
DB_NAME=jobs_scraper
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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
# OpenAI Configuration
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini

View file

@ -0,0 +1,47 @@
package main
import (
"log"
"github.com/jobs-scraper/apps/cron-analyzer/internal"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/joho/godotenv"
)
func main() {
if err := godotenv.Load("cmd/cli/.local.env"); err != nil {
log.Printf("No .local.env file found: %v, trying .env", err)
if err := godotenv.Load("cmd/cli/.env"); err != nil {
log.Println("No .env file found, using system environment variables")
}
}
dbConfig := infrastructure.LoadConfigFromEnv()
db, err := infrastructure.NewConnection(dbConfig)
if err != nil {
log.Fatalf("Error connecting to db: %v", err)
}
err = db.Ping()
if err != nil {
log.Fatal("Error pinging db")
}
log.Println("Successfully connected to db")
// Run migrations
log.Println("Running database migrations...")
if err := infrastructure.RunMigrations(db); err != nil {
log.Fatalf("Error running migrations: %v", err)
}
log.Println("Migrations completed successfully")
analyzer := internal.NewJobAnalyzer(db)
if err := analyzer.AnalyzeJobs(); err != nil {
log.Fatalf("Error analyzing jobs: %v", err)
}
log.Println("Job analysis completed successfully")
}

65
apps/cron-analyzer/cv.txt Normal file
View file

@ -0,0 +1,65 @@
# Ziad Elshimy
## Frontend Developer
Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects.
`ziadshimy7@gmail.com`
`01223381370`
`Alexandria`
[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6)
[GitHub](https://github.com/ziadshimy7)
---
## Work Experience
### kari - Fullstack Developer
**Jun 2022 - current**
- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates.
- Led the development of multiple internal projects.
- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills.
- Led task planning and coordinated project timelines, reducing overall project duration by 20%.
### Callibri - Frontend Developer
**Jan 2022 - Mar 2022**
- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement.
- Collaborated closely with senior developers to manage a complex design project, increasing efficiency.
### EJADA - Frontend Developer
**Mar 2021 - Dec 2021**
- Troubleshooted the website's problems and stay up to date on technology.
---
## Education
### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General
**Jan 2017 - Dec 2021**
---
## Skills
- HTML5
- Cascading Style Sheets (CSS)
- SCSS
- Tailwind css
- JavaScript
- TypeScript
- React.js
- Redux.js
- Next.js
- Node.js
- SSR
- Webpack
- Vite
- Go (Golang)
- Microservices
- docker
- docker-compose
---
## Certifications
- CCNA
- The Complete JavaScript Course 2023- Udemy
- React - The Complete Guide - Udemy

29
apps/cron-analyzer/go.mod Normal file
View file

@ -0,0 +1,29 @@
module github.com/jobs-scraper/apps/cron-analyzer
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/libs/repo v0.0.0
github.com/joho/godotenv v1.5.1
)
require (
github.com/eduardolat/openroutergo v0.1.0 // indirect
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/orsinium-labs/enum v1.4.0 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
)
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter
replace github.com/jobs-scraper/libs/repo => ../../libs/repo

79
apps/cron-analyzer/go.sum Normal file
View file

@ -0,0 +1,79 @@
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI=
github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM=
github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -0,0 +1,157 @@
package internal
import (
"database/sql"
"encoding/json"
"log"
"os"
// "path/filepath"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/pkg/openai"
"github.com/jobs-scraper/internal/pkg/openrouter"
"github.com/jobs-scraper/libs/repo"
)
type JobAnalyzer struct {
db *sql.DB
jobRepo *repo.JobRepository
jobDescriptionRepo *repo.JobDescriptionRepository
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
openRouterService openrouter.OpenRouterService
rmq *rabbitmq.RabbitMQClient
openAiService *openai.OpenAIService
}
func NewJobAnalyzer(db *sql.DB) *JobAnalyzer {
apiKey := os.Getenv("OPENROUTER_API_KEY")
model := os.Getenv("CV_AI_MODEL")
openAiapiKey := os.Getenv("OPENAI_API_KEY")
if openAiapiKey == "" {
log.Fatal("OPENAI_API_KEY environment variable is required")
}
// Create OpenAI service
openAiModel := os.Getenv("OPENAI_MODEL")
if openAiModel == "" {
openAiModel = "gpt-4o-mini" // Default to cost-effective model
}
jobRepo := repo.NewJobRepository(db)
jobDescriptionRepo := repo.NewJobDescriptionRepository(db)
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
openRouterService := openrouter.NewOpenRouterService(model, apiKey)
openAiService := openai.NewOpenAIService(openAiapiKey, openAiModel)
rmq, err := rabbitmq.NewRabbitMQClient()
if err != nil {
log.Printf("Warning: Failed to create RabbitMQ client: %v", err)
}
return &JobAnalyzer{
db: db,
jobRepo: jobRepo,
jobDescriptionRepo: jobDescriptionRepo,
jobAnalysisResultRepo: jobAnalysisResultRepo,
openRouterService: openRouterService,
rmq: rmq,
openAiService: openAiService,
}
}
func (ja *JobAnalyzer) AnalyzeJobs() error {
// Get jobs that need analysis (status = created)
jobs, err := ja.jobRepo.GetJobsByStatus(domain.JobStatusCreated)
if err != nil {
return err
}
log.Printf("Found %d jobs to analyze", len(jobs))
cv, err := os.ReadFile("cv.txt")
if err != nil {
return err
}
for _, job := range jobs {
if err := ja.analyzeJob(job, string(cv)); err != nil {
log.Printf("Error analyzing job %d: %v", job.ID, err)
// Update job status to error
if updateErr := ja.jobRepo.UpdateJobStatus(job.ID, domain.JobStatusError); updateErr != nil {
log.Printf("Error updating job status to error: %v", updateErr)
}
continue
}
// Update job status to analyzed
if err := ja.jobRepo.UpdateJobStatus(job.ID, domain.JobStatusAnalyzed); err != nil {
log.Printf("Error updating job status to analyzed: %v", err)
}
log.Printf("Successfully analyzed job %d", job.ID)
}
return nil
}
func (ja *JobAnalyzer) analyzeJob(job domain.Job, cv string) error {
// Get job description
jobDescription, jobCriteria, err := ja.jobDescriptionRepo.GetJobDescriptionByJobID(job.ID)
if err != nil {
return err
}
// Analyze job with CV
result, err := ja.openAiService.AnalyzeJobDescription(cv, domain.JobDescription{
JobID: job.ID,
Description: jobDescription,
Criteria: jobCriteria,
})
if err != nil {
return err
}
log.Printf("Analysis result for job %d: Recommendation=%s, Score=%d", job.ID, result.Recommendation, result.ConfidenceScore)
// Save analysis result to database
analysisResult := &domain.JobAnalysisResult{
JobID: job.ID,
AnalysisResult: result.Summary,
MatchScore: &result.ConfidenceScore,
KeySkills: result.MatchingSkills,
MissingSkills: result.MissingSkills,
Recommendations: &result.Recommendation,
}
if err := ja.jobAnalysisResultRepo.SaveAnalysisResult(analysisResult); err != nil {
log.Printf("Warning: Failed to save analysis result to database: %v", err)
} else {
log.Printf("Successfully saved analysis result for job %d to database", job.ID)
}
// Publish CV analysis message to RabbitMQ if available
if ja.rmq != nil {
message := rabbitmq.CvAnalyzeMessage{
JobID: job.ID,
}
if err := ja.publishCvAnalysisMessage(message); err != nil {
log.Printf("Warning: Failed to publish CV analysis message: %v", err)
}
}
return nil
}
func (ja *JobAnalyzer) publishCvAnalysisMessage(message rabbitmq.CvAnalyzeMessage) error {
data, err := json.Marshal(message)
if err != nil {
return err
}
return ja.rmq.Publish(rabbitmq.CvAnalyzeQueue, rabbitmq.CvAnalyzeExchange, data)
}

66
docker-compose.yml Normal file
View file

@ -0,0 +1,66 @@
version: '3.8'
services:
postgres:
image: postgres:15-alpine
environment:
POSTGRES_DB: jobs_scraper
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
rabbitmq:
image: rabbitmq:3-management-alpine
environment:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQGABCDEFGHIJKLMNOPQRST"
ports:
- "5672:5672"
- "15672:15672"
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "ping"]
interval: 10s
timeout: 5s
retries: 5
redis:
image: redis:7-alpine
ports:
- "6379:6379"
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 10s
timeout: 5s
retries: 5
scraper-playwright:
build:
context: ./services/scraper-playwright
dockerfile: Dockerfile
environment:
REDIS_HOST: redis
REDIS_PORT: 6379
WORKER_ID: playwright_1
LOG_LEVEL: INFO
depends_on:
redis:
condition: service_healthy
restart: unless-stopped
volumes:
postgres_data:
rabbitmq_data:
redis_data:

5
go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper
go 1.24.0
toolchain go1.24.7

0
go.sum Normal file
View file

17
go.work Normal file
View file

@ -0,0 +1,17 @@
go 1.24.0
use (
.
./apps/cron-analyzer
./internal/pkg/domain
./internal/pkg/infrastructure
./internal/pkg/openai
./internal/pkg/openrouter
./internal/pkg/utils
./libs/ports
./libs/repo
./libs/server
./services/api
./services/scraper-google
./services/scraper-linkedin
)

247
go.work.sum Normal file
View file

@ -0,0 +1,247 @@
cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg=
cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4=
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s=
cloud.google.com/go/spanner v1.56.0/go.mod h1:DndqtUKQAt3VLuV2Le+9Y3WTnq5cNKrnLb/Piqcj+h0=
cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY=
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4=
github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest/adal v0.9.16/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A=
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/aws/aws-sdk-go v1.49.6/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU=
github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33/go.mod h1:84XgODVR8uRhmOnUkKGUZKqIMxmjmLOR8Uyp7G/TPwc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI=
github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo=
github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8=
github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk=
github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM=
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds=
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw=
github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/microsoft/go-mssqldb v1.0.0/go.mod h1:+4wZTUnz/SV6nffv+RRRB/ss8jPng5Sho2SmM1l2ts4=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
github.com/mutecomm/go-sqlcipher/v4 v4.4.0/go.mod h1:PyN04SaWalavxRGH9E8ZftG6Ju7rsPrGmQRjrEaVpiY=
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/snowflakedb/gosnowflake v1.6.19/go.mod h1:FM1+PWUdwB9udFDsXdfD58NONC0m+MlOSmQRvimobSM=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI=
go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
golang.org/x/telemetry v0.0.0-20250807160809-1a19826ec488/go.mod h1:fGb/2+tgXXjhjHsTNdVEEMZNWA0quBnfrO+AfoDSAKw=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg=
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s=
google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI=
modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo=
modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8=
modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw=
modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8=
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM=
modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY=
modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k=
modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

View file

@ -0,0 +1 @@
DROP TABLE IF EXISTS jobs;

View file

@ -0,0 +1,9 @@
CREATE TABLE IF NOT EXISTS jobs (
id BIGSERIAL PRIMARY KEY,
title VARCHAR(255) NOT NULL,
company VARCHAR(255) NOT NULL,
company_link TEXT,
location VARCHAR(255),
job_link TEXT UNIQUE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

View file

@ -0,0 +1,12 @@
-- Drop the trigger first
DROP TRIGGER IF EXISTS trigger_update_job_descriptions_timestamp ON job_descriptions;
-- Drop the trigger function
DROP FUNCTION IF EXISTS update_job_descriptions_updated_at();
-- Drop the indexes
DROP INDEX IF EXISTS idx_job_descriptions_criteria;
DROP INDEX IF EXISTS idx_job_descriptions_job_id;
-- Drop the table (this will automatically drop any remaining dependencies)
DROP TABLE IF EXISTS job_descriptions;

View file

@ -0,0 +1,29 @@
CREATE TABLE IF NOT EXISTS job_descriptions (
id SERIAL PRIMARY KEY,
job_id BIGINT NOT NULL,
description TEXT NOT NULL,
job_criteria JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE
);
-- Create an index on job_id for faster lookups
CREATE INDEX idx_job_descriptions_job_id ON job_descriptions(job_id);
-- Create a GIN index on the job_criteria JSONB field for faster searching within the JSON data
CREATE INDEX idx_job_descriptions_criteria ON job_descriptions USING GIN (job_criteria);
-- Add trigger to update updated_at timestamp
CREATE OR REPLACE FUNCTION update_job_descriptions_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
CREATE TRIGGER trigger_update_job_descriptions_timestamp
BEFORE UPDATE ON job_descriptions
FOR EACH ROW
EXECUTE FUNCTION update_job_descriptions_updated_at();

View file

@ -0,0 +1,2 @@
-- Remove unique constraint from job_id column in job_descriptions table
ALTER TABLE job_descriptions DROP CONSTRAINT IF EXISTS unique_job_id;

View file

@ -0,0 +1,2 @@
-- Add unique constraint to job_id column in job_descriptions table
ALTER TABLE job_descriptions ADD CONSTRAINT unique_job_id UNIQUE (job_id);

View file

@ -0,0 +1 @@
ALTER TABLE jobs DROP CONSTRAINT unique_job_link;

View file

@ -0,0 +1 @@
ALTER TABLE jobs ADD CONSTRAINT unique_job_link UNIQUE (job_link);

View file

@ -0,0 +1 @@
ALTER TABLE jobs DROP COLUMN IF EXISTS provider;

View file

@ -0,0 +1 @@
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS provider INTEGER DEFAULT 0;

View file

@ -0,0 +1 @@
ALTER TABLE jobs DROP COLUMN IF EXISTS job_timestamp;

View file

@ -0,0 +1 @@
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS job_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP;

View file

@ -0,0 +1 @@
ALTER TABLE jobs DROP COLUMN status;

View file

@ -0,0 +1 @@
ALTER TABLE jobs ADD COLUMN status INTEGER DEFAULT 0;

View file

@ -0,0 +1,3 @@
DROP INDEX IF EXISTS idx_job_analysis_results_match_score;
DROP INDEX IF EXISTS idx_job_analysis_results_job_id;
DROP TABLE IF EXISTS job_analysis_results;

View file

@ -0,0 +1,14 @@
CREATE TABLE IF NOT EXISTS job_analysis_results (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT NOT NULL,
analysis_result TEXT NOT NULL,
match_score INTEGER,
key_skills TEXT[],
missing_skills TEXT[],
recommendations TEXT,
analyzed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE
);
CREATE INDEX idx_job_analysis_results_job_id ON job_analysis_results(job_id);
CREATE INDEX idx_job_analysis_results_match_score ON job_analysis_results(match_score);

View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/internal/pkg/domain
go 1.24.0
toolchain go1.24.7

View file

@ -0,0 +1,9 @@
package domain
type GoogleLink struct {
ID int64
Title string
Description string
Link string
CompanyName string
}

View file

@ -0,0 +1,7 @@
package domain
type JobDescription struct {
JobID int64
Description string
Criteria map[string]string
}

View file

@ -0,0 +1,6 @@
package domain
type JobWithDescription struct {
Job Job
JobDescription JobDescription
}

View file

@ -0,0 +1,54 @@
package domain
import "time"
// JobProvider represents different job board providers
type JobProvider int
type JobStatus int
const (
LinkedIn JobProvider = iota
Indeed
Glassdoor
Bayt
TokyoDev
JapanDev
Google
)
const (
JobStatusCreated JobStatus = iota
JobStatusAnalyzed
JobStatusError
)
type Job struct {
ID int64
Title string
Company string
CompanyLink string
Location string
JobLink string
Provider JobProvider
JobPostTime *time.Time
Status JobStatus
}
type JobAnalysisResult struct {
ID int64
JobID int64
AnalysisResult string
MatchScore *int
KeySkills []string
MissingSkills []string
Recommendations *string
AnalyzedAt time.Time
}
type SearchQuery struct {
Keywords string `json:"keywords"`
Location string `json:"location"`
NumPages int `json:"numPages"`
FWT string `json:"f_WT"` // Work type filter (1=onsite, 2=remote, 3=hybrid)
}

View file

@ -0,0 +1,173 @@
package infrastructure
import (
"database/sql"
"errors"
"fmt"
"os"
"path/filepath"
"runtime"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/postgres"
_ "github.com/golang-migrate/migrate/v4/source/file"
_ "github.com/lib/pq"
)
// Config holds database configuration
type Config struct {
Host string
Port string
User string
Password string
DBName string
SSLMode string
}
// NewConnection creates a new database connection
func NewConnection(config Config) (*sql.DB, error) {
dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
config.Host, config.Port, config.User, config.Password, config.DBName, config.SSLMode)
db, err := sql.Open("postgres", dsn)
if err != nil {
return nil, fmt.Errorf("failed to open database connection: %w", err)
}
if err := db.Ping(); err != nil {
return nil, fmt.Errorf("failed to ping database: %w", err)
}
return db, nil
}
// LoadConfigFromEnv loads database configuration from environment variables
func LoadConfigFromEnv() Config {
return Config{
Host: getEnv("DB_HOST", "localhost"),
Port: getEnv("DB_PORT", "5432"),
User: getEnv("DB_USER", "postgres"),
Password: getEnv("DB_PASSWORD", "password"),
DBName: getEnv("DB_NAME", "linkedin_jobs"),
SSLMode: getEnv("DB_SSLMODE", "disable"),
}
}
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
}
return defaultValue
}
// RunMigrations runs all pending migrations
func RunMigrations(db *sql.DB) error {
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return fmt.Errorf("failed to create postgres driver: %w", err)
}
// Get the migrations directory path
migrationsPath, err := getMigrationsPath()
if err != nil {
return fmt.Errorf("failed to get migrations path: %w", err)
}
m, err := migrate.NewWithDatabaseInstance(
fmt.Sprintf("file://%s", migrationsPath),
"postgres",
driver,
)
if err != nil {
return fmt.Errorf("failed to create migrate instance: %w", err)
}
if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return fmt.Errorf("failed to run migrations: %w", err)
}
return nil
}
// RollbackMigrations rolls back migrations by the specified number of steps
func RollbackMigrations(db *sql.DB, steps int) error {
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return fmt.Errorf("failed to create postgres driver: %w", err)
}
migrationsPath, err := getMigrationsPath()
if err != nil {
return fmt.Errorf("failed to get migrations path: %w", err)
}
m, err := migrate.NewWithDatabaseInstance(
fmt.Sprintf("file://%s", migrationsPath),
"postgres",
driver,
)
if err != nil {
return fmt.Errorf("failed to create migrate instance: %w", err)
}
defer m.Close()
if err := m.Steps(-steps); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return fmt.Errorf("failed to rollback migrations: %w", err)
}
return nil
}
// GetMigrationVersion returns the current migration version
func GetMigrationVersion(db *sql.DB) (uint, bool, error) {
driver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return 0, false, fmt.Errorf("failed to create postgres driver: %w", err)
}
migrationsPath, err := getMigrationsPath()
if err != nil {
return 0, false, fmt.Errorf("failed to get migrations path: %w", err)
}
m, err := migrate.NewWithDatabaseInstance(
fmt.Sprintf("file://%s", migrationsPath),
"postgres",
driver,
)
if err != nil {
return 0, false, fmt.Errorf("failed to create migrate instance: %w", err)
}
defer m.Close()
version, dirty, err := m.Version()
if err != nil {
if errors.Is(err, migrate.ErrNilVersion) {
return 0, false, nil
}
return 0, false, fmt.Errorf("failed to get migration version: %w", err)
}
return version, dirty, nil
}
func getMigrationsPath() (string, error) {
// Get the directory of the current source file
_, filename, _, ok := runtime.Caller(0)
if !ok {
return "", fmt.Errorf("failed to get current file path")
}
// Get the internal directory (go up from internal/pkg/infrastructure/ to internal/)
// Current path: .../internal/pkg/infrastructure/db.go
// Need to go up: infrastructure -> pkg -> internal
internalDir := filepath.Dir(filepath.Dir(filepath.Dir(filename)))
migrationsPath := filepath.Join(internalDir, "migrations")
// Verify the migrations directory exists
if _, err := os.Stat(migrationsPath); err != nil {
return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err)
}
return migrationsPath, nil
}

View file

@ -0,0 +1,27 @@
module github.com/jobs-scraper/internal/pkg/infrastructure
go 1.24.0
toolchain go1.24.7
require (
github.com/golang-migrate/migrate/v4 v4.19.0
github.com/gorilla/mux v1.8.1
github.com/jobs-scraper/libs/ports v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0
github.com/jobs-scraper/internal/pkg/utils v0.0.0
github.com/lib/pq v1.10.9
github.com/rabbitmq/amqp091-go v1.10.0
)
replace github.com/jobs-scraper/libs/repo => ../../../libs/repo
replace github.com/jobs-scraper/internal/pkg/utils => ../utils
replace github.com/jobs-scraper/libs/ports => ../../../libs/ports
require (
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
golang.org/x/sys v0.36.0 // indirect
)

View file

@ -0,0 +1,71 @@
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -0,0 +1,148 @@
package http
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"strconv"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/internal/pkg/utils"
"github.com/gorilla/mux"
)
// JobHandler handles HTTP requests related to jobs
type JobHandler struct {
jobCommands *ports.JobCommands
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
}
// NewJobHandler creates a new instance of JobHandler
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler {
return &JobHandler{
jobCommands: jobCommands,
jobAnalysisResultRepo: jobAnalysisResultRepo,
}
}
// RegisterRoutes registers all routes to the router
func (h *JobHandler) RegisterRoutes(router *mux.Router) {
jobs := router.PathPrefix("/jobs").Subrouter()
//jobs.Use(AuthMiddleware)
jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST")
jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET")
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
}
// CreateJob handles the creation of a new job
// @Summary Create a new job
// @Description Creates a new job with the provided specifications
// @Tags jobs
// @Accept json
// @Produce json
// @Param job body ports.CreateJobCommand true "Job creation data"
// @Success 201 {object} map[string]string "Successfully created job"
// @Failure 400 {object} map[string]string "Invalid request data"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs [post]
func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
var cmd ports.CreateJobCommand
if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON())
}
err := h.jobCommands.CreateJob.Handle(cmd)
if err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusInternalServerError, err)
}
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
return nil
}
// GetAllAnalysisResults handles retrieving all job analysis results
// @Summary Get all job analysis results
// @Description Retrieves all job analysis results from the database
// @Tags jobs
// @Produce json
// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/analysis [get]
func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error {
results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults()
if err != nil {
slog.Error("Failed to get analysis results", "error", err)
return utils.NewAPIError(http.StatusInternalServerError, err)
}
utils.WriteJSON(w, http.StatusOK, results)
return nil
}
// GetJobAnalysisResult handles retrieving analysis result for a specific job
// @Summary Get job analysis result by job ID
// @Description Retrieves the analysis result for a specific job
// @Tags jobs
// @Produce json
// @Param id path int true "Job ID"
// @Success 200 {object} domain.JobAnalysisResult "Job analysis result"
// @Failure 400 {object} map[string]string "Invalid job ID"
// @Failure 404 {object} map[string]string "Analysis result not found"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/{id}/analysis [get]
func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error {
vars := mux.Vars(r)
jobID, err := strconv.ParseInt(vars["id"], 10, 64)
if err != nil {
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
}
result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID)
if err != nil {
slog.Error("Failed to get analysis result", "jobID", jobID, "error", err)
return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found"))
}
utils.WriteJSON(w, http.StatusOK, result)
return nil
}
// GetTopMatches handles retrieving top matching jobs based on analysis score
// @Summary Get top matching jobs
// @Description Retrieves jobs with high analysis match scores
// @Tags jobs
// @Produce json
// @Param min_score query int false "Minimum match score (default: 70)"
// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results"
// @Failure 400 {object} map[string]string "Invalid minimum score"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/analysis/top-matches [get]
func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error {
minScoreStr := r.URL.Query().Get("min_score")
minScore := 70 // default minimum score
if minScoreStr != "" {
var err error
minScore, err = strconv.Atoi(minScoreStr)
if err != nil {
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score"))
}
}
results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore)
if err != nil {
slog.Error("Failed to get top matches", "minScore", minScore, "error", err)
return utils.NewAPIError(http.StatusInternalServerError, err)
}
utils.WriteJSON(w, http.StatusOK, results)
return nil
}

View file

@ -0,0 +1,143 @@
package http
import (
"context"
"encoding/json"
"log"
"net/http"
"strings"
"time"
)
// CORSMiddleware handles Cross-Origin Resource Sharing
func CORSMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Set CORS headers
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With")
w.Header().Set("Access-Control-Allow-Credentials", "true")
w.Header().Set("Access-Control-Max-Age", "86400")
// Handle preflight OPTIONS requests
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
next.ServeHTTP(w, r)
})
}
// LogsMiddleware logs HTTP requests with timing information
func LogsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
next.ServeHTTP(w, r)
log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start))
})
}
// AuthMiddleware handles JWT token validation, API keys, and basic auth
func AuthMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Skip auth for swagger endpoints and health checks
if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" {
next.ServeHTTP(w, r)
return
}
// Try Bearer token authentication first
authHeader := r.Header.Get("Authorization")
if authHeader != "" {
if strings.HasPrefix(authHeader, "Bearer ") {
token := strings.TrimPrefix(authHeader, "Bearer ")
if token != "" {
userID, err := validateJWTToken(token)
if err == nil {
ctx := context.WithValue(r.Context(), "user_id", userID)
next.ServeHTTP(w, r.WithContext(ctx))
return
}
}
}
}
// Try API Key authentication
apiKey := r.Header.Get("X-API-Key")
if apiKey != "" {
if isValidAPIKey(apiKey) {
ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey))
next.ServeHTTP(w, r.WithContext(ctx))
return
}
}
// Try Basic authentication
username, password, ok := r.BasicAuth()
if ok {
if isValidCredentials(username, password) {
ctx := context.WithValue(r.Context(), "username", username)
next.ServeHTTP(w, r.WithContext(ctx))
return
}
}
// Fallback to cookie-based auth (existing behavior)
token, err := r.Cookie("token")
if err == nil && token.Value == "123" {
next.ServeHTTP(w, r)
return
}
// No valid authentication found
response := map[string]interface{}{
"error": "Unauthorized",
"message": "Valid authentication required. Use Bearer token, API key, or basic auth.",
"code": http.StatusUnauthorized,
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
json.NewEncoder(w).Encode(response)
})
}
// Helper functions for authentication validation
func validateJWTToken(token string) (string, error) {
// TODO: Implement JWT validation logic with your preferred JWT library
// For demonstration, accept a demo token
if token == "valid-jwt-token" {
return "user-123", nil
}
return "", http.ErrAbortHandler
}
func isValidAPIKey(apiKey string) bool {
// TODO: Implement API key validation logic
// For demonstration, accept a demo API key
return apiKey == "demo-api-key-123"
}
func getUserIDFromAPIKey(apiKey string) string {
// TODO: Implement logic to get user ID from API key
// For demonstration, return a mock user ID
return "api-user-123"
}
func isValidCredentials(username, password string) bool {
// TODO: Implement credential validation logic
// For demonstration, accept demo credentials
return username == "admin" && password == "password"
}
// GetUserIDFromContext extracts user ID from request context
func GetUserIDFromContext(ctx context.Context) (string, bool) {
userID, ok := ctx.Value("user_id").(string)
return userID, ok
}
// GetUsernameFromContext extracts username from request context
func GetUsernameFromContext(ctx context.Context) (string, bool) {
username, ok := ctx.Value("username").(string)
return username, ok
}

View file

@ -0,0 +1,5 @@
package rabbitmq
type CvAnalyzeMessage struct {
JobID int64 `json:"jobID"`
}

View file

@ -0,0 +1,250 @@
package rabbitmq
import (
"fmt"
"log"
"os"
"time"
amqp "github.com/rabbitmq/amqp091-go"
)
const (
ScraperExchange = "scraper_exchange"
LinkedInQueue = "scraper.linkedin"
IndeedQueue = "scraper.indeed"
BaytQueue = "scraper.bayt"
TokyoDevQueue = "scraper.tokyodev"
GlassDoorQueue = "scraper.glassdoor"
JapanDevQueue = "scraper.japandev"
GoogleQueue = "scraper.google"
DeadLetterExchange = "scraper_dlx"
CvAnalyzeExchange = "cv_exchange"
CvAnalyzeQueue = "cv.analyze"
)
type RabbitMQClient struct {
Conn *amqp.Connection
Channel *amqp.Channel
}
type MessageHandler func([]byte) error
func NewRabbitMQClient() (*RabbitMQClient, error) {
// Get RabbitMQ URL from environment or use default
rabbitmqURL := os.Getenv("RABBITMQ_URL")
if rabbitmqURL == "" {
rabbitmqURL = "amqp://guest:guest@localhost:5672/"
}
// Connect to RabbitMQ with retry logic
var conn *amqp.Connection
var err error
for i := range 5 {
conn, err = amqp.Dial(rabbitmqURL)
if err == nil {
break
}
log.Printf("Failed to connect to RabbitMQ (attempt %d/5): %v", i+1, err)
time.Sleep(5 * time.Second)
}
if err != nil {
return nil, fmt.Errorf("error connecting to RabbitMQ after 5 attempts: %v", err)
}
// Create a channel
ch, err := conn.Channel()
if err != nil {
conn.Close()
return nil, fmt.Errorf("error creating channel: %v", err)
}
client := &RabbitMQClient{
Conn: conn,
Channel: ch,
}
// Setup exchanges and queues
if err := client.setupInfrastructure(); err != nil {
client.Close()
return nil, fmt.Errorf("error setting up infrastructure: %v", err)
}
log.Println("Successfully connected to RabbitMQ and set up infrastructure")
return client, nil
}
func (r *RabbitMQClient) setupInfrastructure() error {
// Declare dead letter exchange
err := r.Channel.ExchangeDeclare(
DeadLetterExchange,
"direct",
true, // durable
false, // auto-delete
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("error declaring dead letter exchange: %v", err)
}
// Declare main exchange
err = r.Channel.ExchangeDeclare(
ScraperExchange,
"topic",
true, // durable
false, // auto-delete
false, // internal
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("error declaring exchange: %v", err)
}
// Define queues with their routing keys
queues := map[string]string{
LinkedInQueue: "scraper.linkedin",
IndeedQueue: "scraper.indeed",
BaytQueue: "scraper.bayt",
TokyoDevQueue: "scraper.tokyodev",
JapanDevQueue: "scraper.japandev",
GlassDoorQueue: "scraper.glassdoor",
GoogleQueue: "scraper.google",
}
// Declare queues with dead letter exchange and TTL
for queueName, routingKey := range queues {
dlqName := queueName + "_dlq"
_, err := r.Channel.QueueDeclare(
dlqName,
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
return fmt.Errorf("error declaring dead letter queue %s: %v", dlqName, err)
}
err = r.Channel.QueueBind(
dlqName,
queueName, // routing key is the original queue name
DeadLetterExchange,
false,
nil,
)
if err != nil {
return fmt.Errorf("error binding dead letter queue %s: %v", dlqName, err)
}
_, err = r.Channel.QueueDeclare(
queueName,
true, // durable
false, // delete when unused
false, // exclusive
false, // no-wait
amqp.Table{
"x-dead-letter-exchange": DeadLetterExchange,
"x-dead-letter-routing-key": queueName,
"x-message-ttl": int64(24 * time.Hour / time.Millisecond), // 24 hours TTL
"x-max-retries": 3,
},
)
if err != nil {
return fmt.Errorf("error declaring queue %s: %v", queueName, err)
}
err = r.Channel.QueueBind(
queueName,
routingKey,
ScraperExchange,
false,
nil,
)
if err != nil {
return fmt.Errorf("error binding queue %s: %v", queueName, err)
}
}
return nil
}
// Publish publishes a message to a specific routing key
func (r *RabbitMQClient) Publish(routingKey, exchange string, data []byte) error {
log.Printf("Publishing to exchange=%s, routingKey=%s, data=%s", ScraperExchange, routingKey, string(data))
err := r.Channel.Publish(
ScraperExchange,
routingKey,
false,
false,
amqp.Publishing{
ContentType: "application/json",
Body: data,
DeliveryMode: amqp.Persistent,
Timestamp: time.Now().UTC(),
},
)
if err != nil {
log.Printf("Error publishing: %v", err)
return fmt.Errorf("error publishing message: %v", err)
}
return nil
}
// Subscribe creates a subscription to a specific queue with production-ready error handling
func (r *RabbitMQClient) Subscribe(queueName, consumerName string, handler MessageHandler) error {
err := r.Channel.Qos(
1, // prefetch count
0, // prefetch size
false, // global
)
if err != nil {
return fmt.Errorf("error setting QoS: %v", err)
}
// Start consuming messages
msgs, err := r.Channel.Consume(
queueName,
consumerName,
false, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
return fmt.Errorf("error starting consumer: %v", err)
}
go func() {
for msg := range msgs {
// Call the handler
err := handler(msg.Body)
if err != nil {
log.Printf("Error processing message: %v", err)
// Reject and requeue the message (will go to DLX after max retries)
msg.Nack(false, true)
} else {
msg.Ack(false)
}
}
}()
log.Printf("Successfully subscribed to queue: %s ", queueName)
return nil
}
// Close closes the RabbitMQ connection and channel
func (r *RabbitMQClient) Close() {
if r.Channel != nil {
r.Channel.Close()
}
if r.Conn != nil {
r.Conn.Close()
}
}

View file

@ -0,0 +1,157 @@
# OpenAI Service
This package provides an OpenAI-powered service for job analysis and CV generation using the official OpenAI Go SDK.
## Features
- **Job Analysis**: Analyze job descriptions against CVs to determine fit and provide recommendations
- **CV Generation**: Create tailored CVs based on job descriptions
- **Cover Letter Generation**: Generate personalized cover letters
## Setup
1. Install the OpenAI Go SDK:
```bash
go get github.com/sashabaranov/go-openai
```
2. Set your OpenAI API key as an environment variable:
```bash
export OPENAI_API_KEY="your-api-key-here"
```
3. (Optional) Set a custom base URL:
```bash
export OPENAI_BASE_URL="https://your-custom-endpoint.com/v1"
```
## Usage
### Basic Job Analysis
```go
package main
import (
"fmt"
"os"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/openai"
)
func main() {
// Create service
apiKey := os.Getenv("OPENAI_API_KEY")
service := openai.NewOpenAIService(apiKey, "gpt-4o-mini")
// Prepare job description
jobDesc := domain.JobDescription{
JobID: 1,
Description: "Looking for a Go developer...",
Criteria: map[string]string{
"experience": "5+ years",
"languages": "Go, JavaScript",
},
}
// Analyze job
result, err := service.AnalyzeJobDescription(cv, jobDesc)
if err != nil {
panic(err)
}
fmt.Printf("Should apply: %t\n", result.ShouldApply())
fmt.Printf("Confidence: %d%%\n", result.ConfidenceScore)
}
```
### Using Custom Base URL
You can configure a custom base URL in two ways:
**Method 1: Environment Variable**
```bash
export OPENAI_BASE_URL="https://api.openai-proxy.com/v1"
# Service will automatically use this URL
service := openai.NewOpenAIService(apiKey, "gpt-4o-mini")
```
**Method 2: Direct Parameter**
```go
// Use a custom base URL directly
service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4o-mini", "https://api.openai-proxy.com/v1")
// Or use Azure OpenAI
service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4", "https://your-resource.openai.azure.com/")
```
### Generate Tailored CV
```go
tailoredCV, err := service.GenerateCV(originalCV, jobDesc)
if err != nil {
panic(err)
}
fmt.Println(tailoredCV)
```
### Generate Cover Letter
```go
coverLetter, err := service.GenerateCoverLetter(cv, jobDesc, "Tech Company Inc")
if err != nil {
panic(err)
}
fmt.Println(coverLetter)
```
## Models
The service supports all OpenAI models. Common choices:
- `gpt-4o-mini` - Cost-effective, good performance (default)
- `gpt-4o` - Higher quality, more expensive
- `gpt-3.5-turbo` - Fastest, most cost-effective
## Environment Variables
- `OPENAI_API_KEY` - Your OpenAI API key (required)
- `OPENAI_BASE_URL` - Custom base URL for OpenAI API (optional)
- `OPENAI_MODEL` - Default model to use (optional)
## Response Structure
### JobAnalysisResult
```go
type JobAnalysisResult struct {
Recommendation string // "apply" or "do_not_apply"
ConfidenceScore int // 0-100
MatchingSkills []string
MissingSkills []string
ExperienceMatch string // "excellent", "good", "fair", "poor"
Summary string
ImprovementSuggestions []string
}
```
## Error Handling
The service handles common errors:
- API authentication issues
- Rate limiting
- Invalid JSON responses
- Network timeouts
Always check for errors when calling service methods.
## Testing
Run the example test:
```bash
cd internal/pkg/openai
go test -v
```
Make sure to set `OPENAI_API_KEY` environment variable before running tests that make actual API calls.

View file

@ -0,0 +1,10 @@
module github.com/jobs-scraper/internal/pkg/openai
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.41.2
)
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

View file

@ -0,0 +1,2 @@
github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM=
github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=

View file

@ -0,0 +1,27 @@
package openai
import "github.com/jobs-scraper/internal/pkg/domain"
// JobAnalysisService defines the interface for job analysis services
type JobAnalysisService interface {
// AnalyzeJobDescription analyzes a job description against a CV
AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error)
}
// CVService defines the interface for CV-related services
type CVService interface {
// GenerateCV creates a tailored CV based on a job description
GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error)
// GenerateCoverLetter creates a cover letter based on CV and job description
GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error)
}
// AIService combines all AI-related services
type AIService interface {
JobAnalysisService
CVService
}
// Ensure OpenAIService implements AIService
var _ AIService = (*OpenAIService)(nil)

View file

@ -0,0 +1,357 @@
package openai
import (
"context"
"encoding/json"
"fmt"
"log"
"os"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/sashabaranov/go-openai"
)
// JobAnalysisResult represents the structured response from job analysis
type JobAnalysisResult struct {
Recommendation string `json:"recommendation"`
ConfidenceScore int `json:"confidence_score"`
MatchingSkills []string `json:"matching_skills"`
MissingSkills []string `json:"missing_skills"`
ExperienceMatch string `json:"experience_match"`
Summary string `json:"summary"`
ImprovementSuggestions []string `json:"improvement_suggestions"`
}
// ShouldApply returns true if the recommendation is to apply for the job
func (r *JobAnalysisResult) ShouldApply() bool {
return r.Recommendation == "apply"
}
// IsHighConfidence returns true if the confidence score is 70 or above
func (r *JobAnalysisResult) IsHighConfidence() bool {
return r.ConfidenceScore >= 70
}
// OpenAIService provides job analysis using OpenAI's API
type OpenAIService struct {
client *openai.Client
model string
}
// NewOpenAIService creates a new OpenAI service instance
func NewOpenAIService(apiKey, model string) *OpenAIService {
return NewOpenAIServiceWithBaseURL(apiKey, model, "")
}
// NewOpenAIServiceWithBaseURL creates a new OpenAI service instance with a custom base URL
func NewOpenAIServiceWithBaseURL(apiKey, model, baseURL string) *OpenAIService {
if model == "" {
model = openai.GPT4oMini // Default to GPT-4o-mini for cost efficiency
}
// Create client configuration
config := openai.DefaultConfig(apiKey)
// Use provided baseURL, or check environment variable, or use default
if baseURL != "" {
config.BaseURL = baseURL
} else if envBaseURL := os.Getenv("OPENAI_BASE_URL"); envBaseURL != "" {
config.BaseURL = envBaseURL
}
client := openai.NewClientWithConfig(config)
return &OpenAIService{
client: client,
model: model,
}
}
// AnalyzeJobDescription analyzes a job description against a CV using OpenAI
func (s *OpenAIService) AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) {
// Build the user message with all the provided data
userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
2) The job shouldn't require any language skills, preferably only english.
3) The job should be remote, or provide relocation to the country.
CV:
%s
Job Description:
%s
Job Criteria (key-value):
%v
CRITICAL OUTPUT REQUIREMENTS:
- Return ONLY raw JSON - NO markdown formatting whatsoever
- NO backticks, NO code blocks, NO json prefix
- NO additional text before or after the JSON
- Start your response directly with { and end with }
- Use this exact schema and key names:
{
"recommendation": "apply" | "do_not_apply",
"confidence_score": number, // integer 0-100
"matching_skills": [string],
"missing_skills": [string],
"experience_match": "excellent" | "good" | "fair" | "poor",
"summary": string,
"improvement_suggestions": [string]
}`, cv, jobDesc.Description, jobDesc.Criteria)
// Create the chat completion request
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.",
},
{
Role: openai.ChatMessageRoleUser,
Content: userMessage,
},
},
Temperature: 0.3, // Lower temperature for more consistent responses
MaxTokens: 1000,
}
// Execute the request
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return nil, fmt.Errorf("failed to create chat completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from OpenAI API")
}
// Extract JSON from the response
jsonContent := resp.Choices[0].Message.Content
// Log the raw response for debugging
log.Printf("Raw OpenAI response: %s", jsonContent)
// Clean up markdown code blocks if present
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
// Parse the JSON response into our struct
var result JobAnalysisResult
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(200, len(jsonContent))])
}
return &result, nil
}
// GenerateCV creates a tailored CV based on a job description
func (s *OpenAIService) GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error) {
userMessage := fmt.Sprintf(`Based on the original CV and job description below, create a tailored CV that highlights relevant experience and skills for this specific position.
Original CV:
%s
Job Description:
%s
Job Criteria:
%v
Instructions:
1. Keep all factual information accurate - do not fabricate experience or skills
2. Reorganize and emphasize relevant experience and skills
3. Use keywords from the job description where appropriate
4. Maintain professional formatting
5. Focus on achievements and quantifiable results
6. Return the tailored CV in plain text format`, originalCV, jobDesc.Description, jobDesc.Criteria)
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are an expert CV writer who specializes in tailoring resumes for specific job applications. You help candidates present their existing experience and skills in the most relevant way for each position, without fabricating information.",
},
{
Role: openai.ChatMessageRoleUser,
Content: userMessage,
},
},
Temperature: 0.7, // Higher temperature for more creative writing
MaxTokens: 2000,
}
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return "", fmt.Errorf("failed to create chat completion: %v", err)
}
if len(resp.Choices) == 0 {
return "", fmt.Errorf("no response choices received from OpenAI API")
}
return resp.Choices[0].Message.Content, nil
}
// GenerateCoverLetter creates a cover letter based on CV and job description
func (s *OpenAIService) GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error) {
userMessage := fmt.Sprintf(`Create a professional cover letter based on the CV and job description below.
CV:
%s
Job Description:
%s
Job Criteria:
%v
Company Name: %s
Instructions:
1. Write a compelling opening paragraph
2. Highlight relevant experience and skills from the CV
3. Show enthusiasm for the specific role and company
4. Keep it concise (3-4 paragraphs)
5. Use a professional but engaging tone
6. Include a strong closing paragraph`, cv, jobDesc.Description, jobDesc.Criteria, companyName)
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are an expert career counselor who writes compelling cover letters that help candidates stand out while maintaining professionalism.",
},
{
Role: openai.ChatMessageRoleUser,
Content: userMessage,
},
},
Temperature: 0.8, // Higher temperature for more creative and engaging writing
MaxTokens: 1500,
}
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return "", fmt.Errorf("failed to create chat completion: %v", err)
}
if len(resp.Choices) == 0 {
return "", fmt.Errorf("no response choices received from OpenAI API")
}
return resp.Choices[0].Message.Content, nil
}
// min returns the smaller of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
// CreateJobDescription extracts structured job information from HTML content
func (s *OpenAIService) CreateJobDescription(htmlContent string) (*domain.JobDescription, error) {
prompt := fmt.Sprintf(`Extract job posting information from the following HTML content.
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON)
3. All criteria values MUST be strings (never use booleans, numbers, or arrays)
4. Properly escape all quotes inside strings using \"
5. Use only standard ASCII quotes ("), never smart quotes (" " ' ')
Required fields:
- description: Full job description as a single string
- criteria: Object with keys like title, company, location, salary, skills, experience, job_type, remote
Example (SINGLE LINE):
{"description":"Full job description text here...","criteria":{"title":"Software Engineer","location":"City, Country","salary":"$100k-$150k","skills":"Go, Python, Docker","experience":"3-5 years","job_type":"Full-time","remote":"Remote"}}
HTML Content:
%s`, htmlContent)
req := openai.ChatCompletionRequest{
Model: s.model,
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
Content: "You are a JSON extraction bot. You ONLY output valid single-line JSON. No markdown. No code blocks. No explanations. No pretty printing. Just raw minified JSON starting with { and ending with }. Always use standard ASCII double quotes (\"), never smart quotes.",
},
{
Role: openai.ChatMessageRoleUser,
Content: prompt,
},
},
Temperature: 0.1, // Low temperature for consistent structured output
MaxTokens: 2000,
}
resp, err := s.client.CreateChatCompletion(context.Background(), req)
if err != nil {
return nil, fmt.Errorf("failed to execute completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from API")
}
jsonContent := resp.Choices[0].Message.Content
log.Printf("Raw API response for job description: %s", jsonContent)
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
log.Printf("Cleaned JSON (first 500 chars): %s", jsonContent[:min(500, len(jsonContent))])
var result struct {
Description string `json:"description"`
Criteria map[string]string `json:"criteria"`
}
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
if syntaxErr, ok := err.(*json.SyntaxError); ok {
start := int(syntaxErr.Offset) - 50
if start < 0 {
start = 0
}
end := int(syntaxErr.Offset) + 50
if end > len(jsonContent) {
end = len(jsonContent)
}
log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end])
}
return nil, fmt.Errorf("failed to parse JSON response: %v. Cleaned JSON length: %d", err, len(jsonContent))
}
jobDescription := &domain.JobDescription{
Description: result.Description,
Criteria: result.Criteria,
}
return jobDescription, nil
}
// cleanMarkdownCodeBlocks removes markdown code block formatting from JSON response
func cleanMarkdownCodeBlocks(content string) string {
// Remove ```json prefix and ``` suffix if present
content = strings.TrimSpace(content)
// Remove opening code block markers
if strings.HasPrefix(content, "```json") {
content = strings.TrimPrefix(content, "```json")
} else if strings.HasPrefix(content, "```") {
content = strings.TrimPrefix(content, "```")
}
// Remove closing code block markers
content = strings.TrimSuffix(content, "```")
return strings.TrimSpace(content)
}

View file

@ -0,0 +1,12 @@
module github.com/jobs-scraper/internal/pkg/openrouter
go 1.24.0
require (
github.com/eduardolat/openroutergo v0.1.0
github.com/jobs-scraper/internal/pkg/domain v0.0.0
)
require github.com/orsinium-labs/enum v1.4.0 // indirect
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

View file

@ -0,0 +1,6 @@
github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI=
github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY=
github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ=
github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU=
github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM=
github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc=

View file

@ -0,0 +1,227 @@
package openrouter
import (
"encoding/json"
"fmt"
"log"
"strings"
"github.com/eduardolat/openroutergo"
"github.com/jobs-scraper/internal/pkg/domain"
)
// JobAnalysisResult represents the structured response from job analysis
type JobAnalysisResult struct {
Recommendation string `json:"recommendation"`
ConfidenceScore int `json:"confidence_score"`
MatchingSkills []string `json:"matching_skills"`
MissingSkills []string `json:"missing_skills"`
ExperienceMatch string `json:"experience_match"`
Summary string `json:"summary"`
ImprovementSuggestions []string `json:"improvement_suggestions"`
}
// ShouldApply returns true if the recommendation is to apply for the job
func (r *JobAnalysisResult) ShouldApply() bool {
return r.Recommendation == "apply"
}
// IsHighConfidence returns true if the confidence score is 70 or above
func (r *JobAnalysisResult) IsHighConfidence() bool {
return r.ConfidenceScore >= 70
}
type OpenRouterService struct {
model string
apiKey string
}
func NewOpenRouterService(model string, apiKey string) OpenRouterService {
return OpenRouterService{
model: model,
apiKey: apiKey,
}
}
func (s *OpenRouterService) AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) {
client, err := openroutergo.
NewClient().
WithAPIKey(s.apiKey).
Create()
if err != nil {
log.Fatalf("Failed to create client: %v", err)
}
// Build the user message with all the provided data
userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
2) The job shouldn't require any language skills, preferbly only english.
3) The job should be remote, or provide relocation to the country.
CV:
%s
Job Description:
%s
Job Criteria (key-value):
%v
CRITICAL OUTPUT REQUIREMENTS:
- Return ONLY raw JSON - NO markdown formatting whatsoever
- NO backticks, NO code blocks, NO json prefix
- NO additional text before or after the JSON
- Start your response directly with { and end with }
- Use this exact schema and key names:
{
"recommendation": "apply" | "do_not_apply",
"confidence_score": number, // integer 0-100
"matching_skills": [string],
"missing_skills": [string],
"experience_match": "excellent" | "good" | "fair" | "poor",
"summary": string,
"improvement_suggestions": [string]
}`, cv, jobDesc.Description, jobDesc.Criteria)
// Build and execute your request with a fluent API
_, resp, err := client.
NewChatCompletion().
WithModel(s.model).
WithSystemMessage("You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.").
WithUserMessage(userMessage).
Execute()
if err != nil {
return nil, fmt.Errorf("failed to execute completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from API")
}
// Extract JSON from the response (handle markdown code blocks)
jsonContent := resp.Choices[0].Message.Content
// Log the raw response for debugging
log.Printf("Raw API response: %s", jsonContent)
// Check if response looks like HTML/XML (common error response format)
if len(jsonContent) > 0 && jsonContent[0] == '<' {
return nil, fmt.Errorf("API returned HTML/XML instead of JSON. This usually indicates an API error, authentication issue, or rate limiting. Response: %s", jsonContent[:min(200, len(jsonContent))])
}
// Clean up markdown code blocks if present
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
// Parse the JSON response into our struct
var result JobAnalysisResult
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(200, len(jsonContent))])
}
return &result, nil
}
// min returns the smaller of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}
// cleanMarkdownCodeBlocks removes only markdown code block formatting
func cleanMarkdownCodeBlocks(content string) string {
content = strings.TrimSpace(content)
// Remove markdown code blocks
if strings.HasPrefix(content, "```json") {
content = strings.TrimPrefix(content, "```json")
} else if strings.HasPrefix(content, "```") {
content = strings.TrimPrefix(content, "```")
}
content = strings.TrimSuffix(content, "```")
return strings.TrimSpace(content)
}
func (s *OpenRouterService) CreateJobDescription(htmlContent string) (*domain.JobDescription, error) {
client, err := openroutergo.
NewClient().
WithAPIKey(s.apiKey).
Create()
if err != nil {
log.Fatalf("Failed to create client: %v", err)
}
prompt := fmt.Sprintf(`Extract job posting information from the following HTML content.
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON)
3. All criteria values MUST be strings (never use booleans, numbers, or arrays)
4. Properly escape all quotes inside strings using \"
5. Use only standard ASCII quotes ("), never smart quotes (" " ' ')
Required fields:
- description: Full job description as a single string
- criteria: Object with keys like title, company, location, salary, skills, experience, job_type, remote
Example (SINGLE LINE):
{"description":"Full job description text here...","criteria":{"title":"Software Engineer","location":"City, Country","salary":"$100k-$150k","skills":"Go, Python, Docker","experience":"3-5 years","job_type":"Full-time","remote":"Remote"}}
HTML Content:
%s`, htmlContent)
_, resp, err := client.
NewChatCompletion().
WithModel(s.model).
WithSystemMessage("You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.").
WithUserMessage(prompt).
Execute()
if err != nil {
return nil, fmt.Errorf("failed to execute completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from API")
}
jsonContent := resp.Choices[0].Message.Content
log.Printf("Raw API response for job description: %s", jsonContent)
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
log.Printf("Cleaned JSON (first 500 chars): %s", jsonContent[:min(500, len(jsonContent))])
var result struct {
Description string `json:"description"`
Criteria map[string]string `json:"criteria"`
}
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
if syntaxErr, ok := err.(*json.SyntaxError); ok {
start := int(syntaxErr.Offset) - 50
if start < 0 {
start = 0
}
end := int(syntaxErr.Offset) + 50
if end > len(jsonContent) {
end = len(jsonContent)
}
log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end])
}
}
jobDescription := &domain.JobDescription{
Description: result.Description,
Criteria: result.Criteria,
}
return jobDescription, nil
}
// func (s *OpenRouterService) CreateCV(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) {
// }

View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/internal/pkg/utils
go 1.24.0
toolchain go1.24.7

View file

@ -0,0 +1,87 @@
package utils
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
)
// APIError represents an API error with status code and message
type APIError struct {
StatusCode int `json:"statusCode"`
Msg any `json:"msg"`
}
// Error implements the error interface for APIError
func (e APIError) Error() string {
return fmt.Sprintf("api error: %d", e.StatusCode)
}
// NewAPIError creates a new APIError with the given status code and error message
func NewAPIError(statusCode int, err error) APIError {
return APIError{
StatusCode: statusCode,
Msg: err.Error(),
}
}
// InvalidRequestData creates an APIError for validation errors with a map of field errors
func InvalidRequestData(msg string) APIError {
return APIError{
StatusCode: http.StatusUnprocessableEntity,
Msg: msg,
}
}
// InvalidJSON creates an APIError for invalid JSON format
func InvalidJSON() APIError {
return NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid JSON request data"))
}
// APIResponse represents a standardized API response wrapper
type APIResponse struct {
Success bool `json:"success"`
Data any `json:"data,omitempty"`
Error any `json:"error,omitempty"`
}
// WriteJSON writes the response as JSON with given status code, wrapped in a data object
func WriteJSON(w http.ResponseWriter, status int, data any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
response := APIResponse{
Success: status >= 200 && status < 300,
}
if response.Success {
response.Data = data
} else {
response.Error = data
}
json.NewEncoder(w).Encode(response)
}
// APIFunc is the signature for API handler functions
type APIFunc func(w http.ResponseWriter, r *http.Request) error
// Make wraps an APIFunc and handles errors consistently
func Make(h APIFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if err := h(w, r); err != nil {
if apiErr, ok := err.(APIError); ok {
WriteJSON(w, apiErr.StatusCode, apiErr)
} else {
errResp := map[string]any{
"statusCode": http.StatusInternalServerError,
"msg": "internal server error",
}
WriteJSON(w, http.StatusInternalServerError, errResp)
// You could add logging here
slog.Error("HTTP API error", "err", err.Error(), "path", r.URL.Path)
}
}
}
}

View file

@ -0,0 +1,425 @@
package utils
import (
"context"
// "crypto/tls"
"fmt"
"io"
"net/http"
// "net/url"
"time"
)
type RetryConfig struct {
MaxRetries int
BaseDelay time.Duration
MaxDelay time.Duration
}
type RetryableHTTPRequestImpl struct {
client *http.Client
config RetryConfig
}
func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl {
return NewRetryableHTTPRequestWithProxy(config, "")
}
func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl {
// var transport *http.Transport
// if proxyURL != "" {
// proxyUrl, err := url.Parse(proxyURL)
// if err != nil {
// panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err))
// }
// transport = &http.Transport{
// Proxy: http.ProxyURL(proxyUrl),
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// }
// }
client := &http.Client{
Timeout: 30 * time.Second,
// Transport: transport,
}
return &RetryableHTTPRequestImpl{
client: client,
config: config,
}
}
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
var lastErr error
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
for _, header := range headers {
req.Header.Set(header.Get("Key"), header.Get("Value"))
}
resp, err := s.client.Do(req)
if err != nil {
lastErr = err
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
// Success
return resp, nil
} else if resp.StatusCode == http.StatusTooManyRequests {
// 429 Too Many Requests - retry with backoff
resp.Body.Close()
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
} else {
// All other errors (4xx, 5xx) - don't retry
resp.Body.Close()
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
}
// max attempts for now = 3
if attempt < s.config.MaxRetries {
// Increment delay by 2 seconds for each attempt
delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ...
// Cap the delay to MaxDelay if set
if s.config.MaxDelay > 0 && delay > s.config.MaxDelay {
delay = s.config.MaxDelay
}
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries)
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
// Continue to next attempt
}
}
}
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
}
// 5.10.245.81:80
// 172.67.70.206:80
// 190.93.245.33:80
// 185.221.160.176:80
// 141.101.123.156:80
// 103.21.244.129:80
// 103.169.142.59:80
// 172.67.81.185:80
// 37.18.73.60:5566
// 209.38.83.56:1088
// 89.169.36.109:1080
// 103.21.244.248:80
// 172.67.70.92:80
// 172.67.88.34:80
// 141.101.120.38:80
// 172.67.75.255:80
// 172.67.75.171:80
// 45.131.4.157:80
// 141.101.121.208:80
// 141.101.120.70:80
// 103.21.244.70:80
// 103.21.244.140:80
// 103.21.244.138:80
// 154.16.146.43:80
// 128.199.202.122:3128
// 160.153.0.28:80
// 185.170.166.31:80
// 172.67.70.95:80
// 141.101.123.91:80
// 141.101.121.81:80
// 141.101.121.71:80
// 141.101.120.224:80
// 192.111.137.35:4145
// 45.12.31.20:80
// 103.21.244.150:80
// 103.21.244.214:80
// 103.21.244.29:80
// 195.85.23.88:80
// 23.227.39.134:80
// 66.42.224.229:41679
// 172.67.73.123:80
// 79.110.200.27:8000
// 5.182.34.33:80
// 103.21.244.174:80
// 172.67.75.203:80
// 141.101.122.150:80
// 141.101.120.35:80
// 175.47.237.95:6128
// 103.21.244.185:80
// 141.101.122.86:80
// 141.101.121.89:80
// 103.21.244.105:80
// 103.21.244.234:80
// 172.67.70.20:80
// 163.223.172.27:1080
// 172.67.84.128:80
// 103.21.244.57:80
// 31.43.179.191:80
// 173.245.59.61:80
// 141.101.120.48:80
// 185.162.230.19:80
// 172.67.172.150:80
// 159.112.235.63:80
// 172.64.40.184:80
// 103.21.244.55:80
// 103.21.244.49:80
// 172.67.70.228:80
// 103.21.244.8:80
// 66.235.200.77:80
// 45.85.119.147:80
// 172.67.177.231:80
// 173.245.49.28:80
// 110.232.92.49:8080
// 212.183.88.212:80
// 108.162.193.73:80
// 185.238.228.29:80
// 172.67.127.8:80
// 164.38.155.86:80
// 160.153.0.18:80
// 141.101.122.119:80
// 103.21.244.186:80
// 172.67.172.162:80
// 8.218.39.40:10800
// 47.237.132.101:60031
// 222.59.173.105:44193
// 143.110.190.60:1080
// 152.53.194.55:32059
// 115.187.50.40:5678
// 171.254.219.180:1080
// 103.165.157.155:8080
// 45.233.169.57:999
// 222.59.173.105:45035
// 202.40.179.18:4145
// 170.79.181.188:60606
// 37.186.66.36:3629
// 104.200.152.30:4145
// 106.13.58.110:8888
// 202.40.181.220:31247
// 43.135.36.240:80
// 103.82.27.107:10001
// 192.252.209.158:4145
// 142.54.236.97:4145
// 142.54.239.1:4145
// 184.170.245.148:4145
// 68.71.247.130:4145
// 72.195.34.58:4145
// 172.65.90.2:80
// 172.65.90.0:80
// 36.138.53.26:10019
// 183.215.23.242:9091
// 45.131.7.34:80
// 141.101.120.230:80
// 172.64.78.193:80
// 141.101.122.9:80
// 103.21.244.80:80
// 103.21.244.147:80
// 103.21.244.106:80
// 103.21.244.16:80
// 199.34.229.210:80
// 141.101.90.98:80
// 172.67.70.127:80
// 103.21.244.221:80
// 172.67.171.203:80
// 172.67.181.188:80
// 72.195.101.99:4145
// 141.101.120.106:80
// 103.21.244.180:80
// 103.21.244.102:80
// 172.67.70.233:80
// 108.162.193.107:80
// 45.131.208.112:80
// 154.194.12.195:80
// 69.84.182.23:80
// 141.101.120.128:80
// 141.101.120.149:80
// 141.101.120.232:80
// 141.101.120.6:80
// 172.67.98.18:80
// 141.101.113.20:80
// 208.65.90.21:4145
// 198.177.254.131:4145
// 142.54.237.38:4145
// 192.252.220.89:4145
// 199.187.210.54:4145
// 199.102.105.242:4145
// 199.102.107.145:4145
// 198.8.84.3:4145
// 98.182.171.161:4145
// 192.111.129.150:4145
// 184.170.248.5:4145
// 98.188.47.150:4145
// 125.228.94.232:4145
// 125.228.94.153:4145
// 185.162.229.219:80
// 103.21.244.22:80
// 172.64.89.97:80
// 103.21.244.54:80
// 103.21.244.37:80
// 141.101.123.225:80
// 103.21.244.168:80
// 103.21.244.149:80
// 103.21.244.100:80
// 103.21.244.24:80
// 23.227.39.209:80
// 172.67.83.15:80
// 45.131.4.241:80
// 141.101.120.201:80
// 172.67.191.237:80
// 172.67.127.188:80
// 103.21.244.92:80
// 103.160.204.22:80
// 220.197.44.36:3128
// 39.185.41.193:5911
// 23.227.39.121:80
// 139.162.78.109:80
// 188.114.99.144:80
// 45.131.5.37:80
// 45.131.4.250:80
// 23.227.39.65:80
// 141.101.121.191:80
// 172.67.188.16:80
// 172.67.254.148:80
// 221.1.104.177:7302
// 222.59.173.105:44008
// 103.21.244.83:80
// 172.64.149.1:80
// 45.12.30.22:80
// 222.59.173.105:44027
// 172.64.149.26:80
// 103.21.244.46:80
// 45.12.31.242:80
// 23.227.38.195:80
// 172.67.177.162:80
// 103.160.204.200:80
// 172.67.70.129:80
// 141.101.123.245:80
// 185.162.230.117:80
// 185.162.230.183:80
// 69.61.200.104:36181
// 208.65.90.3:4145
// 72.223.188.92:4145
// 192.252.214.17:4145
// 68.71.242.118:4145
// 192.252.210.233:4145
// 107.181.161.81:4145
// 107.181.168.145:4145
// 206.220.175.2:4145
// 72.37.217.3:4145
// 192.252.208.70:14282
// 70.166.167.55:57745
// 192.111.137.37:18762
// 98.188.47.132:4145
// 62.99.138.162:80
// 172.64.90.186:80
// 172.67.74.57:80
// 45.131.6.67:80
// 141.101.122.37:80
// 103.21.244.134:80
// 172.64.155.71:80
// 5.182.34.139:80
// 45.131.6.31:80
// 141.101.121.21:80
// 141.101.120.190:80
// 103.21.244.161:80
// 103.21.244.156:80
// 103.21.244.151:80
// 172.64.89.0:80
// 58.216.109.17:800
// 58.241.88.18:800
// 36.147.78.166:80
// 170.244.26.36:8888
// 170.244.25.52:8888
// 170.244.26.206:8888
// 36.138.53.26:10017
// 201.148.32.162:80
// 203.19.38.114:1080
// 31.43.179.204:80
// 45.12.31.50:80
// 170.244.27.142:8888
// 170.244.26.195:8888
// 170.244.27.58:8888
// 170.244.27.61:8888
// 170.244.27.150:8888
// 47.243.94.125:1080
// 40.177.65.8:80
// 47.57.13.107:80
// 194.158.203.14:80
// 194.219.134.234:80
// 103.21.244.51:80
// 103.21.244.160:80
// 103.21.244.144:80
// 213.33.126.130:80
// 103.21.244.69:80
// 103.21.244.192:80
// 103.21.244.133:80
// 141.193.213.189:80
// 68.71.254.6:4145
// 185.221.160.21:80
// 188.114.99.97:80
// 63.141.128.94:80
// 141.193.213.213:80
// 185.162.228.234:80
// 213.143.113.82:80
// 154.194.12.207:80
// 189.203.181.34:1080
// 202.144.134.150:5678
// 190.242.157.215:8080
// 72.49.49.11:31034
// 142.54.237.34:4145
// 68.71.249.153:48606
// 192.252.209.155:14455
// 192.252.216.86:4145
// 142.54.229.249:4145
// 209.97.150.167:8080
// 192.252.220.92:17328
// 98.178.72.21:10919
// 144.124.228.87:1080
// 183.240.46.42:80
// 32.223.6.94:80
// 192.252.214.20:15864
// 198.177.252.24:4145
// 192.252.208.67:14287
// 198.8.94.170:4145
// 68.71.241.33:4145
// 98.181.137.83:4145
// 45.12.30.139:80
// 103.21.244.97:80
// 65.1.148.157:80
// 199.58.184.97:4145
// 141.101.120.148:80
// 185.238.228.77:80
// 172.67.202.134:80
// 172.67.182.49:80
// 45.131.210.112:80
// 141.101.120.100:80
// 103.21.244.59:80
// 103.21.244.189:80
// 103.21.244.154:80
// 103.21.244.23:80
// 173.245.49.40:80
// 82.200.235.134:38191
// 43.224.118.89:2626
// 119.148.47.226:16464
// 211.230.49.122:3128
// 62.171.159.232:8888
// 115.127.112.34:1080
// 185.191.236.162:3128
// 41.223.119.156:3128
// 103.138.123.242:8082
// 121.169.46.116:1090
// 185.145.185.218:8080
// 45.166.93.113:999
// 163.53.204.178:9813
// 181.224.226.154:8080

9
libs/ports/go.mod Normal file
View file

@ -0,0 +1,9 @@
module github.com/jobs-scraper/libs/ports
go 1.24.0
toolchain go1.24.7
require github.com/jobs-scraper/internal/pkg/domain v0.0.0
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain

View file

@ -0,0 +1,21 @@
package ports
import "github.com/jobs-scraper/internal/pkg/domain"
type JobCommands struct {
CreateJob JobCommandHandler
}
// CreateJobCommand represents the command to create a job
type CreateJobCommand struct {
Location string
Keywords string
FWT string
Provider domain.JobProvider
NumPages int
}
// JobCommandHandler defines the interface for handling job commands
type JobCommandHandler interface {
Handle(cmd CreateJobCommand) error
}

12
libs/repo/go.mod Normal file
View file

@ -0,0 +1,12 @@
module github.com/jobs-scraper/libs/repo
go 1.24.0
toolchain go1.24.7
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/lib/pq v1.10.9
)
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain

2
libs/repo/go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=

View file

@ -0,0 +1,152 @@
package repo
import (
"database/sql"
"fmt"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/lib/pq"
)
type JobAnalysisResultRepository struct {
db *sql.DB
}
func NewJobAnalysisResultRepository(db *sql.DB) *JobAnalysisResultRepository {
return &JobAnalysisResultRepository{db: db}
}
func (r *JobAnalysisResultRepository) SaveAnalysisResult(result *domain.JobAnalysisResult) error {
query := `
INSERT INTO job_analysis_results (job_id, analysis_result, match_score, key_skills, missing_skills, recommendations)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id, analyzed_at
`
err := r.db.QueryRow(
query,
result.JobID,
result.AnalysisResult,
result.MatchScore,
pq.Array(result.KeySkills),
pq.Array(result.MissingSkills),
result.Recommendations,
).Scan(&result.ID, &result.AnalyzedAt)
if err != nil {
return fmt.Errorf("error saving job analysis result: %v", err)
}
return nil
}
func (r *JobAnalysisResultRepository) GetAnalysisResultByJobID(jobID int64) (*domain.JobAnalysisResult, error) {
query := `
SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at
FROM job_analysis_results
WHERE job_id = $1
ORDER BY analyzed_at DESC
LIMIT 1
`
var result domain.JobAnalysisResult
err := r.db.QueryRow(query, jobID).Scan(
&result.ID,
&result.JobID,
&result.AnalysisResult,
&result.MatchScore,
pq.Array(&result.KeySkills),
pq.Array(&result.MissingSkills),
&result.Recommendations,
&result.AnalyzedAt,
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("no analysis result found for job ID %d", jobID)
}
if err != nil {
return nil, fmt.Errorf("error querying job analysis result: %v", err)
}
return &result, nil
}
func (r *JobAnalysisResultRepository) GetAllAnalysisResults() ([]domain.JobAnalysisResult, error) {
query := `
SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at
FROM job_analysis_results
ORDER BY analyzed_at DESC
`
rows, err := r.db.Query(query)
if err != nil {
return nil, fmt.Errorf("error querying job analysis results: %v", err)
}
defer rows.Close()
var results []domain.JobAnalysisResult
for rows.Next() {
var result domain.JobAnalysisResult
err := rows.Scan(
&result.ID,
&result.JobID,
&result.AnalysisResult,
&result.MatchScore,
pq.Array(&result.KeySkills),
pq.Array(&result.MissingSkills),
&result.Recommendations,
&result.AnalyzedAt,
)
if err != nil {
return nil, fmt.Errorf("error scanning job analysis result: %v", err)
}
results = append(results, result)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err)
}
return results, nil
}
func (r *JobAnalysisResultRepository) GetAnalysisResultsByMatchScore(minScore int) ([]domain.JobAnalysisResult, error) {
query := `
SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at
FROM job_analysis_results
WHERE match_score >= $1
ORDER BY match_score DESC, analyzed_at DESC
`
rows, err := r.db.Query(query, minScore)
if err != nil {
return nil, fmt.Errorf("error querying job analysis results by match score: %v", err)
}
defer rows.Close()
var results []domain.JobAnalysisResult
for rows.Next() {
var result domain.JobAnalysisResult
err := rows.Scan(
&result.ID,
&result.JobID,
&result.AnalysisResult,
&result.MatchScore,
pq.Array(&result.KeySkills),
pq.Array(&result.MissingSkills),
&result.Recommendations,
&result.AnalyzedAt,
)
if err != nil {
return nil, fmt.Errorf("error scanning job analysis result: %v", err)
}
results = append(results, result)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err)
}
return results, nil
}

View file

@ -0,0 +1,87 @@
package repo
import (
"database/sql"
"encoding/json"
"fmt"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
)
type JobDescriptionRepository struct {
db *sql.DB
}
type JobDescriptionData struct {
JobID int
Description string
Criteria map[string]string
}
func NewJobDescriptionRepository(db *sql.DB) *JobDescriptionRepository {
return &JobDescriptionRepository{db: db}
}
func (r *JobDescriptionRepository) SaveJobDescriptions(jobDescriptions []domain.JobDescription) error {
if len(jobDescriptions) == 0 {
return nil
}
// Build the VALUES clause dynamically
valueStrings := make([]string, 0, len(jobDescriptions))
valueArgs := make([]interface{}, 0, len(jobDescriptions)*3)
for i, jd := range jobDescriptions {
// Convert criteria map to JSONB
criteriaByte, err := json.Marshal(jd.Criteria)
if err != nil {
return fmt.Errorf("error marshaling job criteria for job %d: %v", jd.JobID, err)
}
valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", i*3+1, i*3+2, i*3+3))
valueArgs = append(valueArgs, jd.JobID, jd.Description, criteriaByte)
}
sqlStatement := fmt.Sprintf(`
INSERT INTO job_descriptions (job_id, description, job_criteria)
VALUES %s
ON CONFLICT (job_id) DO UPDATE SET
description = EXCLUDED.description,
job_criteria = EXCLUDED.job_criteria,
updated_at = CURRENT_TIMESTAMP
`, strings.Join(valueStrings, ","))
_, err := r.db.Exec(sqlStatement, valueArgs...)
if err != nil {
return fmt.Errorf("error saving job descriptions: %v", err)
}
return nil
}
func (r *JobDescriptionRepository) GetJobDescriptionByJobID(jobID int64) (string, map[string]string, error) {
var (
description string
criteriaByte []byte
criteria map[string]string
)
sqlStatement := `SELECT description, job_criteria FROM job_descriptions WHERE job_id = $1`
err := r.db.QueryRow(sqlStatement, jobID).Scan(&description, &criteriaByte)
if err != nil {
if err == sql.ErrNoRows {
return "", nil, nil // No description found
}
return "", nil, fmt.Errorf("error fetching job description: %v", err)
}
// Unmarshal the JSON criteria
if criteriaByte != nil {
if err := json.Unmarshal(criteriaByte, &criteria); err != nil {
return "", nil, fmt.Errorf("error unmarshaling job criteria: %v", err)
}
}
return description, criteria, nil
}

180
libs/repo/job.go Normal file
View file

@ -0,0 +1,180 @@
package repo
import (
"bytes"
"database/sql"
"fmt"
"strconv"
"github.com/jobs-scraper/internal/pkg/domain"
)
type JobRepository struct {
db *sql.DB
}
func NewJobRepository(db *sql.DB) *JobRepository {
return &JobRepository{db: db}
}
func (r *JobRepository) SaveJobs(jobs []domain.Job) error {
if len(jobs) == 0 {
return nil
}
// Deduplicate jobs by ID to avoid duplicate errors
jobMap := make(map[int64]domain.Job)
for _, job := range jobs {
jobMap[job.ID] = job
}
// Convert to slice of pointers for the bulk function
uniqueJobs := make([]*domain.Job, 0, len(jobMap))
for _, job := range jobMap {
jobCopy := job // Important: create a copy to avoid pointer issues
uniqueJobs = append(uniqueJobs, &jobCopy)
}
sqlTemplate := `
INSERT INTO jobs (id, title, company, company_link, location, job_link, job_timestamp, provider)
VALUES %s
ON CONFLICT (job_link) DO UPDATE SET
id = EXCLUDED.id,
title = EXCLUDED.title,
company = EXCLUDED.company,
company_link = EXCLUDED.company_link,
location = EXCLUDED.location,
job_timestamp = EXCLUDED.job_timestamp,
provider = EXCLUDED.provider
`
sqlStatement, vals := prepareQueryCreateBulk(sqlTemplate, uniqueJobs)
_, err := r.db.Exec(sqlStatement, vals...)
if err != nil {
return fmt.Errorf("error inserting jobs: %v", err)
}
return nil
}
func (r *JobRepository) GetAllJobs() ([]domain.Job, error) {
rows, err := r.db.Query("SELECT * FROM jobs")
if err != nil {
return nil, fmt.Errorf("error querying jobs: %v", err)
}
defer rows.Close()
var jobs []domain.Job
for rows.Next() {
var job domain.Job
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil {
return nil, fmt.Errorf("error scanning job row: %v", err)
}
jobs = append(jobs, job)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating over job rows: %v", err)
}
return jobs, nil
}
func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
var job domain.Job
sqlStatement := `
SELECT id, title, company, company_link, location, job_link, job_timestamp
FROM jobs
WHERE id = $1
`
err := r.db.QueryRow(sqlStatement, id).Scan(
&job.ID,
&job.Title,
&job.Company,
&job.CompanyLink,
&job.Location,
&job.JobLink,
&job.JobPostTime,
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("job with ID %d not found", id)
}
if err != nil {
return nil, fmt.Errorf("error querying job: %v", err)
}
return &job, nil
}
func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) {
bf := bytes.Buffer{}
values := make([]interface{}, 0, len(models)*8)
for i, v := range models {
values = append(values, v.ID, v.Title, v.Company,
v.CompanyLink, v.Location, v.JobLink, v.JobPostTime, v.Provider,
)
numFields := 8
n := i * numFields
bf.WriteString("(")
for j := 0; j < numFields; j++ {
bf.WriteString("$")
bf.WriteString(strconv.Itoa(n + j + 1))
if j < numFields-1 {
bf.WriteString(", ")
}
}
bf.WriteString(")")
if i < len(models)-1 {
bf.WriteString(", ")
}
}
return fmt.Sprintf(s, bf.String()), values
}
func (r *JobRepository) GetJobsByStatus(status domain.JobStatus) ([]domain.Job, error) {
query := `
SELECT id, title, company, company_link, location, job_link, job_timestamp
FROM jobs
WHERE status = $1
`
rows, err := r.db.Query(query, int(status))
if err != nil {
return nil, err
}
defer rows.Close()
var jobs []domain.Job
for rows.Next() {
var job domain.Job
err := rows.Scan(
&job.ID,
&job.Title,
&job.Company,
&job.CompanyLink,
&job.Location,
&job.JobLink,
&job.JobPostTime,
)
if err != nil {
return nil, err
}
jobs = append(jobs, job)
}
return jobs, rows.Err()
}
func (r *JobRepository) UpdateJobStatus(jobID int64, status domain.JobStatus) error {
query := `UPDATE jobs SET status = $1 WHERE id = $2`
_, err := r.db.Exec(query, int(status), jobID)
return err
}

5
libs/server/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/libs/server
go 1.24.0
require github.com/gorilla/mux v1.8.1

2
libs/server/go.sum Normal file
View file

@ -0,0 +1,2 @@
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=

40
libs/server/server.go Normal file
View file

@ -0,0 +1,40 @@
package server
import (
"context"
http "net/http"
"time"
"github.com/gorilla/mux"
)
type AppServer struct {
router *mux.Router
httpServer *http.Server
}
func NewServer(router *mux.Router) *AppServer {
port := "8080"
httpServer := &http.Server{
Addr: ":" + port,
Handler: router,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
IdleTimeout: 60 * time.Second,
}
server := &AppServer{
router: router,
httpServer: httpServer,
}
return server
}
func (s *AppServer) Start() error {
return s.httpServer.ListenAndServe()
}
func (s *AppServer) Stop(ctx context.Context) {
s.httpServer.Shutdown(ctx)
}

View file

@ -0,0 +1,37 @@
version: '3.8'
services:
postgres:
image: postgres:15-alpine
environment:
POSTGRES_DB: jobs_scraper
POSTGRES_USER: postgres
POSTGRES_PASSWORD: password
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
rabbitmq:
image: rabbitmq:3-management-alpine
environment:
RABBITMQ_DEFAULT_USER: guest
RABBITMQ_DEFAULT_PASS: guest
ports:
- "5672:5672"
- "15672:15672"
volumes:
- rabbitmq_data:/var/lib/rabbitmq
healthcheck:
test: ["CMD", "rabbitmq-diagnostics", "ping"]
interval: 10s
timeout: 5s
retries: 5
volumes:
postgres_data:
rabbitmq_data:

12
services/api/.env Normal file
View file

@ -0,0 +1,12 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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

View file

@ -0,0 +1,103 @@
package main
import (
"context"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"github.com/gorilla/mux"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/api/internal/app"
httpHandler "github.com/jobs-scraper/services/api/pkg/http"
_ "github.com/jobs-scraper/services/api/pkg/swagger"
"github.com/joho/godotenv"
httpSwagger "github.com/swaggo/http-swagger"
)
// @title Jobs Scraper API
// @version 1.0
// @description API for managing job scraping operations
// @BasePath /
func main() {
// Try to load .local.env first, then fallback to .env
if err := godotenv.Load(".local.env"); err != nil {
log.Println("No .local.env file found, trying .env")
if err := godotenv.Load(".env"); err != nil {
log.Println("No .env file found, using system environment variables")
}
}
dbConfig := infrastructure.LoadConfigFromEnv()
db, err := infrastructure.NewConnection(dbConfig)
if err != nil {
log.Fatal("Error connecting to db")
}
err = db.Ping()
if err != nil {
log.Fatal("Error pinging db")
}
log.Println("Successfully connected to db")
rmq, err := rabbitmq.NewRabbitMQClient()
if err != nil {
log.Fatal("Failed to create RabbitMQ client")
}
defer rmq.Close()
// Run database migrations
if err := infrastructure.RunMigrations(db); err != nil {
log.Fatalf("Failed to run migrations: %v", err)
}
router := mux.NewRouter()
// Swagger endpoint
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
router.Use(httpHandler.CORSMiddleware)
router.Use(httpHandler.LogsMiddleware)
app := app.NewApplication(router, db, rmq)
// Create job analysis result repository
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo)
jobHandler.RegisterRoutes(router)
go func() {
log.Printf("Server running on http %s\n", "8080")
if err := app.Server.Start(); err != nil && err != http.ErrServerClosed {
log.Fatalf("Could not listen on http %s: %v\n", "8080", err)
}
}()
stopChan := make(chan os.Signal, 1)
signal.Notify(
stopChan,
os.Interrupt,
syscall.SIGTERM,
)
<-stopChan
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
app.Server.Stop(ctx)
log.Println("Server shutting down")
os.Exit(0)
}

47
services/api/go.mod Normal file
View file

@ -0,0 +1,47 @@
module github.com/jobs-scraper/services/api
go 1.24.0
require (
github.com/gorilla/mux v1.8.1
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/pkg/utils v0.0.0
github.com/jobs-scraper/libs/ports v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0
github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000
github.com/joho/godotenv v1.5.1
github.com/swaggo/http-swagger v1.3.4
github.com/swaggo/swag v1.16.3
)
require (
github.com/KyleBanks/depth v1.2.1 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/spec v0.20.6 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/tools v0.24.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils
replace github.com/jobs-scraper/libs/ports => ../../libs/ports
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
replace github.com/jobs-scraper/libs/server => ../../libs/server

134
services/api/go.sum Normal file
View file

@ -0,0 +1,134 @@
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA=
github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo=
github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ=
github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc=
github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww=
github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ=
github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -0,0 +1,34 @@
package app
import (
"database/sql"
"github.com/gorilla/mux"
"github.com/jobs-scraper/services/api/internal/commands/job"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/libs/server"
)
type Application struct {
Router *mux.Router
DB *sql.DB
Server *server.AppServer
JobCommands *ports.JobCommands
}
func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient) *Application {
s := server.NewServer(router)
jobRepo := repo.NewJobRepository(db)
return &Application{
Server: s,
Router: router,
DB: db,
JobCommands: &ports.JobCommands{
CreateJob: job.NewCreateJobHandler(jobRepo, rmq),
},
}
}

View file

@ -0,0 +1,61 @@
package job
import (
"encoding/json"
"errors"
"log"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
)
type CreateJob struct {
jobRepo *repo.JobRepository
rmq *rabbitmq.RabbitMQClient
}
func NewCreateJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *CreateJob {
return &CreateJob{
jobRepo: jobRepo,
rmq: rmq,
}
}
func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error {
jobRequest := domain.SearchQuery{
Keywords: cmd.Keywords,
Location: cmd.Location,
FWT: cmd.FWT,
NumPages: cmd.NumPages,
}
jsonData, err := json.Marshal(jobRequest)
if err != nil {
log.Printf("Error marshaling JSON: %v", err)
return err
}
switch cmd.Provider {
case domain.LinkedIn:
if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
log.Printf("Error publishing LinkedIn message: %v", err)
return err
}
log.Printf("Published LinkedIn job request: %s", string(jsonData))
case domain.Glassdoor:
if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
log.Printf("Error publishing Glassdoor message: %v", err)
return err
}
log.Printf("Published Glassdoor job request: %s", string(jsonData))
default:
log.Printf("Unsupported job provider: %v", cmd.Provider)
return errors.New("unsupported job provider")
}
return nil
}
// sk-24a0fbede3a8464b85e22360165a3cc9

View file

@ -0,0 +1,148 @@
package http
import (
"encoding/json"
"fmt"
"log/slog"
"net/http"
"strconv"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/internal/pkg/utils"
"github.com/gorilla/mux"
)
// JobHandler handles HTTP requests related to jobs
type JobHandler struct {
jobCommands *ports.JobCommands
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
}
// NewJobHandler creates a new instance of JobHandler
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler {
return &JobHandler{
jobCommands: jobCommands,
jobAnalysisResultRepo: jobAnalysisResultRepo,
}
}
// RegisterRoutes registers all routes to the router
func (h *JobHandler) RegisterRoutes(router *mux.Router) {
jobs := router.PathPrefix("/jobs").Subrouter()
//jobs.Use(AuthMiddleware)
jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST")
jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET")
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
}
// CreateJob handles the creation of a new job
// @Summary Create a new job
// @Description Creates a new job with the provided specifications
// @Tags jobs
// @Accept json
// @Produce json
// @Param job body ports.CreateJobCommand true "Job creation data"
// @Success 201 {object} map[string]string "Successfully created job"
// @Failure 400 {object} map[string]string "Invalid request data"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs [post]
func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
var cmd ports.CreateJobCommand
if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON())
}
err := h.jobCommands.CreateJob.Handle(cmd)
if err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusInternalServerError, err)
}
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
return nil
}
// GetAllAnalysisResults handles retrieving all job analysis results
// @Summary Get all job analysis results
// @Description Retrieves all job analysis results from the database
// @Tags jobs
// @Produce json
// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/analysis [get]
func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error {
results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults()
if err != nil {
slog.Error("Failed to get analysis results", "error", err)
return utils.NewAPIError(http.StatusInternalServerError, err)
}
utils.WriteJSON(w, http.StatusOK, results)
return nil
}
// GetJobAnalysisResult handles retrieving analysis result for a specific job
// @Summary Get job analysis result by job ID
// @Description Retrieves the analysis result for a specific job
// @Tags jobs
// @Produce json
// @Param id path int true "Job ID"
// @Success 200 {object} domain.JobAnalysisResult "Job analysis result"
// @Failure 400 {object} map[string]string "Invalid job ID"
// @Failure 404 {object} map[string]string "Analysis result not found"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/{id}/analysis [get]
func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error {
vars := mux.Vars(r)
jobID, err := strconv.ParseInt(vars["id"], 10, 64)
if err != nil {
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
}
result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID)
if err != nil {
slog.Error("Failed to get analysis result", "jobID", jobID, "error", err)
return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found"))
}
utils.WriteJSON(w, http.StatusOK, result)
return nil
}
// GetTopMatches handles retrieving top matching jobs based on analysis score
// @Summary Get top matching jobs
// @Description Retrieves jobs with high analysis match scores
// @Tags jobs
// @Produce json
// @Param min_score query int false "Minimum match score (default: 70)"
// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results"
// @Failure 400 {object} map[string]string "Invalid minimum score"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/analysis/top-matches [get]
func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error {
minScoreStr := r.URL.Query().Get("min_score")
minScore := 70 // default minimum score
if minScoreStr != "" {
var err error
minScore, err = strconv.Atoi(minScoreStr)
if err != nil {
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score"))
}
}
results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore)
if err != nil {
slog.Error("Failed to get top matches", "minScore", minScore, "error", err)
return utils.NewAPIError(http.StatusInternalServerError, err)
}
utils.WriteJSON(w, http.StatusOK, results)
return nil
}

View file

@ -0,0 +1,143 @@
package http
import (
"context"
"encoding/json"
"log"
"net/http"
"strings"
"time"
)
// CORSMiddleware handles Cross-Origin Resource Sharing
func CORSMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Set CORS headers
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With")
w.Header().Set("Access-Control-Allow-Credentials", "true")
w.Header().Set("Access-Control-Max-Age", "86400")
// Handle preflight OPTIONS requests
if r.Method == "OPTIONS" {
w.WriteHeader(http.StatusOK)
return
}
next.ServeHTTP(w, r)
})
}
// LogsMiddleware logs HTTP requests with timing information
func LogsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
next.ServeHTTP(w, r)
log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start))
})
}
// AuthMiddleware handles JWT token validation, API keys, and basic auth
func AuthMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Skip auth for swagger endpoints and health checks
if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" {
next.ServeHTTP(w, r)
return
}
// Try Bearer token authentication first
authHeader := r.Header.Get("Authorization")
if authHeader != "" {
if strings.HasPrefix(authHeader, "Bearer ") {
token := strings.TrimPrefix(authHeader, "Bearer ")
if token != "" {
userID, err := validateJWTToken(token)
if err == nil {
ctx := context.WithValue(r.Context(), "user_id", userID)
next.ServeHTTP(w, r.WithContext(ctx))
return
}
}
}
}
// Try API Key authentication
apiKey := r.Header.Get("X-API-Key")
if apiKey != "" {
if isValidAPIKey(apiKey) {
ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey))
next.ServeHTTP(w, r.WithContext(ctx))
return
}
}
// Try Basic authentication
username, password, ok := r.BasicAuth()
if ok {
if isValidCredentials(username, password) {
ctx := context.WithValue(r.Context(), "username", username)
next.ServeHTTP(w, r.WithContext(ctx))
return
}
}
// Fallback to cookie-based auth (existing behavior)
token, err := r.Cookie("token")
if err == nil && token.Value == "123" {
next.ServeHTTP(w, r)
return
}
// No valid authentication found
response := map[string]interface{}{
"error": "Unauthorized",
"message": "Valid authentication required. Use Bearer token, API key, or basic auth.",
"code": http.StatusUnauthorized,
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
json.NewEncoder(w).Encode(response)
})
}
// Helper functions for authentication validation
func validateJWTToken(token string) (string, error) {
// TODO: Implement JWT validation logic with your preferred JWT library
// For demonstration, accept a demo token
if token == "valid-jwt-token" {
return "user-123", nil
}
return "", http.ErrAbortHandler
}
func isValidAPIKey(apiKey string) bool {
// TODO: Implement API key validation logic
// For demonstration, accept a demo API key
return apiKey == "demo-api-key-123"
}
func getUserIDFromAPIKey(apiKey string) string {
// TODO: Implement logic to get user ID from API key
// For demonstration, return a mock user ID
return "api-user-123"
}
func isValidCredentials(username, password string) bool {
// TODO: Implement credential validation logic
// For demonstration, accept demo credentials
return username == "admin" && password == "password"
}
// GetUserIDFromContext extracts user ID from request context
func GetUserIDFromContext(ctx context.Context) (string, bool) {
userID, ok := ctx.Value("user_id").(string)
return userID, ok
}
// GetUsernameFromContext extracts username from request context
func GetUsernameFromContext(ctx context.Context) (string, bool) {
username, ok := ctx.Value("username").(string)
return username, ok
}

View file

@ -0,0 +1,307 @@
// Package swagger Code generated by swaggo/swag. DO NOT EDIT
package swagger
import "github.com/swaggo/swag"
const docTemplate = `{
"schemes": {{ marshal .Schemes }},
"swagger": "2.0",
"info": {
"description": "{{escape .Description}}",
"title": "{{.Title}}",
"contact": {},
"version": "{{.Version}}"
},
"host": "{{.Host}}",
"basePath": "{{.BasePath}}",
"paths": {
"/jobs": {
"post": {
"description": "Creates a new job with the provided specifications",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Create a new job",
"parameters": [
{
"description": "Job creation data",
"name": "job",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/ports.CreateJobCommand"
}
}
],
"responses": {
"201": {
"description": "Successfully created job",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/analysis": {
"get": {
"description": "Retrieves all job analysis results from the database",
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Get all job analysis results",
"responses": {
"200": {
"description": "List of job analysis results",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/domain.JobAnalysisResult"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/analysis/top-matches": {
"get": {
"description": "Retrieves jobs with high analysis match scores",
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Get top matching jobs",
"parameters": [
{
"type": "integer",
"description": "Minimum match score (default: 70)",
"name": "min_score",
"in": "query"
}
],
"responses": {
"200": {
"description": "List of top matching job analysis results",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/domain.JobAnalysisResult"
}
}
},
"400": {
"description": "Invalid minimum score",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/{id}/analysis": {
"get": {
"description": "Retrieves the analysis result for a specific job",
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Get job analysis result by job ID",
"parameters": [
{
"type": "integer",
"description": "Job ID",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "Job analysis result",
"schema": {
"$ref": "#/definitions/domain.JobAnalysisResult"
}
},
"400": {
"description": "Invalid job ID",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"404": {
"description": "Analysis result not found",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
}
},
"definitions": {
"domain.JobAnalysisResult": {
"type": "object",
"properties": {
"analysisResult": {
"type": "string"
},
"analyzedAt": {
"type": "string"
},
"id": {
"type": "integer",
"format": "int64"
},
"jobID": {
"type": "integer",
"format": "int64"
},
"keySkills": {
"type": "array",
"items": {
"type": "string"
}
},
"matchScore": {
"type": "integer"
},
"missingSkills": {
"type": "array",
"items": {
"type": "string"
}
},
"recommendations": {
"type": "string"
}
}
},
"domain.JobProvider": {
"type": "integer",
"enum": [
0,
1,
2,
3,
4,
5
],
"x-enum-varnames": [
"LinkedIn",
"Indeed",
"Glassdoor",
"Bayt",
"TokyoDev",
"JapanDev"
]
},
"ports.CreateJobCommand": {
"type": "object",
"properties": {
"fwt": {
"type": "string"
},
"keywords": {
"type": "string"
},
"location": {
"type": "string"
},
"numPages": {
"type": "integer"
},
"provider": {
"$ref": "#/definitions/domain.JobProvider"
}
}
}
}
}`
// SwaggerInfo holds exported Swagger Info so clients can modify it
var SwaggerInfo = &swag.Spec{
Version: "1.0",
Host: "",
BasePath: "/",
Schemes: []string{},
Title: "Jobs Scraper API",
Description: "API for managing job scraping operations",
InfoInstanceName: "swagger",
SwaggerTemplate: docTemplate,
LeftDelim: "{{",
RightDelim: "}}",
}
func init() {
swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo)
}

View file

@ -0,0 +1,282 @@
{
"swagger": "2.0",
"info": {
"description": "API for managing job scraping operations",
"title": "Jobs Scraper API",
"contact": {},
"version": "1.0"
},
"basePath": "/",
"paths": {
"/jobs": {
"post": {
"description": "Creates a new job with the provided specifications",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Create a new job",
"parameters": [
{
"description": "Job creation data",
"name": "job",
"in": "body",
"required": true,
"schema": {
"$ref": "#/definitions/ports.CreateJobCommand"
}
}
],
"responses": {
"201": {
"description": "Successfully created job",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/analysis": {
"get": {
"description": "Retrieves all job analysis results from the database",
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Get all job analysis results",
"responses": {
"200": {
"description": "List of job analysis results",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/domain.JobAnalysisResult"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/analysis/top-matches": {
"get": {
"description": "Retrieves jobs with high analysis match scores",
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Get top matching jobs",
"parameters": [
{
"type": "integer",
"description": "Minimum match score (default: 70)",
"name": "min_score",
"in": "query"
}
],
"responses": {
"200": {
"description": "List of top matching job analysis results",
"schema": {
"type": "array",
"items": {
"$ref": "#/definitions/domain.JobAnalysisResult"
}
}
},
"400": {
"description": "Invalid minimum score",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/{id}/analysis": {
"get": {
"description": "Retrieves the analysis result for a specific job",
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Get job analysis result by job ID",
"parameters": [
{
"type": "integer",
"description": "Job ID",
"name": "id",
"in": "path",
"required": true
}
],
"responses": {
"200": {
"description": "Job analysis result",
"schema": {
"$ref": "#/definitions/domain.JobAnalysisResult"
}
},
"400": {
"description": "Invalid job ID",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"404": {
"description": "Analysis result not found",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
}
},
"definitions": {
"domain.JobAnalysisResult": {
"type": "object",
"properties": {
"analysisResult": {
"type": "string"
},
"analyzedAt": {
"type": "string"
},
"id": {
"type": "integer",
"format": "int64"
},
"jobID": {
"type": "integer",
"format": "int64"
},
"keySkills": {
"type": "array",
"items": {
"type": "string"
}
},
"matchScore": {
"type": "integer"
},
"missingSkills": {
"type": "array",
"items": {
"type": "string"
}
},
"recommendations": {
"type": "string"
}
}
},
"domain.JobProvider": {
"type": "integer",
"enum": [
0,
1,
2,
3,
4,
5
],
"x-enum-varnames": [
"LinkedIn",
"Indeed",
"Glassdoor",
"Bayt",
"TokyoDev",
"JapanDev"
]
},
"ports.CreateJobCommand": {
"type": "object",
"properties": {
"fwt": {
"type": "string"
},
"keywords": {
"type": "string"
},
"location": {
"type": "string"
},
"numPages": {
"type": "integer"
},
"provider": {
"$ref": "#/definitions/domain.JobProvider"
}
}
}
}
}

View file

@ -0,0 +1,189 @@
basePath: /
definitions:
domain.JobAnalysisResult:
properties:
analysisResult:
type: string
analyzedAt:
type: string
id:
format: int64
type: integer
jobID:
format: int64
type: integer
keySkills:
items:
type: string
type: array
matchScore:
type: integer
missingSkills:
items:
type: string
type: array
recommendations:
type: string
type: object
domain.JobProvider:
enum:
- 0
- 1
- 2
- 3
- 4
- 5
type: integer
x-enum-varnames:
- LinkedIn
- Indeed
- Glassdoor
- Bayt
- TokyoDev
- JapanDev
ports.CreateJobCommand:
properties:
fwt:
type: string
keywords:
type: string
location:
type: string
numPages:
type: integer
provider:
$ref: '#/definitions/domain.JobProvider'
type: object
info:
contact: {}
description: API for managing job scraping operations
title: Jobs Scraper API
version: "1.0"
paths:
/jobs:
post:
consumes:
- application/json
description: Creates a new job with the provided specifications
parameters:
- description: Job creation data
in: body
name: job
required: true
schema:
$ref: '#/definitions/ports.CreateJobCommand'
produces:
- application/json
responses:
"201":
description: Successfully created job
schema:
additionalProperties:
type: string
type: object
"400":
description: Invalid request data
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal server error
schema:
additionalProperties:
type: string
type: object
summary: Create a new job
tags:
- jobs
/jobs/{id}/analysis:
get:
description: Retrieves the analysis result for a specific job
parameters:
- description: Job ID
in: path
name: id
required: true
type: integer
produces:
- application/json
responses:
"200":
description: Job analysis result
schema:
$ref: '#/definitions/domain.JobAnalysisResult'
"400":
description: Invalid job ID
schema:
additionalProperties:
type: string
type: object
"404":
description: Analysis result not found
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal server error
schema:
additionalProperties:
type: string
type: object
summary: Get job analysis result by job ID
tags:
- jobs
/jobs/analysis:
get:
description: Retrieves all job analysis results from the database
produces:
- application/json
responses:
"200":
description: List of job analysis results
schema:
items:
$ref: '#/definitions/domain.JobAnalysisResult'
type: array
"500":
description: Internal server error
schema:
additionalProperties:
type: string
type: object
summary: Get all job analysis results
tags:
- jobs
/jobs/analysis/top-matches:
get:
description: Retrieves jobs with high analysis match scores
parameters:
- description: 'Minimum match score (default: 70)'
in: query
name: min_score
type: integer
produces:
- application/json
responses:
"200":
description: List of top matching job analysis results
schema:
items:
$ref: '#/definitions/domain.JobAnalysisResult'
type: array
"400":
description: Invalid minimum score
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal server error
schema:
additionalProperties:
type: string
type: object
summary: Get top matching jobs
tags:
- jobs
swagger: "2.0"

View file

@ -0,0 +1,12 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=postgres
DB_PASSWORD=password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
CV_AI_MODEL=your_ai_model
OPENROUTER_API_KEY=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

View file

@ -0,0 +1,19 @@
import postgres from "postgres";
import dotenv from "dotenv";
if (dotenv.config({ path: ".local.env" }).error) {
console.log("No .local.env file found, trying .env");
if (dotenv.config().error) {
console.log("No .env file found, using system environment variables");
}
}
const sql = postgres({
host: process.env.DB_HOST,
port: Number(process.env.DB_PORT) || 5432,
database: process.env.DB_NAME,
username: process.env.DB_USER,
password: process.env.DB_PASSWORD,
});
export default sql;

View file

@ -0,0 +1,184 @@
import puppeteer, { Browser, Page } from "puppeteer-core";
import { ChildProcess } from "child_process";
import * as dotenv from "dotenv";
import {
ChromeVersionResponse,
launchChromeWithDebugging,
randomDelay,
waitForChromeReady,
} from "./utils/browser";
import {
RabbitMQClient,
SearchQuery,
createRabbitMQClient,
} from "./utils/rabbitmq";
import { BrowserService } from "./services/browserService";
import { saveJobs } from "./queries/save-jobs";
import { saveJobDescriptions } from "./queries/save-job-description";
if (dotenv.config({ path: ".local.env" }).error) {
console.log("No .local.env file found, trying .env");
if (dotenv.config().error) {
console.log("No .env file found, using system environment variables");
}
}
let globalBrowser: Browser | null = null;
let chrome: ChildProcess;
async function cleanup() {
console.log("Starting cleanup process...");
if (rabbitMQClient) {
console.log("Closing RabbitMQ connection");
await rabbitMQClient.close();
rabbitMQClient = null;
}
if (globalBrowser) {
console.log("Closing browser on app shutdown");
await globalBrowser.close();
globalBrowser = null;
}
if (chrome) {
chrome.kill("SIGINT");
}
console.log("Cleanup completed");
}
async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> {
chrome = launchChromeWithDebugging();
await waitForChromeReady();
const response = await fetch("http://localhost:9222/json/version", {
headers: {
Origin: "",
},
});
if (!response.ok) {
throw new Error("Failed to connect to Chrome debugging port");
}
const chromeInstance = (await response.json()) as ChromeVersionResponse;
const { webSocketDebuggerUrl } = chromeInstance;
const browser = await puppeteer.connect({
browserWSEndpoint: webSocketDebuggerUrl,
defaultViewport: null,
});
globalBrowser = browser;
console.log("Connected to browser successfully");
let pages = await browser.pages();
console.log(`Found ${pages.length} pages`);
let existingcookies = await browser.cookies();
await browser.deleteCookie(...existingcookies);
let page: Page;
if (pages.length === 0) {
console.log("No pages found, creating a new page");
page = await browser.newPage();
} else {
page = pages[0];
}
await page.setUserAgent(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
);
return { browser, page };
}
async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise<void> {
try {
const { page } = await initializeBrowser();
const browserService = new BrowserService(page);
const searchUrl = `https://www.glassdoor.com/Job/jobs.htm`;
await page.goto(searchUrl, {
waitUntil: ["networkidle2", "domcontentloaded"],
timeout: 60000,
});
await randomDelay(1500, 2500);
await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
const jobs = await browserService.extractJobListings(searchQuery.location);
if (jobs.length > 0) {
await saveJobs(jobs);
await saveJobDescriptions(jobs);
console.log(
`saved ${jobs.length} jobs from Glassdoor for query:`,
searchQuery
);
} else {
console.log("No jobs found for this search query");
}
await randomDelay(1500, 2500);
} catch (error) {
console.error("Error scraping Glassdoor jobs:", error);
throw error;
}
if (globalBrowser) {
console.log("Closing browser on app shutdown");
await globalBrowser.close();
globalBrowser = null;
}
if (chrome) {
chrome.kill("SIGINT");
}
}
let rabbitMQClient: RabbitMQClient | null = null;
async function subscribeToGlassdoorQueue(): Promise<void> {
try {
rabbitMQClient = await createRabbitMQClient();
await rabbitMQClient.subscribe(async (searchQuery: SearchQuery) => {
await scrapeGlassdoorJobs(searchQuery);
});
} catch (error) {
console.error("Failed to set up RabbitMQ subscription:", error);
process.exit(1);
}
}
async function main() {
console.log("Starting Glassdoor scraper service...");
await subscribeToGlassdoorQueue();
}
process.on("SIGINT", async () => {
console.log("Received SIGINT signal");
await cleanup();
if (chrome) {
chrome.kill("SIGINT");
}
process.exit(0);
});
process.on("SIGTERM", async () => {
console.log("Received SIGTERM signal");
await cleanup();
if (chrome) {
chrome.kill("SIGTERM");
}
process.exit(0);
});
main().catch((error) => {
console.error("Fatal error:", error);
process.exit(1);
});

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,27 @@
{
"name": "glassdoor-scraper",
"version": "1.0.0",
"description": "",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "ts-node index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"@types/amqplib": "^0.10.7",
"amqplib": "^0.10.9",
"dotenv": "^17.2.3",
"postgres": "^3.4.7",
"puppeteer-core": "^24.25.0"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,33 @@
import sql from "../db/db";
import { Job } from "../types/job";
export const saveJobDescriptions = async (jobs: Job[]) => {
if (jobs.length === 0) {
console.log("No jobs to save descriptions for.");
return;
}
try {
const jobDescriptionsForDb = jobs.map((job) => ({
job_id: job.id,
description: job.description,
job_criteria: job.criteria,
}));
await sql`
INSERT INTO public.job_descriptions ${sql(
jobDescriptionsForDb,
"job_id",
"description",
"job_criteria"
)}
ON CONFLICT (job_id) DO UPDATE SET
description = EXCLUDED.description,
job_criteria = EXCLUDED.job_criteria,
updated_at = CURRENT_TIMESTAMP
`;
console.log(`Updated descriptions for ${jobs.length} jobs.`);
} catch (error) {
console.error("Error updating job descriptions:", error);
}
};

View file

@ -0,0 +1,52 @@
import sql from "../db/db";
import { Job } from "../types/job";
export const saveJobs = async (jobs: Job[]) => {
if (jobs.length === 0) {
console.log("No jobs to save.");
return;
}
try {
const jobsForDb = jobs.map((job) => ({
id: job.id,
title: job.title,
company: job.company,
location: job.location,
company_link: job.companyLink,
job_link: job.jobLink,
provider: job.provider,
status: job.status,
job_timestamp:
job.jobPostTime instanceof Date && !isNaN(job.jobPostTime.getTime())
? job.jobPostTime
: new Date(),
}));
const result = await sql`
INSERT INTO public.jobs ${sql(
jobsForDb,
"id",
"title",
"company",
"location",
"company_link",
"job_link",
"provider",
"status",
"job_timestamp"
)}
ON CONFLICT (id) DO UPDATE SET
title = EXCLUDED.title,
company = EXCLUDED.company,
company_link = EXCLUDED.company_link,
location = EXCLUDED.location,
job_link = EXCLUDED.job_link,
job_timestamp = EXCLUDED.job_timestamp
`;
console.log(`Saved ${jobs.length} jobs to the database.`);
} catch (error) {
console.error("Error saving jobs to the database:", error);
}
};

View file

@ -0,0 +1,185 @@
import { Page } from "puppeteer-core";
import { randomDelay } from "../utils/browser";
import { Job } from "../types/job";
export class BrowserService {
private page: Page;
constructor(page: Page) {
this.page = page;
}
async searchJobs(keywords: string, location: string) {
try {
const keywordInput = await this.page.waitForSelector(
"#searchBar-jobTitle, input[name='sc.keyword'], input[placeholder*='Job title']",
{ timeout: 10000 }
);
if (keywordInput) {
await keywordInput.click({ clickCount: 3 });
await keywordInput.type(keywords, { delay: 130 });
await randomDelay(1000, 2000);
}
const locationInput = await this.page.waitForSelector(
"#searchBar-location, input[name='locKeyword'], input[placeholder*='Location']",
{ timeout: 5000 }
);
if (locationInput) {
await locationInput.click({ clickCount: 3 });
await locationInput.type(location, { delay: 130 });
await randomDelay(1000, 2000);
await locationInput.press("Enter");
await this.page.waitForNavigation({
waitUntil: "networkidle2",
timeout: 30000,
});
}
} catch (error) {
console.log(
"Search form not found, attempting to extract jobs from current page..."
);
}
}
async extractJobListings(location: string): Promise<Job[]> {
try {
await this.page.waitForSelector(
'li[data-test="jobListing"], .JobsList_jobListItem__wjTHv, [data-jobid]',
{ timeout: 10000 }
);
} catch (error) {
console.log("No job listings found on the page");
return [];
}
const jobs = await this.page.evaluate((location) => {
const jobElements = document.querySelectorAll(`
li[data-test="jobListing"],
.JobsList_jobListItem__wjTHv,
li[data-jobid],
.react-job-listing
`) as NodeListOf<HTMLElement>;
const jobs: any[] = [];
console.log("Found job elements:", jobElements.length);
jobElements.forEach((element, index) => {
try {
const jobId =
element.getAttribute("data-jobid") ||
element.getAttribute("data-brandviews")?.match(/jlid=(\d+)/)?.[1] ||
`job-${index + 1}`;
const titleElement = element.querySelector(`
a[data-test="job-title"],
.JobCard_jobTitle__GLyJ1,
.jobLink,
a[id*="job-title"]
`) as HTMLAnchorElement;
const companyElement = element.querySelector(`
.EmployerProfile_compactEmployerName__9MGcV,
[data-test="employer-name"] a,
.employerName,
.EmployerProfile_employerNameContainer__ptolz span
`) as HTMLElement;
const descriptionElement = element.querySelector(`
.JobDetails_jobDescription__uW_fK,
.JobDetails_showHidden__C_FOA,
[class*="jobDescription"]
`) as HTMLElement;
const jobPostingTimeElement = element.querySelector(`
.JobCard_listingAge__jJsuc,
[data-test="job-age"],
.listingAge
`) as HTMLElement;
const title = titleElement?.textContent?.trim() || "";
const company = companyElement?.textContent?.trim() || "";
const description = descriptionElement?.textContent?.trim() || "";
const jobPosting = jobPostingTimeElement?.textContent?.trim() || "";
const jobLink = titleElement?.href
? titleElement.href.startsWith("http")
? titleElement.href
: `https://www.glassdoor.com${titleElement.href}`
: "";
const companyLink =
companyElement?.tagName === "A"
? (companyElement as HTMLAnchorElement).href?.startsWith("http")
? (companyElement as HTMLAnchorElement).href
: `https://www.glassdoor.com${
(companyElement as HTMLAnchorElement).href
}`
: "";
let jobPostingDate = new Date();
try {
if (jobPosting.includes("d")) {
const match = jobPosting.match(/(\d+)\s*d/);
if (match) {
const daysAgo = parseInt(match[1], 10);
if (!isNaN(daysAgo) && daysAgo >= 0) {
jobPostingDate.setDate(jobPostingDate.getDate() - daysAgo);
}
}
} else if (jobPosting.includes("h")) {
const match = jobPosting.match(/(\d+)\s*h/);
if (match) {
const hoursAgo = parseInt(match[1], 10);
if (!isNaN(hoursAgo) && hoursAgo >= 0) {
jobPostingDate.setHours(jobPostingDate.getHours() - hoursAgo);
}
}
}
if (isNaN(jobPostingDate.getTime())) {
jobPostingDate = new Date();
}
} catch (error) {
console.warn(
`Failed to parse job posting date: "${jobPosting}", using current date`
);
jobPostingDate = new Date();
}
const parsedJobId = Number.parseInt(jobId, 10);
jobs.push({
title,
company,
location,
description: description,
criteria: {},
jobLink,
companyLink,
provider: 2,
id: parsedJobId,
jobPostTime: jobPostingDate.toISOString(),
status: 0,
});
} catch (error) {
console.error(`Error extracting job ${index}:`, error);
}
});
console.log("Extracted jobs:", jobs.length);
return jobs;
}, location);
console.log(`Found ${jobs.length} jobs on Glassdoor`);
return jobs.map((job) => ({
...job,
jobPostTime: new Date(job.jobPostTime),
}));
}
}

View file

@ -0,0 +1,35 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": [
"**/*.ts",
"**/*.tsx"
],
"exclude": [
"node_modules",
"dist"
]
}

View file

@ -0,0 +1,13 @@
export interface Job {
id: number;
title: string;
company: string;
location: string;
description: string;
criteria: Record<string, string>;
jobLink: string;
companyLink: string;
provider: 2;
jobPostTime: string;
status: number;
}

View file

@ -0,0 +1,139 @@
import fs from "fs";
import path from "path";
import os from "os";
import { spawn } from "child_process";
import net from "net";
export interface ChromeVersionResponse {
webSocketDebuggerUrl: string;
}
const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile");
export const getLocalISOString = (date = new Date()) => {
const offset = date.getTimezoneOffset();
const offsetAbs = Math.abs(offset);
const hours = Math.floor(offsetAbs / 60)
.toString()
.padStart(2, "0");
const minutes = (offsetAbs % 60).toString().padStart(2, "0");
const sign = offset > 0 ? "-" : "+";
return new Date(date.getTime() - offset * 60000)
.toISOString()
.replace("Z", `${sign}${hours}:${minutes}`);
};
function getChromePath(): string {
const platform = os.platform();
const paths: Record<NodeJS.Platform, string[]> = {
darwin: [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
],
win32: [
process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
process.env["PROGRAMFILES(X86)"] +
"\\Google\\Chrome\\Application\\chrome.exe",
],
linux: [
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
],
aix: [],
freebsd: [],
openbsd: [],
sunos: [],
android: [],
haiku: [],
cygwin: [],
netbsd: [],
};
const platformPaths = paths[platform] || [];
for (const path of platformPaths) {
try {
if (fs.existsSync(path)) {
return path;
}
} catch (err) {
continue;
}
}
throw new Error("Could not find Chrome/Chromium installation");
}
export function launchChromeWithDebugging() {
try {
const chromePath = getChromePath();
const args = [
"--remote-debugging-port=9222",
`--user-data-dir=${userDataDir}`,
"--remote-allow-origins=*",
"--incognito",
// "--headless",
// `--proxy-server=46.3.202.240:2120`,
// `--proxy-server=http://jN8AqiMh:Pd9FEs6N@212.193.184.228:64616`,
];
console.log(`Launching Chrome at: ${chromePath}`);
const chromeProcess = spawn(chromePath, args, {
detached: true,
stdio: "ignore",
});
chromeProcess.unref();
chromeProcess.on("error", (err) => {
console.error("Failed to start Chrome:", err);
});
chromeProcess.on("exit", (code) => {
if (code !== 0) {
console.error(`Chrome process exited with code ${code}`);
}
});
console.log("Chrome launched successfully with debugging port 9222");
return chromeProcess;
} catch (error) {
console.error("Failed to launch Chrome:", error);
process.exit(1);
}
}
export async function waitForChromeReady(port = 9222, timeout = 30000) {
const startTime = Date.now();
return new Promise((resolve, reject) => {
function check() {
const client = net.createConnection({ port }, () => {
client.end();
resolve("");
});
client.on("error", () => {
if (Date.now() - startTime > timeout) {
reject(new Error("Timeout waiting for Chrome to start"));
} else {
setTimeout(check, 500);
}
});
}
check();
});
}
export function randomDelay(min: number, max: number): Promise<void> {
const delay = Math.floor(Math.random() * (max - min) + min);
return new Promise((resolve) => setTimeout(resolve, delay));
}

View file

@ -0,0 +1,165 @@
import * as amqp from "amqplib";
export interface RabbitMQConfig {
url: string;
queueName: string;
exchangeName: string;
exchangeType: "direct" | "topic" | "fanout" | "headers";
durable: boolean;
}
export interface SearchQuery {
keywords: string;
location: string;
fwt?: string;
numPages: number;
}
export class RabbitMQClient {
private connection: amqp.ChannelModel | null = null;
private channel: amqp.Channel | null = null;
private config: RabbitMQConfig;
constructor(config: RabbitMQConfig) {
this.config = config;
}
async connect(): Promise<void> {
try {
console.log(`Connecting to RabbitMQ at: ${this.config.url}`);
this.connection = await amqp.connect(this.config.url);
console.log("Connected to RabbitMQ successfully");
if (!this.connection) {
throw new Error("Failed to establish RabbitMQ connection");
}
this.channel = await this.connection.createChannel();
if (!this.channel) {
throw new Error("Failed to create RabbitMQ channel");
}
// Ensure exchange and queue exist
await this.channel.assertExchange(
this.config.exchangeName,
this.config.exchangeType,
{ durable: this.config.durable }
);
await this.channel.assertQueue(this.config.queueName, {
durable: this.config.durable,
arguments: {
"x-dead-letter-exchange": "scraper_dlx",
"x-dead-letter-routing-key": this.config.queueName,
"x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds
"x-max-retries": 3,
},
});
await this.channel.bindQueue(
this.config.queueName,
this.config.exchangeName,
this.config.queueName
);
// Set prefetch to process one message at a time
await this.channel.prefetch(1);
// Handle connection events
this.connection.on("close", () => {
console.log("RabbitMQ connection closed");
this.connection = null;
this.channel = null;
});
this.connection.on("error", (error: Error) => {
console.error("RabbitMQ connection error:", error);
this.connection = null;
this.channel = null;
});
} catch (error) {
console.error("Failed to connect to RabbitMQ:", error);
throw error;
}
}
async subscribe(
messageHandler: (message: SearchQuery) => Promise<void>
): Promise<void> {
if (!this.channel) {
throw new Error(
"RabbitMQ channel not initialized. Call connect() first."
);
}
console.log(
`Waiting for messages from ${this.config.queueName}. To exit press CTRL+C`
);
await this.channel.consume(
this.config.queueName,
async (msg: amqp.ConsumeMessage | null) => {
if (msg && this.channel) {
try {
const messageContent = msg.content.toString();
console.log(`Received message: ${messageContent}`);
const searchQuery: SearchQuery = JSON.parse(messageContent);
console.log(`Processing Glassdoor scrape request:`, searchQuery);
// Process the message using the provided handler
await messageHandler(searchQuery);
// Acknowledge the message on success
this.channel.ack(msg);
console.log("Message processed successfully");
} catch (error) {
console.error("Error processing message:", error);
// Reject the message and don't requeue it
if (this.channel) {
this.channel.nack(msg, false, false);
}
}
}
}
);
}
async close(): Promise<void> {
try {
if (this.channel) {
await this.channel.close();
this.channel = null;
}
if (this.connection) {
await this.connection.close();
this.connection = null;
}
console.log("RabbitMQ connection closed gracefully");
} catch (error) {
console.error("Error closing RabbitMQ connection:", error);
}
}
isConnected(): boolean {
return this.connection !== null && this.channel !== null;
}
}
export function createRabbitMQConfig(): RabbitMQConfig {
return {
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor",
exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange",
exchangeType: "topic",
durable: true,
};
}
export async function createRabbitMQClient(): Promise<RabbitMQClient> {
const config = createRabbitMQConfig();
const client = new RabbitMQClient(config);
await client.connect();
return client;
}

View file

@ -0,0 +1,47 @@
# Git
.git
.gitignore
# Environment files (will be mounted or set via docker-compose)
.local.env
*.env
# Documentation
*.md
DOCKER_README.md
RECOMMENDED_MODELS.md
# Development files
.vscode
.idea
*.swp
*.swo
*~
# Build artifacts
*.exe
*.dll
*.so
*.dylib
scraper-google
# Test files
*_test.go
testdata/
# Logs
*.log
logs/
# Temporary files
tmp/
temp/
# OS files
.DS_Store
Thumbs.db
# Docker files (not needed in image)
Dockerfile
docker-compose.yml
.dockerignore

View file

@ -0,0 +1,16 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini
CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
OPENROUTER_API_KEY=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

View file

@ -0,0 +1,18 @@
# Database Configuration
DB_HOST=postgres
DB_PORT=5432
DB_NAME=jobs_scraper
DB_USER=postgres
DB_PASSWORD=postgres
# OpenRouter Configuration (for job description extraction)
OPENROUTER_API_KEY=your_openrouter_api_key_here
CV_AI_MODEL=google/gemini-flash-1.5
# Optional: OpenAI Configuration (alternative to OpenRouter)
# OPENAI_API_KEY=your_openai_api_key_here
# OPENAI_BASE_URL=https://api.openai.com/v1
# Chrome/Chromedp Configuration (usually defaults are fine)
CHROMEDP_DISABLE_GPU=true
CHROMEDP_NO_SANDBOX=true

View file

@ -0,0 +1,40 @@
# Build stage
FROM golang:1.24-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git
WORKDIR /build
# Copy go mod files
COPY go.mod go.sum ./
RUN go mod download
# Copy the entire monorepo
COPY . .
# Build the scraper-google service
WORKDIR /build/services/scraper-google
RUN CGO_ENABLED=0 GOOS=linux go build -o scraper-google .
# Runtime stage
FROM chromedp/headless-shell:latest
# Install ca-certificates for HTTPS requests
RUN apt-get update && apt-get install -y \
ca-certificates \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
# Copy the binary from builder
COPY --from=builder /build/services/scraper-google/scraper-google .
# Copy environment files if they exist (optional)
COPY --from=builder /build/services/scraper-google/.env* ./
# Expose any ports if needed (not required for this service)
# EXPOSE 8080
# Run the scraper
CMD ["./scraper-google"]

View file

@ -0,0 +1,67 @@
.PHONY: help build up down logs restart clean rebuild
help: ## Show this help message
@echo 'Usage: make [target]'
@echo ''
@echo 'Available targets:'
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
build: ## Build the Docker images
docker-compose build
up: ## Start all services
docker-compose up -d
down: ## Stop all services
docker-compose down
logs: ## View logs (use 'make logs service=scraper-google' for specific service)
@if [ -z "$(service)" ]; then \
docker-compose logs -f; \
else \
docker-compose logs -f $(service); \
fi
restart: ## Restart all services
docker-compose restart
restart-scraper: ## Restart just the scraper service
docker-compose restart scraper-google
clean: ## Stop services and remove volumes (WARNING: deletes data)
docker-compose down -v
rebuild: ## Rebuild and restart services
docker-compose down
docker-compose build --no-cache
docker-compose up -d
ps: ## Show running containers
docker-compose ps
stats: ## Show resource usage
docker stats
exec-db: ## Connect to PostgreSQL database
docker-compose exec postgres psql -U postgres -d jobs_scraper
backup-db: ## Backup the database
@mkdir -p backups
docker-compose exec postgres pg_dump -U postgres jobs_scraper > backups/backup_$(shell date +%Y%m%d_%H%M%S).sql
@echo "Database backed up to backups/backup_$(shell date +%Y%m%d_%H%M%S).sql"
restore-db: ## Restore database from backup (use 'make restore-db file=backup.sql')
@if [ -z "$(file)" ]; then \
echo "Error: Please specify backup file. Usage: make restore-db file=backup.sql"; \
exit 1; \
fi
docker-compose exec -T postgres psql -U postgres jobs_scraper < $(file)
shell-scraper: ## Open shell in scraper container
docker-compose exec scraper-google sh
dev: ## Run in development mode with live logs
docker-compose up --build
test: ## Run tests (if you have any)
docker-compose exec scraper-google go test ./...

View file

@ -0,0 +1,34 @@
package analyzer
import (
"fmt"
"os"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/openai"
// "github.com/jobs-scraper/internal/pkg/openrouter"
)
func GetJobDescription(htmlContent string) (*domain.JobDescription, error) {
// apiKey := os.Getenv("OPENROUTER_API_KEY")
// model := os.Getenv("CV_AI_MODEL")
openAiapiKey := os.Getenv("OPENAI_API_KEY")
if openAiapiKey == "" {
return nil, fmt.Errorf("OPENAI_API_KEY environment variable is required")
}
openAiModel := os.Getenv("OPENAI_MODEL")
if openAiModel == "" {
openAiModel = "gpt-4o-mini"
}
openAiService := openai.NewOpenAIService(openAiapiKey, openAiModel)
// client := openrouter.NewOpenRouterService(model, apiKey)
result, err := openAiService.CreateJobDescription(htmlContent)
if err != nil {
return nil, err
}
return result, nil
}

View file

@ -0,0 +1,49 @@
version: "3.8"
services:
scraper-google:
build:
context: ../../
dockerfile: services/scraper-google/Dockerfile
container_name: jobs-scraper-google
environment:
DB_HOST: postgres
DB_PORT: 5432
DB_NAME: jobs_scraper
DB_USER: scraper_user
DB_PASSWORD: Cocowawa_12345
# OpenRouter Configuration
OPENROUTER_API_KEY: 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
CV_AI_MODEL: alibaba/tongyi-deepresearch-30b-a3b:free
# Optional: OpenAI Configuration
OPENAI_API_KEY: sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL: https://hubai.loe.gg/v1
OPEN_AI_MODEL: gpt-4o-mini
# Chrome/Chromedp Configuration
CHROMEDP_DISABLE_GPU: true
CHROMEDP_NO_SANDBOX: true
volumes:
- ./.env:/app/.env:ro
restart: unless-stopped
networks:
- scraper-network
# Add resource limits to prevent runaway chrome instances
deploy:
resources:
limits:
memory: 2G
cpus: "2"
reservations:
memory: 512M
cpus: "0.5"
volumes:
postgres_data:
driver: local
networks:
scraper-network:
driver: bridge

Some files were not shown because too many files have changed in this diff Show more