diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..df22772 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# Git +.git +.gitignore + +# Documentation +README.md +*.md + +# IDE +.vscode +.idea + +# Logs +*.log + +# OS +.DS_Store +Thumbs.db + +# Development files +.local.env + +# Playwright +.playwright-mcp + +# Node modules (if any) +node_modules/ + +# Temporary files +*.tmp +*.temp diff --git a/.env b/.env new file mode 100644 index 0000000..89a50f1 --- /dev/null +++ b/.env @@ -0,0 +1,16 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + + +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..650d543 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,116 @@ +name: CI/CD Pipeline + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15 + env: + POSTGRES_PASSWORD: postgres + POSTGRES_DB: jobs_scraper_test + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + rabbitmq: + image: rabbitmq:3-management + ports: + - 5672:5672 + - 15672:15672 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.24' + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install dependencies + run: | + go mod download + cd services/scraper-glassdoor && npm ci + + - name: Run tests - API Service + run: | + cd services/api + go test ./... + + - name: Run tests - LinkedIn Scraper + run: | + cd services/scraper-linkedin + go test ./... + + - name: Run tests - Cron Analyzer + run: | + cd apps/cron-analyzer + go test ./... + + - name: Run tests - CV Analyzer + run: | + cd apps/cv-analyzer + go test ./... + + - name: Build all services + run: | + make build-api + make build-scraper-linkedin + make build-scraper-glassdoor + make build-cron-analyzer + make build-cv-analyzer + + docker: + needs: test + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push API service + uses: docker/build-push-action@v5 + with: + context: ./services/api + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/jobs-scraper-api:latest + + - name: Build and push LinkedIn scraper + uses: docker/build-push-action@v5 + with: + context: ./services/scraper-linkedin + push: true + tags: ${{ secrets.DOCKER_USERNAME }}/jobs-scraper-linkedin:latest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..12cede3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +# Local environment files (contains real secrets) +.local.env + +# IDE files +.vscode/settings.json +.idea/ + +# Go build artifacts +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.test +*.out +# go.work + +# Dependency directories +vendor/ + +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Logs +*.log +logs/ + +# Temporary files +*.tmp +*.temp + +node_modules +dist \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..59f05d7 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,23 @@ +stages: + - deploy + +deploy_production: + stage: deploy + image: alpine:3.20 + environment: + name: production + url: https://jobs-scraper.ai-assistant-bot.xyz + rules: + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + when: manual + allow_failure: false + before_script: + - apk add --no-cache openssh-client rsync + - mkdir -p ~/.ssh + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa + - ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts + script: + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper" + - rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/" + - ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build" diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..f98e31a --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,108 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Scraper", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/services/scraper-linkedin/main.go", + "cwd": "${workspaceFolder}/services/scraper-linkedin", + "env": { + "GO_ENV": "development" + }, + "args": [], + "showLog": true, + "console": "integratedTerminal" + }, + { + "name": "Launch CV analyze services", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/apps/cron-analyzer/cmd/cli/main.go", + "cwd": "${workspaceFolder}/apps/cron-analyzer", + "env": { + "GO_ENV": "development" + }, + "args": [], + "showLog": true, + "console": "integratedTerminal" + }, + { + "name": "Launch OpenAi CV analyze services", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/apps/cron-analyzer/cmd/openai-demo/main.go", + "cwd": "${workspaceFolder}/apps/cron-analyzer", + "env": { + "GO_ENV": "development" + }, + "args": [], + "showLog": true, + "console": "integratedTerminal" + }, + { + "name": "Launch API Service", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${workspaceFolder}/services/api/cmd/server/main.go", + "cwd": "${workspaceFolder}/services/api", + "env": { + "GO_ENV": "development" + }, + "args": [], + "showLog": true, + "console": "integratedTerminal" + }, + { + "name": "Launch Glassdoor Scraper", + "type": "node", + "request": "launch", + "program": "${workspaceFolder}/services/scraper-glassdoor/index.ts", + "cwd": "${workspaceFolder}/services/scraper-glassdoor", + "runtimeArgs": ["-r", "ts-node/register"], + "env": { + "NODE_ENV": "development" + }, + "console": "integratedTerminal" + }, + { + "name": "Launch Current File", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${file}", + "console": "integratedTerminal" + }, + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${fileDirname}", + "console": "integratedTerminal" + }, + { + "name": "Test Current Package", + "type": "go", + "request": "launch", + "mode": "test", + "program": "${fileDirname}", + "console": "integratedTerminal" + }, + { + "name": "Test Current File", + "type": "go", + "request": "launch", + "mode": "test", + "program": "${file}", + "console": "integratedTerminal" + } + ] +} diff --git a/.vscode/mcp.json b/.vscode/mcp.json new file mode 100644 index 0000000..cef6b84 --- /dev/null +++ b/.vscode/mcp.json @@ -0,0 +1,3 @@ +{ + "servers": {} +} \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ccf6782 --- /dev/null +++ b/Makefile @@ -0,0 +1,151 @@ +# Jobs Scraper Monorepo Makefile + +# Variables +APP_NAME=jobs-scraper +API_DIR=./services/api +SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin +SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor +SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright +CRON_ANALYZER_DIR=./apps/cron-analyzer +CV_ANALYZER_DIR=./apps/cv-analyzer +DOCS_DIR=./services/api/pkg/swagger +BIN_DIR=./bin +API_MAIN_FILE=$(API_DIR)/cmd/server/main.go +SCRAPER_LINKEDIN_MAIN_FILE=$(SCRAPER_LINKEDIN_DIR)/main.go +CRON_MAIN_FILE=$(CRON_ANALYZER_DIR)/cmd/cli/main.go +CV_MAIN_FILE=$(CV_ANALYZER_DIR)/cmd/cli/main.go + +# Default target +.DEFAULT_GOAL := help + +# RabbitMQ server +rabbitmq-server: + @echo "Starting RabbitMQ server..." + docker run --rm -p 5672:5672 -p 15672:15672 rabbitmq:3-management + +# Services +.PHONY: run-api +run-api: ## Run the API service + @echo "Running $(APP_NAME) API service..." + @cd $(API_DIR) && go run cmd/server/main.go + +.PHONY: build-api +build-api: ## Build the API service + @echo "Building API service..." + @cd $(API_DIR) && go build -o bin/api cmd/server/main.go + +.PHONY: run-scraper-linkedin +run-scraper-linkedin: ## Run the LinkedIn scraper service + @echo "Running LinkedIn scraper service..." + @cd $(SCRAPER_LINKEDIN_DIR) && go run main.go + +.PHONY: build-scraper-linkedin +build-scraper-linkedin: ## Build the LinkedIn scraper service + @echo "Building LinkedIn scraper service..." + @cd $(SCRAPER_LINKEDIN_DIR) && go build -o bin/scraper main.go + +.PHONY: run-scraper-glassdoor +run-scraper-glassdoor: ## Run the Glassdoor scraper service + @echo "Running Glassdoor scraper service..." + @cd $(SCRAPER_GLASSDOOR_DIR) && npm run dev + +.PHONY: build-scraper-glassdoor +build-scraper-glassdoor: ## Build the Glassdoor scraper service + @echo "Building Glassdoor scraper service..." + @cd $(SCRAPER_GLASSDOOR_DIR) && npm run build + +.PHONY: setup-scraper-playwright +setup-scraper-playwright: ## Setup the Playwright scraper service + @echo "Setting up Playwright scraper service..." + @cd $(SCRAPER_PLAYWRIGHT_DIR) && bash setup.sh + +.PHONY: run-scraper-playwright +run-scraper-playwright: ## Run the Playwright scraper service + @echo "Running Playwright scraper service..." + @cd $(SCRAPER_PLAYWRIGHT_DIR) && source venv/bin/activate && python playwright_worker.py + +.PHONY: docker-scraper-playwright +docker-scraper-playwright: ## Build and run Playwright scraper in Docker + @echo "Building Docker image for Playwright scraper..." + @cd $(SCRAPER_PLAYWRIGHT_DIR) && docker build -t playwright-scraper . + @echo "Running Playwright scraper in Docker..." + @docker run --rm --name playwright-scraper playwright-scraper + +# Apps +.PHONY: run-cron-analyzer +run-cron-analyzer: ## Run the job analysis cron service + @echo "Starting Job Analysis Cron Service..." + @cd $(CRON_ANALYZER_DIR) && go run cmd/cli/main.go + +.PHONY: build-cron-analyzer +build-cron-analyzer: ## Build the cron analyzer app + @echo "Building cron analyzer app..." + @cd $(CRON_ANALYZER_DIR) && go build -o bin/cron-analyzer cmd/cli/main.go + +.PHONY: run-cv-analyzer +run-cv-analyzer: ## Run the CV analyzer app + @echo "Running CV analyzer app..." + @cd $(CV_ANALYZER_DIR) && go run cmd/cli/main.go + +.PHONY: build-cv-analyzer +build-cv-analyzer: ## Build the CV analyzer app + @echo "Building CV analyzer app..." + @cd $(CV_ANALYZER_DIR) && go build -o bin/cv-analyzer cmd/cli/main.go + +.PHONY: run-openai-demo +run-openai-demo: ## Run the OpenAI service demo + @echo "Running OpenAI service demo..." + @cd $(CRON_ANALYZER_DIR) && go run cmd/openai-demo/main.go + +.PHONY: build-openai-demo +build-openai-demo: ## Build the OpenAI service demo + @echo "Building OpenAI service demo..." + @cd $(CRON_ANALYZER_DIR) && go build -o bin/openai-demo cmd/openai-demo/main.go + +# Legacy aliases for backward compatibility +.PHONY: run +run: run-api ## Alias for run-api + +.PHONY: scraper +scraper: run-scraper-linkedin ## Alias for run-scraper-linkedin + +.PHONY: cron +cron: run-cron-analyzer ## Alias for run-cron-analyzer + +.PHONY: migrate +migrate: ## Run database migrations + @echo "Running database migrations..." + @go run $(MIGRATE_MAIN_FILE) + +.PHONY: clean +clean: ## Clean build artifacts + @echo "Cleaning build artifacts..." + @rm -rf $(BIN_DIR) + @rm -f $(SCRAPER_DIR)/scraper + @echo "Clean complete" + +.PHONY: help +help: ## Show this help message + @echo "Jobs Scraper Monorepo - Available commands:" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' + +# Swagger documentation +.PHONY: swagger +swagger: ## Generate Swagger documentation + @echo "Generating Swagger documentation..." + @go run github.com/swaggo/swag/cmd/swag@latest init -g $(API_MAIN_FILE) -o $(DOCS_DIR) + @echo "Swagger documentation generated in $(DOCS_DIR)/" + +# Docker operations (if needed in the future) +.PHONY: docker-build +docker-build: ## Build Docker image + @echo "Building Docker image..." + @docker build -t $(APP_NAME) . + @echo "Docker image built: $(APP_NAME)" + +.PHONY: docker-run +docker-run: ## Run Docker container + @echo "Running Docker container..." + @docker run -p 8080:8080 $(APP_NAME) + diff --git a/apps/cron-analyzer/.env b/apps/cron-analyzer/.env new file mode 100644 index 0000000..38db1b6 --- /dev/null +++ b/apps/cron-analyzer/.env @@ -0,0 +1,17 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 + +# OpenAI Configuration +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini \ No newline at end of file diff --git a/apps/cron-analyzer/cmd/cli/.env b/apps/cron-analyzer/cmd/cli/.env new file mode 100644 index 0000000..bd06050 --- /dev/null +++ b/apps/cron-analyzer/cmd/cli/.env @@ -0,0 +1,17 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=jobs_scraper +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 + +# OpenAI Configuration +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini \ No newline at end of file diff --git a/apps/cron-analyzer/cmd/cli/main.go b/apps/cron-analyzer/cmd/cli/main.go new file mode 100644 index 0000000..41124c8 --- /dev/null +++ b/apps/cron-analyzer/cmd/cli/main.go @@ -0,0 +1,47 @@ +package main + +import ( + "log" + + "github.com/jobs-scraper/apps/cron-analyzer/internal" + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/joho/godotenv" +) + +func main() { + if err := godotenv.Load("cmd/cli/.local.env"); err != nil { + log.Printf("No .local.env file found: %v, trying .env", err) + if err := godotenv.Load("cmd/cli/.env"); err != nil { + log.Println("No .env file found, using system environment variables") + } + } + + dbConfig := infrastructure.LoadConfigFromEnv() + + db, err := infrastructure.NewConnection(dbConfig) + if err != nil { + log.Fatalf("Error connecting to db: %v", err) + } + + err = db.Ping() + if err != nil { + log.Fatal("Error pinging db") + } + + log.Println("Successfully connected to db") + + // Run migrations + log.Println("Running database migrations...") + if err := infrastructure.RunMigrations(db); err != nil { + log.Fatalf("Error running migrations: %v", err) + } + log.Println("Migrations completed successfully") + + analyzer := internal.NewJobAnalyzer(db) + + if err := analyzer.AnalyzeJobs(); err != nil { + log.Fatalf("Error analyzing jobs: %v", err) + } + + log.Println("Job analysis completed successfully") +} diff --git a/apps/cron-analyzer/cv.txt b/apps/cron-analyzer/cv.txt new file mode 100644 index 0000000..915af7b --- /dev/null +++ b/apps/cron-analyzer/cv.txt @@ -0,0 +1,65 @@ +# Ziad Elshimy +## Frontend Developer + +Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects. + +`ziadshimy7@gmail.com` +`01223381370` +`Alexandria` +[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6) +[GitHub](https://github.com/ziadshimy7) + +--- + +## Work Experience + +### kari - Fullstack Developer +**Jun 2022 - current** +- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates. +- Led the development of multiple internal projects. +- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills. +- Led task planning and coordinated project timelines, reducing overall project duration by 20%. + +### Callibri - Frontend Developer +**Jan 2022 - Mar 2022** +- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement. +- Collaborated closely with senior developers to manage a complex design project, increasing efficiency. + +### EJADA - Frontend Developer +**Mar 2021 - Dec 2021** +- Troubleshooted the website's problems and stay up to date on technology. + +--- + +## Education + +### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General +**Jan 2017 - Dec 2021** + +--- + +## Skills +- HTML5 +- Cascading Style Sheets (CSS) +- SCSS +- Tailwind css +- JavaScript +- TypeScript +- React.js +- Redux.js +- Next.js +- Node.js +- SSR +- Webpack +- Vite +- Go (Golang) +- Microservices +- docker +- docker-compose + +--- + +## Certifications +- CCNA +- The Complete JavaScript Course 2023- Udemy +- React - The Complete Guide - Udemy \ No newline at end of file diff --git a/apps/cron-analyzer/go.mod b/apps/cron-analyzer/go.mod new file mode 100644 index 0000000..ec40919 --- /dev/null +++ b/apps/cron-analyzer/go.mod @@ -0,0 +1,29 @@ +module github.com/jobs-scraper/apps/cron-analyzer + +go 1.24.0 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/joho/godotenv v1.5.1 +) + +require ( + github.com/eduardolat/openroutergo v0.1.0 // indirect + github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/orsinium-labs/enum v1.4.0 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure + +replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter + +replace github.com/jobs-scraper/libs/repo => ../../libs/repo diff --git a/apps/cron-analyzer/go.sum b/apps/cron-analyzer/go.sum new file mode 100644 index 0000000..e1b7cd6 --- /dev/null +++ b/apps/cron-analyzer/go.sum @@ -0,0 +1,79 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI= +github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM= +github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/cron-analyzer/internal/analyze-jobs.go b/apps/cron-analyzer/internal/analyze-jobs.go new file mode 100644 index 0000000..d04d3fe --- /dev/null +++ b/apps/cron-analyzer/internal/analyze-jobs.go @@ -0,0 +1,157 @@ +package internal + +import ( + "database/sql" + "encoding/json" + "log" + "os" + + // "path/filepath" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/internal/pkg/openai" + "github.com/jobs-scraper/internal/pkg/openrouter" + "github.com/jobs-scraper/libs/repo" +) + +type JobAnalyzer struct { + db *sql.DB + jobRepo *repo.JobRepository + jobDescriptionRepo *repo.JobDescriptionRepository + jobAnalysisResultRepo *repo.JobAnalysisResultRepository + openRouterService openrouter.OpenRouterService + rmq *rabbitmq.RabbitMQClient + openAiService *openai.OpenAIService +} + +func NewJobAnalyzer(db *sql.DB) *JobAnalyzer { + apiKey := os.Getenv("OPENROUTER_API_KEY") + model := os.Getenv("CV_AI_MODEL") + openAiapiKey := os.Getenv("OPENAI_API_KEY") + if openAiapiKey == "" { + log.Fatal("OPENAI_API_KEY environment variable is required") + } + + // Create OpenAI service + openAiModel := os.Getenv("OPENAI_MODEL") + if openAiModel == "" { + openAiModel = "gpt-4o-mini" // Default to cost-effective model + } + + jobRepo := repo.NewJobRepository(db) + jobDescriptionRepo := repo.NewJobDescriptionRepository(db) + jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) + openRouterService := openrouter.NewOpenRouterService(model, apiKey) + openAiService := openai.NewOpenAIService(openAiapiKey, openAiModel) + + rmq, err := rabbitmq.NewRabbitMQClient() + if err != nil { + log.Printf("Warning: Failed to create RabbitMQ client: %v", err) + } + + return &JobAnalyzer{ + db: db, + jobRepo: jobRepo, + jobDescriptionRepo: jobDescriptionRepo, + jobAnalysisResultRepo: jobAnalysisResultRepo, + openRouterService: openRouterService, + rmq: rmq, + openAiService: openAiService, + } +} + +func (ja *JobAnalyzer) AnalyzeJobs() error { + // Get jobs that need analysis (status = created) + jobs, err := ja.jobRepo.GetJobsByStatus(domain.JobStatusCreated) + if err != nil { + return err + } + + log.Printf("Found %d jobs to analyze", len(jobs)) + + cv, err := os.ReadFile("cv.txt") + + if err != nil { + return err + } + + for _, job := range jobs { + if err := ja.analyzeJob(job, string(cv)); err != nil { + log.Printf("Error analyzing job %d: %v", job.ID, err) + // Update job status to error + if updateErr := ja.jobRepo.UpdateJobStatus(job.ID, domain.JobStatusError); updateErr != nil { + log.Printf("Error updating job status to error: %v", updateErr) + } + continue + } + + // Update job status to analyzed + if err := ja.jobRepo.UpdateJobStatus(job.ID, domain.JobStatusAnalyzed); err != nil { + log.Printf("Error updating job status to analyzed: %v", err) + } + + log.Printf("Successfully analyzed job %d", job.ID) + } + + return nil +} + +func (ja *JobAnalyzer) analyzeJob(job domain.Job, cv string) error { + // Get job description + jobDescription, jobCriteria, err := ja.jobDescriptionRepo.GetJobDescriptionByJobID(job.ID) + if err != nil { + return err + } + + // Analyze job with CV + result, err := ja.openAiService.AnalyzeJobDescription(cv, domain.JobDescription{ + JobID: job.ID, + Description: jobDescription, + Criteria: jobCriteria, + }) + + if err != nil { + return err + } + + log.Printf("Analysis result for job %d: Recommendation=%s, Score=%d", job.ID, result.Recommendation, result.ConfidenceScore) + + // Save analysis result to database + analysisResult := &domain.JobAnalysisResult{ + JobID: job.ID, + AnalysisResult: result.Summary, + MatchScore: &result.ConfidenceScore, + KeySkills: result.MatchingSkills, + MissingSkills: result.MissingSkills, + Recommendations: &result.Recommendation, + } + + if err := ja.jobAnalysisResultRepo.SaveAnalysisResult(analysisResult); err != nil { + log.Printf("Warning: Failed to save analysis result to database: %v", err) + } else { + log.Printf("Successfully saved analysis result for job %d to database", job.ID) + } + + // Publish CV analysis message to RabbitMQ if available + if ja.rmq != nil { + message := rabbitmq.CvAnalyzeMessage{ + JobID: job.ID, + } + + if err := ja.publishCvAnalysisMessage(message); err != nil { + log.Printf("Warning: Failed to publish CV analysis message: %v", err) + } + } + + return nil +} + +func (ja *JobAnalyzer) publishCvAnalysisMessage(message rabbitmq.CvAnalyzeMessage) error { + data, err := json.Marshal(message) + if err != nil { + return err + } + + return ja.rmq.Publish(rabbitmq.CvAnalyzeQueue, rabbitmq.CvAnalyzeExchange, data) +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7786ee9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,66 @@ +version: '3.8' + +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_DB: jobs_scraper + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + rabbitmq: + image: rabbitmq:3-management-alpine + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + RABBITMQ_ERLANG_COOKIE: "SWQOKODSQALRPCLNMEQGABCDEFGHIJKLMNOPQRST" + ports: + - "5672:5672" + - "15672:15672" + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + scraper-playwright: + build: + context: ./services/scraper-playwright + dockerfile: Dockerfile + environment: + REDIS_HOST: redis + REDIS_PORT: 6379 + WORKER_ID: playwright_1 + LOG_LEVEL: INFO + depends_on: + redis: + condition: service_healthy + restart: unless-stopped + +volumes: + postgres_data: + rabbitmq_data: + redis_data: diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..bde6a6a --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper + +go 1.24.0 + +toolchain go1.24.7 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/go.work b/go.work new file mode 100644 index 0000000..e8ae214 --- /dev/null +++ b/go.work @@ -0,0 +1,17 @@ +go 1.24.0 + +use ( + . + ./apps/cron-analyzer + ./internal/pkg/domain + ./internal/pkg/infrastructure + ./internal/pkg/openai + ./internal/pkg/openrouter + ./internal/pkg/utils + ./libs/ports + ./libs/repo + ./libs/server + ./services/api + ./services/scraper-google + ./services/scraper-linkedin +) diff --git a/go.work.sum b/go.work.sum new file mode 100644 index 0000000..e430d1b --- /dev/null +++ b/go.work.sum @@ -0,0 +1,247 @@ +cel.dev/expr v0.16.0/go.mod h1:TRSuuV7DlVCE/uwv5QbAiW/v8l5O8C4eEPHeu7gf7Sg= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= +cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/spanner v1.56.0/go.mod h1:DndqtUKQAt3VLuV2Le+9Y3WTnq5cNKrnLb/Piqcj+h0= +cloud.google.com/go/storage v1.38.0/go.mod h1:tlUADB0mAb9BgYls9lq+8MGkfzOXuLrnHXlpHmvFJoY= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= +github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.16/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/aws/aws-sdk-go v1.49.6/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= +github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33/go.mod h1:84XgODVR8uRhmOnUkKGUZKqIMxmjmLOR8Uyp7G/TPwc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo= +github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= +github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= +github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= +github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/microsoft/go-mssqldb v1.0.0/go.mod h1:+4wZTUnz/SV6nffv+RRRB/ss8jPng5Sho2SmM1l2ts4= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= +github.com/mutecomm/go-sqlcipher/v4 v4.4.0/go.mod h1:PyN04SaWalavxRGH9E8ZftG6Ju7rsPrGmQRjrEaVpiY= +github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= +github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/snowflakedb/gosnowflake v1.6.19/go.mod h1:FM1+PWUdwB9udFDsXdfD58NONC0m+MlOSmQRvimobSM= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE= +go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.29.0/go.mod h1:jlRVBe7+Z1wyxFSUs48L6OBQZ5JwH2Hg/Vbl+t9rAgI= +go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/telemetry v0.0.0-20250807160809-1a19826ec488/go.mod h1:fGb/2+tgXXjhjHsTNdVEEMZNWA0quBnfrO+AfoDSAKw= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= +modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= +modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= +modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw= +modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= +modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s= +modernc.org/lldb v1.0.0/go.mod h1:jcRvJGWfCGodDZz8BPwiKMJxGJngQ/5DrRapkQnLob8= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/ql v1.0.0/go.mod h1:xGVyrLIatPcO2C1JvI/Co8c0sr6y91HKFNy4pt9JXEY= +modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= +modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/migrations/001_jobs.down.sql b/internal/migrations/001_jobs.down.sql new file mode 100644 index 0000000..d8824d8 --- /dev/null +++ b/internal/migrations/001_jobs.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS jobs; \ No newline at end of file diff --git a/internal/migrations/001_jobs.up.sql b/internal/migrations/001_jobs.up.sql new file mode 100644 index 0000000..bea9b08 --- /dev/null +++ b/internal/migrations/001_jobs.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS jobs ( + id BIGSERIAL PRIMARY KEY, + title VARCHAR(255) NOT NULL, + company VARCHAR(255) NOT NULL, + company_link TEXT, + location VARCHAR(255), + job_link TEXT UNIQUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); \ No newline at end of file diff --git a/internal/migrations/002_job_descriptions.down.sql b/internal/migrations/002_job_descriptions.down.sql new file mode 100644 index 0000000..84c5cfa --- /dev/null +++ b/internal/migrations/002_job_descriptions.down.sql @@ -0,0 +1,12 @@ +-- Drop the trigger first +DROP TRIGGER IF EXISTS trigger_update_job_descriptions_timestamp ON job_descriptions; + +-- Drop the trigger function +DROP FUNCTION IF EXISTS update_job_descriptions_updated_at(); + +-- Drop the indexes +DROP INDEX IF EXISTS idx_job_descriptions_criteria; +DROP INDEX IF EXISTS idx_job_descriptions_job_id; + +-- Drop the table (this will automatically drop any remaining dependencies) +DROP TABLE IF EXISTS job_descriptions; \ No newline at end of file diff --git a/internal/migrations/002_job_descriptions.up.sql b/internal/migrations/002_job_descriptions.up.sql new file mode 100644 index 0000000..b53dbe2 --- /dev/null +++ b/internal/migrations/002_job_descriptions.up.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS job_descriptions ( + id SERIAL PRIMARY KEY, + job_id BIGINT NOT NULL, + description TEXT NOT NULL, + job_criteria JSONB DEFAULT '{}'::jsonb, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE +); + +-- Create an index on job_id for faster lookups +CREATE INDEX idx_job_descriptions_job_id ON job_descriptions(job_id); + +-- Create a GIN index on the job_criteria JSONB field for faster searching within the JSON data +CREATE INDEX idx_job_descriptions_criteria ON job_descriptions USING GIN (job_criteria); + +-- Add trigger to update updated_at timestamp +CREATE OR REPLACE FUNCTION update_job_descriptions_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ language 'plpgsql'; + +CREATE TRIGGER trigger_update_job_descriptions_timestamp + BEFORE UPDATE ON job_descriptions + FOR EACH ROW + EXECUTE FUNCTION update_job_descriptions_updated_at(); \ No newline at end of file diff --git a/internal/migrations/003_add_unique_constraint_job_id.down.sql b/internal/migrations/003_add_unique_constraint_job_id.down.sql new file mode 100644 index 0000000..2dda6f9 --- /dev/null +++ b/internal/migrations/003_add_unique_constraint_job_id.down.sql @@ -0,0 +1,2 @@ +-- Remove unique constraint from job_id column in job_descriptions table +ALTER TABLE job_descriptions DROP CONSTRAINT IF EXISTS unique_job_id; diff --git a/internal/migrations/003_add_unique_constraint_job_id.up.sql b/internal/migrations/003_add_unique_constraint_job_id.up.sql new file mode 100644 index 0000000..45cbc4f --- /dev/null +++ b/internal/migrations/003_add_unique_constraint_job_id.up.sql @@ -0,0 +1,2 @@ +-- Add unique constraint to job_id column in job_descriptions table +ALTER TABLE job_descriptions ADD CONSTRAINT unique_job_id UNIQUE (job_id); diff --git a/internal/migrations/004_job_link_unique.down.sql b/internal/migrations/004_job_link_unique.down.sql new file mode 100644 index 0000000..61c9488 --- /dev/null +++ b/internal/migrations/004_job_link_unique.down.sql @@ -0,0 +1 @@ +ALTER TABLE jobs DROP CONSTRAINT unique_job_link; diff --git a/internal/migrations/004_job_link_unique.up.sql b/internal/migrations/004_job_link_unique.up.sql new file mode 100644 index 0000000..86348a3 --- /dev/null +++ b/internal/migrations/004_job_link_unique.up.sql @@ -0,0 +1 @@ +ALTER TABLE jobs ADD CONSTRAINT unique_job_link UNIQUE (job_link); diff --git a/internal/migrations/005_job_provider.down.sql b/internal/migrations/005_job_provider.down.sql new file mode 100644 index 0000000..88623e9 --- /dev/null +++ b/internal/migrations/005_job_provider.down.sql @@ -0,0 +1 @@ +ALTER TABLE jobs DROP COLUMN IF EXISTS provider; \ No newline at end of file diff --git a/internal/migrations/005_job_provider.up.sql b/internal/migrations/005_job_provider.up.sql new file mode 100644 index 0000000..94415f7 --- /dev/null +++ b/internal/migrations/005_job_provider.up.sql @@ -0,0 +1 @@ +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS provider INTEGER DEFAULT 0; \ No newline at end of file diff --git a/internal/migrations/006_job_timestamp.down.sql b/internal/migrations/006_job_timestamp.down.sql new file mode 100644 index 0000000..13b7717 --- /dev/null +++ b/internal/migrations/006_job_timestamp.down.sql @@ -0,0 +1 @@ +ALTER TABLE jobs DROP COLUMN IF EXISTS job_timestamp; \ No newline at end of file diff --git a/internal/migrations/006_job_timestamp.up.sql b/internal/migrations/006_job_timestamp.up.sql new file mode 100644 index 0000000..83bfe08 --- /dev/null +++ b/internal/migrations/006_job_timestamp.up.sql @@ -0,0 +1 @@ +ALTER TABLE jobs ADD COLUMN IF NOT EXISTS job_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP; \ No newline at end of file diff --git a/internal/migrations/007_add_job_status.down.sql b/internal/migrations/007_add_job_status.down.sql new file mode 100644 index 0000000..dbe4623 --- /dev/null +++ b/internal/migrations/007_add_job_status.down.sql @@ -0,0 +1 @@ +ALTER TABLE jobs DROP COLUMN status; diff --git a/internal/migrations/007_add_job_status.up.sql b/internal/migrations/007_add_job_status.up.sql new file mode 100644 index 0000000..01d1ddd --- /dev/null +++ b/internal/migrations/007_add_job_status.up.sql @@ -0,0 +1 @@ +ALTER TABLE jobs ADD COLUMN status INTEGER DEFAULT 0; diff --git a/internal/migrations/008_job_analysis_results.down.sql b/internal/migrations/008_job_analysis_results.down.sql new file mode 100644 index 0000000..a48f381 --- /dev/null +++ b/internal/migrations/008_job_analysis_results.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_job_analysis_results_match_score; +DROP INDEX IF EXISTS idx_job_analysis_results_job_id; +DROP TABLE IF EXISTS job_analysis_results; diff --git a/internal/migrations/008_job_analysis_results.up.sql b/internal/migrations/008_job_analysis_results.up.sql new file mode 100644 index 0000000..e72d63b --- /dev/null +++ b/internal/migrations/008_job_analysis_results.up.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS job_analysis_results ( + id BIGSERIAL PRIMARY KEY, + job_id BIGINT NOT NULL, + analysis_result TEXT NOT NULL, + match_score INTEGER, + key_skills TEXT[], + missing_skills TEXT[], + recommendations TEXT, + analyzed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (job_id) REFERENCES jobs(id) ON DELETE CASCADE +); + +CREATE INDEX idx_job_analysis_results_job_id ON job_analysis_results(job_id); +CREATE INDEX idx_job_analysis_results_match_score ON job_analysis_results(match_score); diff --git a/internal/pkg/domain/go.mod b/internal/pkg/domain/go.mod new file mode 100644 index 0000000..5447dc2 --- /dev/null +++ b/internal/pkg/domain/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper/internal/pkg/domain + +go 1.24.0 + +toolchain go1.24.7 diff --git a/internal/pkg/domain/google-link.go b/internal/pkg/domain/google-link.go new file mode 100644 index 0000000..3fd9102 --- /dev/null +++ b/internal/pkg/domain/google-link.go @@ -0,0 +1,9 @@ +package domain + +type GoogleLink struct { + ID int64 + Title string + Description string + Link string + CompanyName string +} diff --git a/internal/pkg/domain/job-description.go b/internal/pkg/domain/job-description.go new file mode 100644 index 0000000..bfe4d99 --- /dev/null +++ b/internal/pkg/domain/job-description.go @@ -0,0 +1,7 @@ +package domain + +type JobDescription struct { + JobID int64 + Description string + Criteria map[string]string +} diff --git a/internal/pkg/domain/job-with-description.go b/internal/pkg/domain/job-with-description.go new file mode 100644 index 0000000..6720fb2 --- /dev/null +++ b/internal/pkg/domain/job-with-description.go @@ -0,0 +1,6 @@ +package domain + +type JobWithDescription struct { + Job Job + JobDescription JobDescription +} diff --git a/internal/pkg/domain/job.go b/internal/pkg/domain/job.go new file mode 100644 index 0000000..fb84665 --- /dev/null +++ b/internal/pkg/domain/job.go @@ -0,0 +1,54 @@ +package domain + +import "time" + +// JobProvider represents different job board providers +type JobProvider int + +type JobStatus int + +const ( + LinkedIn JobProvider = iota + Indeed + Glassdoor + Bayt + TokyoDev + JapanDev + Google +) + +const ( + JobStatusCreated JobStatus = iota + JobStatusAnalyzed + JobStatusError +) + +type Job struct { + ID int64 + Title string + Company string + CompanyLink string + Location string + JobLink string + Provider JobProvider + JobPostTime *time.Time + Status JobStatus +} + +type JobAnalysisResult struct { + ID int64 + JobID int64 + AnalysisResult string + MatchScore *int + KeySkills []string + MissingSkills []string + Recommendations *string + AnalyzedAt time.Time +} + +type SearchQuery struct { + Keywords string `json:"keywords"` + Location string `json:"location"` + NumPages int `json:"numPages"` + FWT string `json:"f_WT"` // Work type filter (1=onsite, 2=remote, 3=hybrid) +} diff --git a/internal/pkg/infrastructure/db.go b/internal/pkg/infrastructure/db.go new file mode 100644 index 0000000..54e9bbf --- /dev/null +++ b/internal/pkg/infrastructure/db.go @@ -0,0 +1,173 @@ +package infrastructure + +import ( + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/postgres" + _ "github.com/golang-migrate/migrate/v4/source/file" + _ "github.com/lib/pq" +) + +// Config holds database configuration +type Config struct { + Host string + Port string + User string + Password string + DBName string + SSLMode string +} + +// NewConnection creates a new database connection +func NewConnection(config Config) (*sql.DB, error) { + dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", + config.Host, config.Port, config.User, config.Password, config.DBName, config.SSLMode) + + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, fmt.Errorf("failed to open database connection: %w", err) + } + + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("failed to ping database: %w", err) + } + + return db, nil +} + +// LoadConfigFromEnv loads database configuration from environment variables +func LoadConfigFromEnv() Config { + return Config{ + Host: getEnv("DB_HOST", "localhost"), + Port: getEnv("DB_PORT", "5432"), + User: getEnv("DB_USER", "postgres"), + Password: getEnv("DB_PASSWORD", "password"), + DBName: getEnv("DB_NAME", "linkedin_jobs"), + SSLMode: getEnv("DB_SSLMODE", "disable"), + } +} + +func getEnv(key, defaultValue string) string { + if value := os.Getenv(key); value != "" { + return value + } + return defaultValue +} + +// RunMigrations runs all pending migrations +func RunMigrations(db *sql.DB) error { + driver, err := postgres.WithInstance(db, &postgres.Config{}) + if err != nil { + return fmt.Errorf("failed to create postgres driver: %w", err) + } + + // Get the migrations directory path + migrationsPath, err := getMigrationsPath() + if err != nil { + return fmt.Errorf("failed to get migrations path: %w", err) + } + + m, err := migrate.NewWithDatabaseInstance( + fmt.Sprintf("file://%s", migrationsPath), + "postgres", + driver, + ) + if err != nil { + return fmt.Errorf("failed to create migrate instance: %w", err) + } + + if err := m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) { + return fmt.Errorf("failed to run migrations: %w", err) + } + + return nil +} + +// RollbackMigrations rolls back migrations by the specified number of steps +func RollbackMigrations(db *sql.DB, steps int) error { + driver, err := postgres.WithInstance(db, &postgres.Config{}) + if err != nil { + return fmt.Errorf("failed to create postgres driver: %w", err) + } + + migrationsPath, err := getMigrationsPath() + if err != nil { + return fmt.Errorf("failed to get migrations path: %w", err) + } + + m, err := migrate.NewWithDatabaseInstance( + fmt.Sprintf("file://%s", migrationsPath), + "postgres", + driver, + ) + if err != nil { + return fmt.Errorf("failed to create migrate instance: %w", err) + } + defer m.Close() + + if err := m.Steps(-steps); err != nil && !errors.Is(err, migrate.ErrNoChange) { + return fmt.Errorf("failed to rollback migrations: %w", err) + } + + return nil +} + +// GetMigrationVersion returns the current migration version +func GetMigrationVersion(db *sql.DB) (uint, bool, error) { + driver, err := postgres.WithInstance(db, &postgres.Config{}) + if err != nil { + return 0, false, fmt.Errorf("failed to create postgres driver: %w", err) + } + + migrationsPath, err := getMigrationsPath() + if err != nil { + return 0, false, fmt.Errorf("failed to get migrations path: %w", err) + } + + m, err := migrate.NewWithDatabaseInstance( + fmt.Sprintf("file://%s", migrationsPath), + "postgres", + driver, + ) + if err != nil { + return 0, false, fmt.Errorf("failed to create migrate instance: %w", err) + } + defer m.Close() + + version, dirty, err := m.Version() + if err != nil { + if errors.Is(err, migrate.ErrNilVersion) { + return 0, false, nil + } + return 0, false, fmt.Errorf("failed to get migration version: %w", err) + } + + return version, dirty, nil +} + +func getMigrationsPath() (string, error) { + // Get the directory of the current source file + _, filename, _, ok := runtime.Caller(0) + if !ok { + return "", fmt.Errorf("failed to get current file path") + } + + // Get the internal directory (go up from internal/pkg/infrastructure/ to internal/) + // Current path: .../internal/pkg/infrastructure/db.go + // Need to go up: infrastructure -> pkg -> internal + internalDir := filepath.Dir(filepath.Dir(filepath.Dir(filename))) + migrationsPath := filepath.Join(internalDir, "migrations") + + // Verify the migrations directory exists + if _, err := os.Stat(migrationsPath); err != nil { + return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err) + } + + return migrationsPath, nil +} diff --git a/internal/pkg/infrastructure/go.mod b/internal/pkg/infrastructure/go.mod new file mode 100644 index 0000000..52a2850 --- /dev/null +++ b/internal/pkg/infrastructure/go.mod @@ -0,0 +1,27 @@ +module github.com/jobs-scraper/internal/pkg/infrastructure + +go 1.24.0 + +toolchain go1.24.7 + +require ( + github.com/golang-migrate/migrate/v4 v4.19.0 + github.com/gorilla/mux v1.8.1 + github.com/jobs-scraper/libs/ports v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/lib/pq v1.10.9 + github.com/rabbitmq/amqp091-go v1.10.0 +) + +replace github.com/jobs-scraper/libs/repo => ../../../libs/repo + +replace github.com/jobs-scraper/internal/pkg/utils => ../utils + +replace github.com/jobs-scraper/libs/ports => ../../../libs/ports + +require ( + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + golang.org/x/sys v0.36.0 // indirect +) diff --git a/internal/pkg/infrastructure/go.sum b/internal/pkg/infrastructure/go.sum new file mode 100644 index 0000000..4880d04 --- /dev/null +++ b/internal/pkg/infrastructure/go.sum @@ -0,0 +1,71 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pkg/infrastructure/http/job.go b/internal/pkg/infrastructure/http/job.go new file mode 100644 index 0000000..1dae58b --- /dev/null +++ b/internal/pkg/infrastructure/http/job.go @@ -0,0 +1,148 @@ +package http + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strconv" + + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/internal/pkg/utils" + + "github.com/gorilla/mux" +) + +// JobHandler handles HTTP requests related to jobs +type JobHandler struct { + jobCommands *ports.JobCommands + jobAnalysisResultRepo *repo.JobAnalysisResultRepository +} + +// NewJobHandler creates a new instance of JobHandler +func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler { + return &JobHandler{ + jobCommands: jobCommands, + jobAnalysisResultRepo: jobAnalysisResultRepo, + } +} + +// RegisterRoutes registers all routes to the router +func (h *JobHandler) RegisterRoutes(router *mux.Router) { + jobs := router.PathPrefix("/jobs").Subrouter() + + //jobs.Use(AuthMiddleware) + + jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST") + jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET") + jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") + jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") +} + +// CreateJob handles the creation of a new job +// @Summary Create a new job +// @Description Creates a new job with the provided specifications +// @Tags jobs +// @Accept json +// @Produce json +// @Param job body ports.CreateJobCommand true "Job creation data" +// @Success 201 {object} map[string]string "Successfully created job" +// @Failure 400 {object} map[string]string "Invalid request data" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs [post] +func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error { + var cmd ports.CreateJobCommand + if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON()) + } + + err := h.jobCommands.CreateJob.Handle(cmd) + if err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"}) + return nil +} + +// GetAllAnalysisResults handles retrieving all job analysis results +// @Summary Get all job analysis results +// @Description Retrieves all job analysis results from the database +// @Tags jobs +// @Produce json +// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/analysis [get] +func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error { + results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults() + if err != nil { + slog.Error("Failed to get analysis results", "error", err) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusOK, results) + return nil +} + +// GetJobAnalysisResult handles retrieving analysis result for a specific job +// @Summary Get job analysis result by job ID +// @Description Retrieves the analysis result for a specific job +// @Tags jobs +// @Produce json +// @Param id path int true "Job ID" +// @Success 200 {object} domain.JobAnalysisResult "Job analysis result" +// @Failure 400 {object} map[string]string "Invalid job ID" +// @Failure 404 {object} map[string]string "Analysis result not found" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/{id}/analysis [get] +func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error { + vars := mux.Vars(r) + jobID, err := strconv.ParseInt(vars["id"], 10, 64) + if err != nil { + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID")) + } + + result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID) + if err != nil { + slog.Error("Failed to get analysis result", "jobID", jobID, "error", err) + return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found")) + } + + utils.WriteJSON(w, http.StatusOK, result) + return nil +} + +// GetTopMatches handles retrieving top matching jobs based on analysis score +// @Summary Get top matching jobs +// @Description Retrieves jobs with high analysis match scores +// @Tags jobs +// @Produce json +// @Param min_score query int false "Minimum match score (default: 70)" +// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results" +// @Failure 400 {object} map[string]string "Invalid minimum score" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/analysis/top-matches [get] +func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error { + minScoreStr := r.URL.Query().Get("min_score") + minScore := 70 // default minimum score + + if minScoreStr != "" { + var err error + minScore, err = strconv.Atoi(minScoreStr) + if err != nil { + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score")) + } + } + + results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore) + if err != nil { + slog.Error("Failed to get top matches", "minScore", minScore, "error", err) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusOK, results) + return nil +} diff --git a/internal/pkg/infrastructure/http/middlewares.go b/internal/pkg/infrastructure/http/middlewares.go new file mode 100644 index 0000000..3b6e0da --- /dev/null +++ b/internal/pkg/infrastructure/http/middlewares.go @@ -0,0 +1,143 @@ +package http + +import ( + "context" + "encoding/json" + "log" + "net/http" + "strings" + "time" +) + +// CORSMiddleware handles Cross-Origin Resource Sharing +func CORSMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Set CORS headers + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Max-Age", "86400") + + // Handle preflight OPTIONS requests + if r.Method == "OPTIONS" { + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) +} + +// LogsMiddleware logs HTTP requests with timing information +func LogsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start)) + }) +} + +// AuthMiddleware handles JWT token validation, API keys, and basic auth +func AuthMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Skip auth for swagger endpoints and health checks + if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" { + next.ServeHTTP(w, r) + return + } + + // Try Bearer token authentication first + authHeader := r.Header.Get("Authorization") + if authHeader != "" { + if strings.HasPrefix(authHeader, "Bearer ") { + token := strings.TrimPrefix(authHeader, "Bearer ") + if token != "" { + userID, err := validateJWTToken(token) + if err == nil { + ctx := context.WithValue(r.Context(), "user_id", userID) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + } + } + + // Try API Key authentication + apiKey := r.Header.Get("X-API-Key") + if apiKey != "" { + if isValidAPIKey(apiKey) { + ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey)) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + + // Try Basic authentication + username, password, ok := r.BasicAuth() + if ok { + if isValidCredentials(username, password) { + ctx := context.WithValue(r.Context(), "username", username) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + + // Fallback to cookie-based auth (existing behavior) + token, err := r.Cookie("token") + if err == nil && token.Value == "123" { + next.ServeHTTP(w, r) + return + } + + // No valid authentication found + response := map[string]interface{}{ + "error": "Unauthorized", + "message": "Valid authentication required. Use Bearer token, API key, or basic auth.", + "code": http.StatusUnauthorized, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(response) + }) +} + +// Helper functions for authentication validation +func validateJWTToken(token string) (string, error) { + // TODO: Implement JWT validation logic with your preferred JWT library + // For demonstration, accept a demo token + if token == "valid-jwt-token" { + return "user-123", nil + } + return "", http.ErrAbortHandler +} + +func isValidAPIKey(apiKey string) bool { + // TODO: Implement API key validation logic + // For demonstration, accept a demo API key + return apiKey == "demo-api-key-123" +} + +func getUserIDFromAPIKey(apiKey string) string { + // TODO: Implement logic to get user ID from API key + // For demonstration, return a mock user ID + return "api-user-123" +} + +func isValidCredentials(username, password string) bool { + // TODO: Implement credential validation logic + // For demonstration, accept demo credentials + return username == "admin" && password == "password" +} + +// GetUserIDFromContext extracts user ID from request context +func GetUserIDFromContext(ctx context.Context) (string, bool) { + userID, ok := ctx.Value("user_id").(string) + return userID, ok +} + +// GetUsernameFromContext extracts username from request context +func GetUsernameFromContext(ctx context.Context) (string, bool) { + username, ok := ctx.Value("username").(string) + return username, ok +} diff --git a/internal/pkg/infrastructure/rabbitmq/messages.go b/internal/pkg/infrastructure/rabbitmq/messages.go new file mode 100644 index 0000000..a9d3c52 --- /dev/null +++ b/internal/pkg/infrastructure/rabbitmq/messages.go @@ -0,0 +1,5 @@ +package rabbitmq + +type CvAnalyzeMessage struct { + JobID int64 `json:"jobID"` +} diff --git a/internal/pkg/infrastructure/rabbitmq/rabbitmq.go b/internal/pkg/infrastructure/rabbitmq/rabbitmq.go new file mode 100644 index 0000000..447c0ef --- /dev/null +++ b/internal/pkg/infrastructure/rabbitmq/rabbitmq.go @@ -0,0 +1,250 @@ +package rabbitmq + +import ( + "fmt" + "log" + "os" + "time" + + amqp "github.com/rabbitmq/amqp091-go" +) + +const ( + ScraperExchange = "scraper_exchange" + LinkedInQueue = "scraper.linkedin" + IndeedQueue = "scraper.indeed" + BaytQueue = "scraper.bayt" + TokyoDevQueue = "scraper.tokyodev" + GlassDoorQueue = "scraper.glassdoor" + JapanDevQueue = "scraper.japandev" + GoogleQueue = "scraper.google" + DeadLetterExchange = "scraper_dlx" + CvAnalyzeExchange = "cv_exchange" + CvAnalyzeQueue = "cv.analyze" +) + +type RabbitMQClient struct { + Conn *amqp.Connection + Channel *amqp.Channel +} + +type MessageHandler func([]byte) error + +func NewRabbitMQClient() (*RabbitMQClient, error) { + // Get RabbitMQ URL from environment or use default + rabbitmqURL := os.Getenv("RABBITMQ_URL") + if rabbitmqURL == "" { + rabbitmqURL = "amqp://guest:guest@localhost:5672/" + } + + // Connect to RabbitMQ with retry logic + var conn *amqp.Connection + var err error + + for i := range 5 { + conn, err = amqp.Dial(rabbitmqURL) + if err == nil { + break + } + log.Printf("Failed to connect to RabbitMQ (attempt %d/5): %v", i+1, err) + time.Sleep(5 * time.Second) + } + + if err != nil { + return nil, fmt.Errorf("error connecting to RabbitMQ after 5 attempts: %v", err) + } + + // Create a channel + ch, err := conn.Channel() + if err != nil { + conn.Close() + return nil, fmt.Errorf("error creating channel: %v", err) + } + + client := &RabbitMQClient{ + Conn: conn, + Channel: ch, + } + + // Setup exchanges and queues + if err := client.setupInfrastructure(); err != nil { + client.Close() + return nil, fmt.Errorf("error setting up infrastructure: %v", err) + } + + log.Println("Successfully connected to RabbitMQ and set up infrastructure") + return client, nil +} + +func (r *RabbitMQClient) setupInfrastructure() error { + // Declare dead letter exchange + err := r.Channel.ExchangeDeclare( + DeadLetterExchange, + "direct", + true, // durable + false, // auto-delete + false, // internal + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("error declaring dead letter exchange: %v", err) + } + + // Declare main exchange + err = r.Channel.ExchangeDeclare( + ScraperExchange, + "topic", + true, // durable + false, // auto-delete + false, // internal + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("error declaring exchange: %v", err) + } + + // Define queues with their routing keys + queues := map[string]string{ + LinkedInQueue: "scraper.linkedin", + IndeedQueue: "scraper.indeed", + BaytQueue: "scraper.bayt", + TokyoDevQueue: "scraper.tokyodev", + JapanDevQueue: "scraper.japandev", + GlassDoorQueue: "scraper.glassdoor", + GoogleQueue: "scraper.google", + } + + // Declare queues with dead letter exchange and TTL + for queueName, routingKey := range queues { + dlqName := queueName + "_dlq" + _, err := r.Channel.QueueDeclare( + dlqName, + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + nil, // arguments + ) + if err != nil { + return fmt.Errorf("error declaring dead letter queue %s: %v", dlqName, err) + } + + err = r.Channel.QueueBind( + dlqName, + queueName, // routing key is the original queue name + DeadLetterExchange, + false, + nil, + ) + if err != nil { + return fmt.Errorf("error binding dead letter queue %s: %v", dlqName, err) + } + + _, err = r.Channel.QueueDeclare( + queueName, + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + amqp.Table{ + "x-dead-letter-exchange": DeadLetterExchange, + "x-dead-letter-routing-key": queueName, + "x-message-ttl": int64(24 * time.Hour / time.Millisecond), // 24 hours TTL + "x-max-retries": 3, + }, + ) + if err != nil { + return fmt.Errorf("error declaring queue %s: %v", queueName, err) + } + + err = r.Channel.QueueBind( + queueName, + routingKey, + ScraperExchange, + false, + nil, + ) + if err != nil { + return fmt.Errorf("error binding queue %s: %v", queueName, err) + } + } + + return nil +} + +// Publish publishes a message to a specific routing key +func (r *RabbitMQClient) Publish(routingKey, exchange string, data []byte) error { + log.Printf("Publishing to exchange=%s, routingKey=%s, data=%s", ScraperExchange, routingKey, string(data)) + err := r.Channel.Publish( + ScraperExchange, + routingKey, + false, + false, + amqp.Publishing{ + ContentType: "application/json", + Body: data, + DeliveryMode: amqp.Persistent, + Timestamp: time.Now().UTC(), + }, + ) + if err != nil { + log.Printf("Error publishing: %v", err) + return fmt.Errorf("error publishing message: %v", err) + } + return nil +} + +// Subscribe creates a subscription to a specific queue with production-ready error handling +func (r *RabbitMQClient) Subscribe(queueName, consumerName string, handler MessageHandler) error { + err := r.Channel.Qos( + 1, // prefetch count + 0, // prefetch size + false, // global + ) + if err != nil { + return fmt.Errorf("error setting QoS: %v", err) + } + + // Start consuming messages + msgs, err := r.Channel.Consume( + queueName, + consumerName, + false, // auto-ack + false, // exclusive + false, // no-local + false, // no-wait + nil, // args + ) + if err != nil { + return fmt.Errorf("error starting consumer: %v", err) + } + + go func() { + for msg := range msgs { + // Call the handler + err := handler(msg.Body) + if err != nil { + log.Printf("Error processing message: %v", err) + // Reject and requeue the message (will go to DLX after max retries) + msg.Nack(false, true) + } else { + msg.Ack(false) + } + } + }() + + log.Printf("Successfully subscribed to queue: %s ", queueName) + return nil +} + +// Close closes the RabbitMQ connection and channel +func (r *RabbitMQClient) Close() { + if r.Channel != nil { + r.Channel.Close() + } + if r.Conn != nil { + r.Conn.Close() + } +} diff --git a/internal/pkg/openai/README.md b/internal/pkg/openai/README.md new file mode 100644 index 0000000..6a17f65 --- /dev/null +++ b/internal/pkg/openai/README.md @@ -0,0 +1,157 @@ +# OpenAI Service + +This package provides an OpenAI-powered service for job analysis and CV generation using the official OpenAI Go SDK. + +## Features + +- **Job Analysis**: Analyze job descriptions against CVs to determine fit and provide recommendations +- **CV Generation**: Create tailored CVs based on job descriptions +- **Cover Letter Generation**: Generate personalized cover letters + +## Setup + +1. Install the OpenAI Go SDK: +```bash +go get github.com/sashabaranov/go-openai +``` + +2. Set your OpenAI API key as an environment variable: +```bash +export OPENAI_API_KEY="your-api-key-here" +``` + +3. (Optional) Set a custom base URL: +```bash +export OPENAI_BASE_URL="https://your-custom-endpoint.com/v1" +``` + +## Usage + +### Basic Job Analysis + +```go +package main + +import ( + "fmt" + "os" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/openai" +) + +func main() { + // Create service + apiKey := os.Getenv("OPENAI_API_KEY") + service := openai.NewOpenAIService(apiKey, "gpt-4o-mini") + + // Prepare job description + jobDesc := domain.JobDescription{ + JobID: 1, + Description: "Looking for a Go developer...", + Criteria: map[string]string{ + "experience": "5+ years", + "languages": "Go, JavaScript", + }, + } + + // Analyze job + result, err := service.AnalyzeJobDescription(cv, jobDesc) + if err != nil { + panic(err) + } + + fmt.Printf("Should apply: %t\n", result.ShouldApply()) + fmt.Printf("Confidence: %d%%\n", result.ConfidenceScore) +} +``` + +### Using Custom Base URL + +You can configure a custom base URL in two ways: + +**Method 1: Environment Variable** +```bash +export OPENAI_BASE_URL="https://api.openai-proxy.com/v1" +# Service will automatically use this URL +service := openai.NewOpenAIService(apiKey, "gpt-4o-mini") +``` + +**Method 2: Direct Parameter** +```go +// Use a custom base URL directly +service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4o-mini", "https://api.openai-proxy.com/v1") + +// Or use Azure OpenAI +service := openai.NewOpenAIServiceWithBaseURL(apiKey, "gpt-4", "https://your-resource.openai.azure.com/") +``` + +### Generate Tailored CV + +```go +tailoredCV, err := service.GenerateCV(originalCV, jobDesc) +if err != nil { + panic(err) +} +fmt.Println(tailoredCV) +``` + +### Generate Cover Letter + +```go +coverLetter, err := service.GenerateCoverLetter(cv, jobDesc, "Tech Company Inc") +if err != nil { + panic(err) +} +fmt.Println(coverLetter) +``` + +## Models + +The service supports all OpenAI models. Common choices: + +- `gpt-4o-mini` - Cost-effective, good performance (default) +- `gpt-4o` - Higher quality, more expensive +- `gpt-3.5-turbo` - Fastest, most cost-effective + +## Environment Variables + +- `OPENAI_API_KEY` - Your OpenAI API key (required) +- `OPENAI_BASE_URL` - Custom base URL for OpenAI API (optional) +- `OPENAI_MODEL` - Default model to use (optional) + +## Response Structure + +### JobAnalysisResult + +```go +type JobAnalysisResult struct { + Recommendation string // "apply" or "do_not_apply" + ConfidenceScore int // 0-100 + MatchingSkills []string + MissingSkills []string + ExperienceMatch string // "excellent", "good", "fair", "poor" + Summary string + ImprovementSuggestions []string +} +``` + +## Error Handling + +The service handles common errors: +- API authentication issues +- Rate limiting +- Invalid JSON responses +- Network timeouts + +Always check for errors when calling service methods. + +## Testing + +Run the example test: +```bash +cd internal/pkg/openai +go test -v +``` + +Make sure to set `OPENAI_API_KEY` environment variable before running tests that make actual API calls. diff --git a/internal/pkg/openai/go.mod b/internal/pkg/openai/go.mod new file mode 100644 index 0000000..5790c22 --- /dev/null +++ b/internal/pkg/openai/go.mod @@ -0,0 +1,10 @@ +module github.com/jobs-scraper/internal/pkg/openai + +go 1.24.0 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000 + github.com/sashabaranov/go-openai v1.41.2 +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../domain diff --git a/internal/pkg/openai/go.sum b/internal/pkg/openai/go.sum new file mode 100644 index 0000000..17713c1 --- /dev/null +++ b/internal/pkg/openai/go.sum @@ -0,0 +1,2 @@ +github.com/sashabaranov/go-openai v1.41.2 h1:vfPRBZNMpnqu8ELsclWcAvF19lDNgh1t6TVfFFOPiSM= +github.com/sashabaranov/go-openai v1.41.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg= diff --git a/internal/pkg/openai/interface.go b/internal/pkg/openai/interface.go new file mode 100644 index 0000000..7a89640 --- /dev/null +++ b/internal/pkg/openai/interface.go @@ -0,0 +1,27 @@ +package openai + +import "github.com/jobs-scraper/internal/pkg/domain" + +// JobAnalysisService defines the interface for job analysis services +type JobAnalysisService interface { + // AnalyzeJobDescription analyzes a job description against a CV + AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) +} + +// CVService defines the interface for CV-related services +type CVService interface { + // GenerateCV creates a tailored CV based on a job description + GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error) + + // GenerateCoverLetter creates a cover letter based on CV and job description + GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error) +} + +// AIService combines all AI-related services +type AIService interface { + JobAnalysisService + CVService +} + +// Ensure OpenAIService implements AIService +var _ AIService = (*OpenAIService)(nil) diff --git a/internal/pkg/openai/openai.go b/internal/pkg/openai/openai.go new file mode 100644 index 0000000..7012a6c --- /dev/null +++ b/internal/pkg/openai/openai.go @@ -0,0 +1,357 @@ +package openai + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "strings" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/sashabaranov/go-openai" +) + +// JobAnalysisResult represents the structured response from job analysis +type JobAnalysisResult struct { + Recommendation string `json:"recommendation"` + ConfidenceScore int `json:"confidence_score"` + MatchingSkills []string `json:"matching_skills"` + MissingSkills []string `json:"missing_skills"` + ExperienceMatch string `json:"experience_match"` + Summary string `json:"summary"` + ImprovementSuggestions []string `json:"improvement_suggestions"` +} + +// ShouldApply returns true if the recommendation is to apply for the job +func (r *JobAnalysisResult) ShouldApply() bool { + return r.Recommendation == "apply" +} + +// IsHighConfidence returns true if the confidence score is 70 or above +func (r *JobAnalysisResult) IsHighConfidence() bool { + return r.ConfidenceScore >= 70 +} + +// OpenAIService provides job analysis using OpenAI's API +type OpenAIService struct { + client *openai.Client + model string +} + +// NewOpenAIService creates a new OpenAI service instance +func NewOpenAIService(apiKey, model string) *OpenAIService { + return NewOpenAIServiceWithBaseURL(apiKey, model, "") +} + +// NewOpenAIServiceWithBaseURL creates a new OpenAI service instance with a custom base URL +func NewOpenAIServiceWithBaseURL(apiKey, model, baseURL string) *OpenAIService { + if model == "" { + model = openai.GPT4oMini // Default to GPT-4o-mini for cost efficiency + } + + // Create client configuration + config := openai.DefaultConfig(apiKey) + + // Use provided baseURL, or check environment variable, or use default + if baseURL != "" { + config.BaseURL = baseURL + } else if envBaseURL := os.Getenv("OPENAI_BASE_URL"); envBaseURL != "" { + config.BaseURL = envBaseURL + } + + client := openai.NewClientWithConfig(config) + + return &OpenAIService{ + client: client, + model: model, + } +} + +// AnalyzeJobDescription analyzes a job description against a CV using OpenAI +func (s *OpenAIService) AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) { + // Build the user message with all the provided data + userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below. + 1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv. + for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills. + + 2) The job shouldn't require any language skills, preferably only english. + + 3) The job should be remote, or provide relocation to the country. + CV: + %s + + Job Description: + %s + + Job Criteria (key-value): + %v + + CRITICAL OUTPUT REQUIREMENTS: + - Return ONLY raw JSON - NO markdown formatting whatsoever + - NO backticks, NO code blocks, NO json prefix + - NO additional text before or after the JSON + - Start your response directly with { and end with } + - Use this exact schema and key names: + { + "recommendation": "apply" | "do_not_apply", + "confidence_score": number, // integer 0-100 + "matching_skills": [string], + "missing_skills": [string], + "experience_match": "excellent" | "good" | "fair" | "poor", + "summary": string, + "improvement_suggestions": [string] + }`, cv, jobDesc.Description, jobDesc.Criteria) + + // Create the chat completion request + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: userMessage, + }, + }, + Temperature: 0.3, // Lower temperature for more consistent responses + MaxTokens: 1000, + } + + // Execute the request + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return nil, fmt.Errorf("failed to create chat completion: %v", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response choices received from OpenAI API") + } + + // Extract JSON from the response + jsonContent := resp.Choices[0].Message.Content + + // Log the raw response for debugging + log.Printf("Raw OpenAI response: %s", jsonContent) + + // Clean up markdown code blocks if present + jsonContent = cleanMarkdownCodeBlocks(jsonContent) + + // Parse the JSON response into our struct + var result JobAnalysisResult + if err := json.Unmarshal([]byte(jsonContent), &result); err != nil { + return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(200, len(jsonContent))]) + } + + return &result, nil +} + +// GenerateCV creates a tailored CV based on a job description +func (s *OpenAIService) GenerateCV(originalCV string, jobDesc domain.JobDescription) (string, error) { + userMessage := fmt.Sprintf(`Based on the original CV and job description below, create a tailored CV that highlights relevant experience and skills for this specific position. + + Original CV: + %s + + Job Description: + %s + + Job Criteria: + %v + + Instructions: + 1. Keep all factual information accurate - do not fabricate experience or skills + 2. Reorganize and emphasize relevant experience and skills + 3. Use keywords from the job description where appropriate + 4. Maintain professional formatting + 5. Focus on achievements and quantifiable results + 6. Return the tailored CV in plain text format`, originalCV, jobDesc.Description, jobDesc.Criteria) + + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are an expert CV writer who specializes in tailoring resumes for specific job applications. You help candidates present their existing experience and skills in the most relevant way for each position, without fabricating information.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: userMessage, + }, + }, + Temperature: 0.7, // Higher temperature for more creative writing + MaxTokens: 2000, + } + + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return "", fmt.Errorf("failed to create chat completion: %v", err) + } + + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no response choices received from OpenAI API") + } + + return resp.Choices[0].Message.Content, nil +} + +// GenerateCoverLetter creates a cover letter based on CV and job description +func (s *OpenAIService) GenerateCoverLetter(cv string, jobDesc domain.JobDescription, companyName string) (string, error) { + userMessage := fmt.Sprintf(`Create a professional cover letter based on the CV and job description below. + + CV: + %s + + Job Description: + %s + + Job Criteria: + %v + + Company Name: %s + + Instructions: + 1. Write a compelling opening paragraph + 2. Highlight relevant experience and skills from the CV + 3. Show enthusiasm for the specific role and company + 4. Keep it concise (3-4 paragraphs) + 5. Use a professional but engaging tone + 6. Include a strong closing paragraph`, cv, jobDesc.Description, jobDesc.Criteria, companyName) + + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are an expert career counselor who writes compelling cover letters that help candidates stand out while maintaining professionalism.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: userMessage, + }, + }, + Temperature: 0.8, // Higher temperature for more creative and engaging writing + MaxTokens: 1500, + } + + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return "", fmt.Errorf("failed to create chat completion: %v", err) + } + + if len(resp.Choices) == 0 { + return "", fmt.Errorf("no response choices received from OpenAI API") + } + + return resp.Choices[0].Message.Content, nil +} + +// min returns the smaller of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// CreateJobDescription extracts structured job information from HTML content +func (s *OpenAIService) CreateJobDescription(htmlContent string) (*domain.JobDescription, error) { + prompt := fmt.Sprintf(`Extract job posting information from the following HTML content. + +CRITICAL RULES: +1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations +2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON) +3. All criteria values MUST be strings (never use booleans, numbers, or arrays) +4. Properly escape all quotes inside strings using \" +5. Use only standard ASCII quotes ("), never smart quotes (" " ' ') + +Required fields: +- description: Full job description as a single string +- criteria: Object with keys like title, company, location, salary, skills, experience, job_type, remote + +Example (SINGLE LINE): +{"description":"Full job description text here...","criteria":{"title":"Software Engineer","location":"City, Country","salary":"$100k-$150k","skills":"Go, Python, Docker","experience":"3-5 years","job_type":"Full-time","remote":"Remote"}} + +HTML Content: +%s`, htmlContent) + + req := openai.ChatCompletionRequest{ + Model: s.model, + Messages: []openai.ChatCompletionMessage{ + { + Role: openai.ChatMessageRoleSystem, + Content: "You are a JSON extraction bot. You ONLY output valid single-line JSON. No markdown. No code blocks. No explanations. No pretty printing. Just raw minified JSON starting with { and ending with }. Always use standard ASCII double quotes (\"), never smart quotes.", + }, + { + Role: openai.ChatMessageRoleUser, + Content: prompt, + }, + }, + Temperature: 0.1, // Low temperature for consistent structured output + MaxTokens: 2000, + } + + resp, err := s.client.CreateChatCompletion(context.Background(), req) + if err != nil { + return nil, fmt.Errorf("failed to execute completion: %v", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response choices received from API") + } + + jsonContent := resp.Choices[0].Message.Content + + log.Printf("Raw API response for job description: %s", jsonContent) + + jsonContent = cleanMarkdownCodeBlocks(jsonContent) + + log.Printf("Cleaned JSON (first 500 chars): %s", jsonContent[:min(500, len(jsonContent))]) + + var result struct { + Description string `json:"description"` + Criteria map[string]string `json:"criteria"` + } + + if err := json.Unmarshal([]byte(jsonContent), &result); err != nil { + if syntaxErr, ok := err.(*json.SyntaxError); ok { + start := int(syntaxErr.Offset) - 50 + if start < 0 { + start = 0 + } + end := int(syntaxErr.Offset) + 50 + if end > len(jsonContent) { + end = len(jsonContent) + } + log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end]) + } + return nil, fmt.Errorf("failed to parse JSON response: %v. Cleaned JSON length: %d", err, len(jsonContent)) + } + + jobDescription := &domain.JobDescription{ + Description: result.Description, + Criteria: result.Criteria, + } + + return jobDescription, nil +} + +// cleanMarkdownCodeBlocks removes markdown code block formatting from JSON response +func cleanMarkdownCodeBlocks(content string) string { + // Remove ```json prefix and ``` suffix if present + content = strings.TrimSpace(content) + + // Remove opening code block markers + if strings.HasPrefix(content, "```json") { + content = strings.TrimPrefix(content, "```json") + } else if strings.HasPrefix(content, "```") { + content = strings.TrimPrefix(content, "```") + } + + // Remove closing code block markers + content = strings.TrimSuffix(content, "```") + + return strings.TrimSpace(content) +} diff --git a/internal/pkg/openrouter/go.mod b/internal/pkg/openrouter/go.mod new file mode 100644 index 0000000..23a87a4 --- /dev/null +++ b/internal/pkg/openrouter/go.mod @@ -0,0 +1,12 @@ +module github.com/jobs-scraper/internal/pkg/openrouter + +go 1.24.0 + +require ( + github.com/eduardolat/openroutergo v0.1.0 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 +) + +require github.com/orsinium-labs/enum v1.4.0 // indirect + +replace github.com/jobs-scraper/internal/pkg/domain => ../domain diff --git a/internal/pkg/openrouter/go.sum b/internal/pkg/openrouter/go.sum new file mode 100644 index 0000000..9c8e70f --- /dev/null +++ b/internal/pkg/openrouter/go.sum @@ -0,0 +1,6 @@ +github.com/eduardolat/openroutergo v0.1.0 h1:ZD5pG0emgICeHKC4KCtEDD2BIW2LdhDKa2KMbbhrSxI= +github.com/eduardolat/openroutergo v0.1.0/go.mod h1:JVthRi3X9+DtJobL0QFeRqGdCYFj+02fJKbxEngaGAY= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= +github.com/orsinium-labs/enum v1.4.0 h1:3NInlfV76kuAg0kq2FFUondmg3WO7gMEgrPPrlzLDUM= +github.com/orsinium-labs/enum v1.4.0/go.mod h1:Qj5IK2pnElZtkZbGDxZMjpt7SUsn4tqE5vRelmWaBbc= diff --git a/internal/pkg/openrouter/openrouter.go b/internal/pkg/openrouter/openrouter.go new file mode 100644 index 0000000..ecdb8f6 --- /dev/null +++ b/internal/pkg/openrouter/openrouter.go @@ -0,0 +1,227 @@ +package openrouter + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + "github.com/eduardolat/openroutergo" + "github.com/jobs-scraper/internal/pkg/domain" +) + +// JobAnalysisResult represents the structured response from job analysis +type JobAnalysisResult struct { + Recommendation string `json:"recommendation"` + ConfidenceScore int `json:"confidence_score"` + MatchingSkills []string `json:"matching_skills"` + MissingSkills []string `json:"missing_skills"` + ExperienceMatch string `json:"experience_match"` + Summary string `json:"summary"` + ImprovementSuggestions []string `json:"improvement_suggestions"` +} + +// ShouldApply returns true if the recommendation is to apply for the job +func (r *JobAnalysisResult) ShouldApply() bool { + return r.Recommendation == "apply" +} + +// IsHighConfidence returns true if the confidence score is 70 or above +func (r *JobAnalysisResult) IsHighConfidence() bool { + return r.ConfidenceScore >= 70 +} + +type OpenRouterService struct { + model string + apiKey string +} + +func NewOpenRouterService(model string, apiKey string) OpenRouterService { + return OpenRouterService{ + model: model, + apiKey: apiKey, + } +} + +func (s *OpenRouterService) AnalyzeJobDescription(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) { + client, err := openroutergo. + NewClient(). + WithAPIKey(s.apiKey). + Create() + if err != nil { + log.Fatalf("Failed to create client: %v", err) + } + + // Build the user message with all the provided data + userMessage := fmt.Sprintf(`Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below. + 1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv. + for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills. + + 2) The job shouldn't require any language skills, preferbly only english. + + 3) The job should be remote, or provide relocation to the country. + CV: + %s + + Job Description: + %s + + Job Criteria (key-value): + %v + + CRITICAL OUTPUT REQUIREMENTS: + - Return ONLY raw JSON - NO markdown formatting whatsoever + - NO backticks, NO code blocks, NO json prefix + - NO additional text before or after the JSON + - Start your response directly with { and end with } + - Use this exact schema and key names: + { + "recommendation": "apply" | "do_not_apply", + "confidence_score": number, // integer 0-100 + "matching_skills": [string], + "missing_skills": [string], + "experience_match": "excellent" | "good" | "fair" | "poor", + "summary": string, + "improvement_suggestions": [string] + }`, cv, jobDesc.Description, jobDesc.Criteria) + + // Build and execute your request with a fluent API + _, resp, err := client. + NewChatCompletion(). + WithModel(s.model). + WithSystemMessage("You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }."). + WithUserMessage(userMessage). + Execute() + if err != nil { + return nil, fmt.Errorf("failed to execute completion: %v", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response choices received from API") + } + + // Extract JSON from the response (handle markdown code blocks) + jsonContent := resp.Choices[0].Message.Content + + // Log the raw response for debugging + log.Printf("Raw API response: %s", jsonContent) + + // Check if response looks like HTML/XML (common error response format) + if len(jsonContent) > 0 && jsonContent[0] == '<' { + return nil, fmt.Errorf("API returned HTML/XML instead of JSON. This usually indicates an API error, authentication issue, or rate limiting. Response: %s", jsonContent[:min(200, len(jsonContent))]) + } + + // Clean up markdown code blocks if present + jsonContent = cleanMarkdownCodeBlocks(jsonContent) + + // Parse the JSON response into our struct + var result JobAnalysisResult + if err := json.Unmarshal([]byte(jsonContent), &result); err != nil { + return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(200, len(jsonContent))]) + } + + return &result, nil +} + +// min returns the smaller of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// cleanMarkdownCodeBlocks removes only markdown code block formatting +func cleanMarkdownCodeBlocks(content string) string { + content = strings.TrimSpace(content) + + // Remove markdown code blocks + if strings.HasPrefix(content, "```json") { + content = strings.TrimPrefix(content, "```json") + } else if strings.HasPrefix(content, "```") { + content = strings.TrimPrefix(content, "```") + } + content = strings.TrimSuffix(content, "```") + + return strings.TrimSpace(content) +} + +func (s *OpenRouterService) CreateJobDescription(htmlContent string) (*domain.JobDescription, error) { + client, err := openroutergo. + NewClient(). + WithAPIKey(s.apiKey). + Create() + if err != nil { + log.Fatalf("Failed to create client: %v", err) + } + + prompt := fmt.Sprintf(`Extract job posting information from the following HTML content. + +CRITICAL RULES: +1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations +2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON) +3. All criteria values MUST be strings (never use booleans, numbers, or arrays) +4. Properly escape all quotes inside strings using \" +5. Use only standard ASCII quotes ("), never smart quotes (" " ' ') + +Required fields: +- description: Full job description as a single string +- criteria: Object with keys like title, company, location, salary, skills, experience, job_type, remote + +Example (SINGLE LINE): +{"description":"Full job description text here...","criteria":{"title":"Software Engineer","location":"City, Country","salary":"$100k-$150k","skills":"Go, Python, Docker","experience":"3-5 years","job_type":"Full-time","remote":"Remote"}} + +HTML Content: +%s`, htmlContent) + + _, resp, err := client. + NewChatCompletion(). + WithModel(s.model). + WithSystemMessage("You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }."). + WithUserMessage(prompt). + Execute() + if err != nil { + return nil, fmt.Errorf("failed to execute completion: %v", err) + } + + if len(resp.Choices) == 0 { + return nil, fmt.Errorf("no response choices received from API") + } + + jsonContent := resp.Choices[0].Message.Content + + log.Printf("Raw API response for job description: %s", jsonContent) + + jsonContent = cleanMarkdownCodeBlocks(jsonContent) + + log.Printf("Cleaned JSON (first 500 chars): %s", jsonContent[:min(500, len(jsonContent))]) + + var result struct { + Description string `json:"description"` + Criteria map[string]string `json:"criteria"` + } + + if err := json.Unmarshal([]byte(jsonContent), &result); err != nil { + if syntaxErr, ok := err.(*json.SyntaxError); ok { + start := int(syntaxErr.Offset) - 50 + if start < 0 { + start = 0 + } + end := int(syntaxErr.Offset) + 50 + if end > len(jsonContent) { + end = len(jsonContent) + } + log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end]) + } + } + + jobDescription := &domain.JobDescription{ + Description: result.Description, + Criteria: result.Criteria, + } + + return jobDescription, nil +} + +// func (s *OpenRouterService) CreateCV(cv string, jobDesc domain.JobDescription) (*JobAnalysisResult, error) { +// } diff --git a/internal/pkg/utils/go.mod b/internal/pkg/utils/go.mod new file mode 100644 index 0000000..1f9e86e --- /dev/null +++ b/internal/pkg/utils/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper/internal/pkg/utils + +go 1.24.0 + +toolchain go1.24.7 diff --git a/internal/pkg/utils/http.go b/internal/pkg/utils/http.go new file mode 100644 index 0000000..7ba79da --- /dev/null +++ b/internal/pkg/utils/http.go @@ -0,0 +1,87 @@ +package utils + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" +) + +// APIError represents an API error with status code and message +type APIError struct { + StatusCode int `json:"statusCode"` + Msg any `json:"msg"` +} + +// Error implements the error interface for APIError +func (e APIError) Error() string { + return fmt.Sprintf("api error: %d", e.StatusCode) +} + +// NewAPIError creates a new APIError with the given status code and error message +func NewAPIError(statusCode int, err error) APIError { + return APIError{ + StatusCode: statusCode, + Msg: err.Error(), + } +} + +// InvalidRequestData creates an APIError for validation errors with a map of field errors +func InvalidRequestData(msg string) APIError { + return APIError{ + StatusCode: http.StatusUnprocessableEntity, + Msg: msg, + } +} + +// InvalidJSON creates an APIError for invalid JSON format +func InvalidJSON() APIError { + return NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid JSON request data")) +} + +// APIResponse represents a standardized API response wrapper +type APIResponse struct { + Success bool `json:"success"` + Data any `json:"data,omitempty"` + Error any `json:"error,omitempty"` +} + +// WriteJSON writes the response as JSON with given status code, wrapped in a data object +func WriteJSON(w http.ResponseWriter, status int, data any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + + response := APIResponse{ + Success: status >= 200 && status < 300, + } + + if response.Success { + response.Data = data + } else { + response.Error = data + } + + json.NewEncoder(w).Encode(response) +} + +// APIFunc is the signature for API handler functions +type APIFunc func(w http.ResponseWriter, r *http.Request) error + +// Make wraps an APIFunc and handles errors consistently +func Make(h APIFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := h(w, r); err != nil { + if apiErr, ok := err.(APIError); ok { + WriteJSON(w, apiErr.StatusCode, apiErr) + } else { + errResp := map[string]any{ + "statusCode": http.StatusInternalServerError, + "msg": "internal server error", + } + WriteJSON(w, http.StatusInternalServerError, errResp) + // You could add logging here + slog.Error("HTTP API error", "err", err.Error(), "path", r.URL.Path) + } + } + } +} diff --git a/internal/pkg/utils/retryable-http-request.go b/internal/pkg/utils/retryable-http-request.go new file mode 100644 index 0000000..aa0bde1 --- /dev/null +++ b/internal/pkg/utils/retryable-http-request.go @@ -0,0 +1,425 @@ +package utils + +import ( + "context" + // "crypto/tls" + "fmt" + "io" + "net/http" + // "net/url" + "time" +) + +type RetryConfig struct { + MaxRetries int + BaseDelay time.Duration + MaxDelay time.Duration +} + +type RetryableHTTPRequestImpl struct { + client *http.Client + config RetryConfig +} + +func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl { + return NewRetryableHTTPRequestWithProxy(config, "") +} + +func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl { + // var transport *http.Transport + + // if proxyURL != "" { + // proxyUrl, err := url.Parse(proxyURL) + // if err != nil { + // panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err)) + // } + + // transport = &http.Transport{ + // Proxy: http.ProxyURL(proxyUrl), + // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + // } + // } + + client := &http.Client{ + Timeout: 30 * time.Second, + // Transport: transport, + } + + return &RetryableHTTPRequestImpl{ + client: client, + config: config, + } +} + +func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) { + + var lastErr error + + for attempt := 0; attempt <= s.config.MaxRetries; attempt++ { + req, err := http.NewRequestWithContext(ctx, method, url, body) + + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") + for _, header := range headers { + req.Header.Set(header.Get("Key"), header.Get("Value")) + } + + resp, err := s.client.Do(req) + if err != nil { + lastErr = err + fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err) + } else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices { + // Success + return resp, nil + } else if resp.StatusCode == http.StatusTooManyRequests { + // 429 Too Many Requests - retry with backoff + resp.Body.Close() + lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status) + fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode) + } else { + // All other errors (4xx, 5xx) - don't retry + resp.Body.Close() + return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status) + } + + // max attempts for now = 3 + if attempt < s.config.MaxRetries { + // Increment delay by 2 seconds for each attempt + delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ... + + // Cap the delay to MaxDelay if set + if s.config.MaxDelay > 0 && delay > s.config.MaxDelay { + delay = s.config.MaxDelay + } + + fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries) + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(delay): + // Continue to next attempt + } + } + } + + return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr) +} + +// 5.10.245.81:80 +// 172.67.70.206:80 +// 190.93.245.33:80 +// 185.221.160.176:80 +// 141.101.123.156:80 +// 103.21.244.129:80 +// 103.169.142.59:80 +// 172.67.81.185:80 +// 37.18.73.60:5566 +// 209.38.83.56:1088 +// 89.169.36.109:1080 +// 103.21.244.248:80 +// 172.67.70.92:80 +// 172.67.88.34:80 +// 141.101.120.38:80 +// 172.67.75.255:80 +// 172.67.75.171:80 +// 45.131.4.157:80 +// 141.101.121.208:80 +// 141.101.120.70:80 +// 103.21.244.70:80 +// 103.21.244.140:80 +// 103.21.244.138:80 +// 154.16.146.43:80 +// 128.199.202.122:3128 +// 160.153.0.28:80 +// 185.170.166.31:80 +// 172.67.70.95:80 +// 141.101.123.91:80 +// 141.101.121.81:80 +// 141.101.121.71:80 +// 141.101.120.224:80 +// 192.111.137.35:4145 +// 45.12.31.20:80 +// 103.21.244.150:80 +// 103.21.244.214:80 +// 103.21.244.29:80 +// 195.85.23.88:80 +// 23.227.39.134:80 +// 66.42.224.229:41679 +// 172.67.73.123:80 +// 79.110.200.27:8000 +// 5.182.34.33:80 +// 103.21.244.174:80 +// 172.67.75.203:80 +// 141.101.122.150:80 +// 141.101.120.35:80 +// 175.47.237.95:6128 +// 103.21.244.185:80 +// 141.101.122.86:80 +// 141.101.121.89:80 +// 103.21.244.105:80 +// 103.21.244.234:80 +// 172.67.70.20:80 +// 163.223.172.27:1080 +// 172.67.84.128:80 +// 103.21.244.57:80 +// 31.43.179.191:80 +// 173.245.59.61:80 +// 141.101.120.48:80 +// 185.162.230.19:80 +// 172.67.172.150:80 +// 159.112.235.63:80 +// 172.64.40.184:80 +// 103.21.244.55:80 +// 103.21.244.49:80 +// 172.67.70.228:80 +// 103.21.244.8:80 +// 66.235.200.77:80 +// 45.85.119.147:80 +// 172.67.177.231:80 +// 173.245.49.28:80 +// 110.232.92.49:8080 +// 212.183.88.212:80 +// 108.162.193.73:80 +// 185.238.228.29:80 +// 172.67.127.8:80 +// 164.38.155.86:80 +// 160.153.0.18:80 +// 141.101.122.119:80 +// 103.21.244.186:80 +// 172.67.172.162:80 +// 8.218.39.40:10800 +// 47.237.132.101:60031 +// 222.59.173.105:44193 +// 143.110.190.60:1080 +// 152.53.194.55:32059 +// 115.187.50.40:5678 +// 171.254.219.180:1080 +// 103.165.157.155:8080 +// 45.233.169.57:999 +// 222.59.173.105:45035 +// 202.40.179.18:4145 +// 170.79.181.188:60606 +// 37.186.66.36:3629 +// 104.200.152.30:4145 +// 106.13.58.110:8888 +// 202.40.181.220:31247 +// 43.135.36.240:80 +// 103.82.27.107:10001 +// 192.252.209.158:4145 +// 142.54.236.97:4145 +// 142.54.239.1:4145 +// 184.170.245.148:4145 +// 68.71.247.130:4145 +// 72.195.34.58:4145 +// 172.65.90.2:80 +// 172.65.90.0:80 +// 36.138.53.26:10019 +// 183.215.23.242:9091 +// 45.131.7.34:80 +// 141.101.120.230:80 +// 172.64.78.193:80 +// 141.101.122.9:80 +// 103.21.244.80:80 +// 103.21.244.147:80 +// 103.21.244.106:80 +// 103.21.244.16:80 +// 199.34.229.210:80 +// 141.101.90.98:80 +// 172.67.70.127:80 +// 103.21.244.221:80 +// 172.67.171.203:80 +// 172.67.181.188:80 +// 72.195.101.99:4145 +// 141.101.120.106:80 +// 103.21.244.180:80 +// 103.21.244.102:80 +// 172.67.70.233:80 +// 108.162.193.107:80 +// 45.131.208.112:80 +// 154.194.12.195:80 +// 69.84.182.23:80 +// 141.101.120.128:80 +// 141.101.120.149:80 +// 141.101.120.232:80 +// 141.101.120.6:80 +// 172.67.98.18:80 +// 141.101.113.20:80 +// 208.65.90.21:4145 +// 198.177.254.131:4145 +// 142.54.237.38:4145 +// 192.252.220.89:4145 +// 199.187.210.54:4145 +// 199.102.105.242:4145 +// 199.102.107.145:4145 +// 198.8.84.3:4145 +// 98.182.171.161:4145 +// 192.111.129.150:4145 +// 184.170.248.5:4145 +// 98.188.47.150:4145 +// 125.228.94.232:4145 +// 125.228.94.153:4145 +// 185.162.229.219:80 +// 103.21.244.22:80 +// 172.64.89.97:80 +// 103.21.244.54:80 +// 103.21.244.37:80 +// 141.101.123.225:80 +// 103.21.244.168:80 +// 103.21.244.149:80 +// 103.21.244.100:80 +// 103.21.244.24:80 +// 23.227.39.209:80 +// 172.67.83.15:80 +// 45.131.4.241:80 +// 141.101.120.201:80 +// 172.67.191.237:80 +// 172.67.127.188:80 +// 103.21.244.92:80 +// 103.160.204.22:80 +// 220.197.44.36:3128 +// 39.185.41.193:5911 +// 23.227.39.121:80 +// 139.162.78.109:80 +// 188.114.99.144:80 +// 45.131.5.37:80 +// 45.131.4.250:80 +// 23.227.39.65:80 +// 141.101.121.191:80 +// 172.67.188.16:80 +// 172.67.254.148:80 +// 221.1.104.177:7302 +// 222.59.173.105:44008 +// 103.21.244.83:80 +// 172.64.149.1:80 +// 45.12.30.22:80 +// 222.59.173.105:44027 +// 172.64.149.26:80 +// 103.21.244.46:80 +// 45.12.31.242:80 +// 23.227.38.195:80 +// 172.67.177.162:80 +// 103.160.204.200:80 +// 172.67.70.129:80 +// 141.101.123.245:80 +// 185.162.230.117:80 +// 185.162.230.183:80 +// 69.61.200.104:36181 +// 208.65.90.3:4145 +// 72.223.188.92:4145 +// 192.252.214.17:4145 +// 68.71.242.118:4145 +// 192.252.210.233:4145 +// 107.181.161.81:4145 +// 107.181.168.145:4145 +// 206.220.175.2:4145 +// 72.37.217.3:4145 +// 192.252.208.70:14282 +// 70.166.167.55:57745 +// 192.111.137.37:18762 +// 98.188.47.132:4145 +// 62.99.138.162:80 +// 172.64.90.186:80 +// 172.67.74.57:80 +// 45.131.6.67:80 +// 141.101.122.37:80 +// 103.21.244.134:80 +// 172.64.155.71:80 +// 5.182.34.139:80 +// 45.131.6.31:80 +// 141.101.121.21:80 +// 141.101.120.190:80 +// 103.21.244.161:80 +// 103.21.244.156:80 +// 103.21.244.151:80 +// 172.64.89.0:80 +// 58.216.109.17:800 +// 58.241.88.18:800 +// 36.147.78.166:80 +// 170.244.26.36:8888 +// 170.244.25.52:8888 +// 170.244.26.206:8888 +// 36.138.53.26:10017 +// 201.148.32.162:80 +// 203.19.38.114:1080 +// 31.43.179.204:80 +// 45.12.31.50:80 +// 170.244.27.142:8888 +// 170.244.26.195:8888 +// 170.244.27.58:8888 +// 170.244.27.61:8888 +// 170.244.27.150:8888 +// 47.243.94.125:1080 +// 40.177.65.8:80 +// 47.57.13.107:80 +// 194.158.203.14:80 +// 194.219.134.234:80 +// 103.21.244.51:80 +// 103.21.244.160:80 +// 103.21.244.144:80 +// 213.33.126.130:80 +// 103.21.244.69:80 +// 103.21.244.192:80 +// 103.21.244.133:80 +// 141.193.213.189:80 +// 68.71.254.6:4145 +// 185.221.160.21:80 +// 188.114.99.97:80 +// 63.141.128.94:80 +// 141.193.213.213:80 +// 185.162.228.234:80 +// 213.143.113.82:80 +// 154.194.12.207:80 +// 189.203.181.34:1080 +// 202.144.134.150:5678 +// 190.242.157.215:8080 +// 72.49.49.11:31034 +// 142.54.237.34:4145 +// 68.71.249.153:48606 +// 192.252.209.155:14455 +// 192.252.216.86:4145 +// 142.54.229.249:4145 +// 209.97.150.167:8080 +// 192.252.220.92:17328 +// 98.178.72.21:10919 +// 144.124.228.87:1080 +// 183.240.46.42:80 +// 32.223.6.94:80 +// 192.252.214.20:15864 +// 198.177.252.24:4145 +// 192.252.208.67:14287 +// 198.8.94.170:4145 +// 68.71.241.33:4145 +// 98.181.137.83:4145 +// 45.12.30.139:80 +// 103.21.244.97:80 +// 65.1.148.157:80 +// 199.58.184.97:4145 +// 141.101.120.148:80 +// 185.238.228.77:80 +// 172.67.202.134:80 +// 172.67.182.49:80 +// 45.131.210.112:80 +// 141.101.120.100:80 +// 103.21.244.59:80 +// 103.21.244.189:80 +// 103.21.244.154:80 +// 103.21.244.23:80 +// 173.245.49.40:80 +// 82.200.235.134:38191 +// 43.224.118.89:2626 +// 119.148.47.226:16464 +// 211.230.49.122:3128 +// 62.171.159.232:8888 +// 115.127.112.34:1080 +// 185.191.236.162:3128 +// 41.223.119.156:3128 +// 103.138.123.242:8082 +// 121.169.46.116:1090 +// 185.145.185.218:8080 +// 45.166.93.113:999 +// 163.53.204.178:9813 +// 181.224.226.154:8080 diff --git a/libs/ports/go.mod b/libs/ports/go.mod new file mode 100644 index 0000000..1bb16ea --- /dev/null +++ b/libs/ports/go.mod @@ -0,0 +1,9 @@ +module github.com/jobs-scraper/libs/ports + +go 1.24.0 + +toolchain go1.24.7 + +require github.com/jobs-scraper/internal/pkg/domain v0.0.0 + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain diff --git a/libs/ports/job-commands.go b/libs/ports/job-commands.go new file mode 100644 index 0000000..9a834c0 --- /dev/null +++ b/libs/ports/job-commands.go @@ -0,0 +1,21 @@ +package ports + +import "github.com/jobs-scraper/internal/pkg/domain" + +type JobCommands struct { + CreateJob JobCommandHandler +} + +// CreateJobCommand represents the command to create a job +type CreateJobCommand struct { + Location string + Keywords string + FWT string + Provider domain.JobProvider + NumPages int +} + +// JobCommandHandler defines the interface for handling job commands +type JobCommandHandler interface { + Handle(cmd CreateJobCommand) error +} diff --git a/libs/repo/go.mod b/libs/repo/go.mod new file mode 100644 index 0000000..18e388c --- /dev/null +++ b/libs/repo/go.mod @@ -0,0 +1,12 @@ +module github.com/jobs-scraper/libs/repo + +go 1.24.0 + +toolchain go1.24.7 + +require ( + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/lib/pq v1.10.9 +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain diff --git a/libs/repo/go.sum b/libs/repo/go.sum new file mode 100644 index 0000000..aeddeae --- /dev/null +++ b/libs/repo/go.sum @@ -0,0 +1,2 @@ +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= diff --git a/libs/repo/job-analysis-result.go b/libs/repo/job-analysis-result.go new file mode 100644 index 0000000..8ba0f28 --- /dev/null +++ b/libs/repo/job-analysis-result.go @@ -0,0 +1,152 @@ +package repo + +import ( + "database/sql" + "fmt" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/lib/pq" +) + +type JobAnalysisResultRepository struct { + db *sql.DB +} + +func NewJobAnalysisResultRepository(db *sql.DB) *JobAnalysisResultRepository { + return &JobAnalysisResultRepository{db: db} +} + +func (r *JobAnalysisResultRepository) SaveAnalysisResult(result *domain.JobAnalysisResult) error { + query := ` + INSERT INTO job_analysis_results (job_id, analysis_result, match_score, key_skills, missing_skills, recommendations) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, analyzed_at + ` + + err := r.db.QueryRow( + query, + result.JobID, + result.AnalysisResult, + result.MatchScore, + pq.Array(result.KeySkills), + pq.Array(result.MissingSkills), + result.Recommendations, + ).Scan(&result.ID, &result.AnalyzedAt) + + if err != nil { + return fmt.Errorf("error saving job analysis result: %v", err) + } + + return nil +} + +func (r *JobAnalysisResultRepository) GetAnalysisResultByJobID(jobID int64) (*domain.JobAnalysisResult, error) { + query := ` + SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at + FROM job_analysis_results + WHERE job_id = $1 + ORDER BY analyzed_at DESC + LIMIT 1 + ` + + var result domain.JobAnalysisResult + err := r.db.QueryRow(query, jobID).Scan( + &result.ID, + &result.JobID, + &result.AnalysisResult, + &result.MatchScore, + pq.Array(&result.KeySkills), + pq.Array(&result.MissingSkills), + &result.Recommendations, + &result.AnalyzedAt, + ) + + if err == sql.ErrNoRows { + return nil, fmt.Errorf("no analysis result found for job ID %d", jobID) + } + + if err != nil { + return nil, fmt.Errorf("error querying job analysis result: %v", err) + } + + return &result, nil +} + +func (r *JobAnalysisResultRepository) GetAllAnalysisResults() ([]domain.JobAnalysisResult, error) { + query := ` + SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at + FROM job_analysis_results + ORDER BY analyzed_at DESC + ` + + rows, err := r.db.Query(query) + if err != nil { + return nil, fmt.Errorf("error querying job analysis results: %v", err) + } + defer rows.Close() + + var results []domain.JobAnalysisResult + for rows.Next() { + var result domain.JobAnalysisResult + err := rows.Scan( + &result.ID, + &result.JobID, + &result.AnalysisResult, + &result.MatchScore, + pq.Array(&result.KeySkills), + pq.Array(&result.MissingSkills), + &result.Recommendations, + &result.AnalyzedAt, + ) + if err != nil { + return nil, fmt.Errorf("error scanning job analysis result: %v", err) + } + results = append(results, result) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err) + } + + return results, nil +} + +func (r *JobAnalysisResultRepository) GetAnalysisResultsByMatchScore(minScore int) ([]domain.JobAnalysisResult, error) { + query := ` + SELECT id, job_id, analysis_result, match_score, key_skills, missing_skills, recommendations, analyzed_at + FROM job_analysis_results + WHERE match_score >= $1 + ORDER BY match_score DESC, analyzed_at DESC + ` + + rows, err := r.db.Query(query, minScore) + if err != nil { + return nil, fmt.Errorf("error querying job analysis results by match score: %v", err) + } + defer rows.Close() + + var results []domain.JobAnalysisResult + for rows.Next() { + var result domain.JobAnalysisResult + err := rows.Scan( + &result.ID, + &result.JobID, + &result.AnalysisResult, + &result.MatchScore, + pq.Array(&result.KeySkills), + pq.Array(&result.MissingSkills), + &result.Recommendations, + &result.AnalyzedAt, + ) + if err != nil { + return nil, fmt.Errorf("error scanning job analysis result: %v", err) + } + results = append(results, result) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over job analysis result rows: %v", err) + } + + return results, nil +} diff --git a/libs/repo/job-description.go b/libs/repo/job-description.go new file mode 100644 index 0000000..f1cbc68 --- /dev/null +++ b/libs/repo/job-description.go @@ -0,0 +1,87 @@ +package repo + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + + "github.com/jobs-scraper/internal/pkg/domain" +) + +type JobDescriptionRepository struct { + db *sql.DB +} + +type JobDescriptionData struct { + JobID int + Description string + Criteria map[string]string +} + +func NewJobDescriptionRepository(db *sql.DB) *JobDescriptionRepository { + return &JobDescriptionRepository{db: db} +} + +func (r *JobDescriptionRepository) SaveJobDescriptions(jobDescriptions []domain.JobDescription) error { + if len(jobDescriptions) == 0 { + return nil + } + + // Build the VALUES clause dynamically + valueStrings := make([]string, 0, len(jobDescriptions)) + valueArgs := make([]interface{}, 0, len(jobDescriptions)*3) + + for i, jd := range jobDescriptions { + // Convert criteria map to JSONB + criteriaByte, err := json.Marshal(jd.Criteria) + if err != nil { + return fmt.Errorf("error marshaling job criteria for job %d: %v", jd.JobID, err) + } + + valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", i*3+1, i*3+2, i*3+3)) + valueArgs = append(valueArgs, jd.JobID, jd.Description, criteriaByte) + } + + sqlStatement := fmt.Sprintf(` + INSERT INTO job_descriptions (job_id, description, job_criteria) + VALUES %s + ON CONFLICT (job_id) DO UPDATE SET + description = EXCLUDED.description, + job_criteria = EXCLUDED.job_criteria, + updated_at = CURRENT_TIMESTAMP + `, strings.Join(valueStrings, ",")) + + _, err := r.db.Exec(sqlStatement, valueArgs...) + if err != nil { + return fmt.Errorf("error saving job descriptions: %v", err) + } + + return nil +} + +func (r *JobDescriptionRepository) GetJobDescriptionByJobID(jobID int64) (string, map[string]string, error) { + var ( + description string + criteriaByte []byte + criteria map[string]string + ) + + sqlStatement := `SELECT description, job_criteria FROM job_descriptions WHERE job_id = $1` + err := r.db.QueryRow(sqlStatement, jobID).Scan(&description, &criteriaByte) + if err != nil { + if err == sql.ErrNoRows { + return "", nil, nil // No description found + } + return "", nil, fmt.Errorf("error fetching job description: %v", err) + } + + // Unmarshal the JSON criteria + if criteriaByte != nil { + if err := json.Unmarshal(criteriaByte, &criteria); err != nil { + return "", nil, fmt.Errorf("error unmarshaling job criteria: %v", err) + } + } + + return description, criteria, nil +} diff --git a/libs/repo/job.go b/libs/repo/job.go new file mode 100644 index 0000000..9a03ed5 --- /dev/null +++ b/libs/repo/job.go @@ -0,0 +1,180 @@ +package repo + +import ( + "bytes" + "database/sql" + "fmt" + "strconv" + + "github.com/jobs-scraper/internal/pkg/domain" +) + +type JobRepository struct { + db *sql.DB +} + +func NewJobRepository(db *sql.DB) *JobRepository { + return &JobRepository{db: db} +} + +func (r *JobRepository) SaveJobs(jobs []domain.Job) error { + if len(jobs) == 0 { + return nil + } + + // Deduplicate jobs by ID to avoid duplicate errors + jobMap := make(map[int64]domain.Job) + for _, job := range jobs { + jobMap[job.ID] = job + } + + // Convert to slice of pointers for the bulk function + uniqueJobs := make([]*domain.Job, 0, len(jobMap)) + for _, job := range jobMap { + jobCopy := job // Important: create a copy to avoid pointer issues + uniqueJobs = append(uniqueJobs, &jobCopy) + } + + sqlTemplate := ` + INSERT INTO jobs (id, title, company, company_link, location, job_link, job_timestamp, provider) + VALUES %s + ON CONFLICT (job_link) DO UPDATE SET + id = EXCLUDED.id, + title = EXCLUDED.title, + company = EXCLUDED.company, + company_link = EXCLUDED.company_link, + location = EXCLUDED.location, + job_timestamp = EXCLUDED.job_timestamp, + provider = EXCLUDED.provider + ` + + sqlStatement, vals := prepareQueryCreateBulk(sqlTemplate, uniqueJobs) + + _, err := r.db.Exec(sqlStatement, vals...) + if err != nil { + return fmt.Errorf("error inserting jobs: %v", err) + } + + return nil +} + +func (r *JobRepository) GetAllJobs() ([]domain.Job, error) { + rows, err := r.db.Query("SELECT * FROM jobs") + if err != nil { + return nil, fmt.Errorf("error querying jobs: %v", err) + } + defer rows.Close() + + var jobs []domain.Job + for rows.Next() { + var job domain.Job + if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil { + return nil, fmt.Errorf("error scanning job row: %v", err) + } + jobs = append(jobs, job) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating over job rows: %v", err) + } + + return jobs, nil +} +func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) { + var job domain.Job + + sqlStatement := ` + SELECT id, title, company, company_link, location, job_link, job_timestamp + FROM jobs + WHERE id = $1 + ` + + err := r.db.QueryRow(sqlStatement, id).Scan( + &job.ID, + &job.Title, + &job.Company, + &job.CompanyLink, + &job.Location, + &job.JobLink, + &job.JobPostTime, + ) + + if err == sql.ErrNoRows { + return nil, fmt.Errorf("job with ID %d not found", id) + } + + if err != nil { + return nil, fmt.Errorf("error querying job: %v", err) + } + + return &job, nil +} + +func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) { + bf := bytes.Buffer{} + values := make([]interface{}, 0, len(models)*8) + + for i, v := range models { + values = append(values, v.ID, v.Title, v.Company, + v.CompanyLink, v.Location, v.JobLink, v.JobPostTime, v.Provider, + ) + + numFields := 8 + n := i * numFields + + bf.WriteString("(") + for j := 0; j < numFields; j++ { + bf.WriteString("$") + bf.WriteString(strconv.Itoa(n + j + 1)) + if j < numFields-1 { + bf.WriteString(", ") + } + } + bf.WriteString(")") + if i < len(models)-1 { + bf.WriteString(", ") + } + } + + return fmt.Sprintf(s, bf.String()), values +} + +func (r *JobRepository) GetJobsByStatus(status domain.JobStatus) ([]domain.Job, error) { + query := ` + SELECT id, title, company, company_link, location, job_link, job_timestamp + FROM jobs + WHERE status = $1 + ` + + rows, err := r.db.Query(query, int(status)) + if err != nil { + return nil, err + } + defer rows.Close() + + var jobs []domain.Job + for rows.Next() { + var job domain.Job + err := rows.Scan( + &job.ID, + &job.Title, + &job.Company, + &job.CompanyLink, + &job.Location, + &job.JobLink, + &job.JobPostTime, + ) + if err != nil { + return nil, err + } + jobs = append(jobs, job) + } + + return jobs, rows.Err() +} + +func (r *JobRepository) UpdateJobStatus(jobID int64, status domain.JobStatus) error { + query := `UPDATE jobs SET status = $1 WHERE id = $2` + _, err := r.db.Exec(query, int(status), jobID) + return err +} diff --git a/libs/server/go.mod b/libs/server/go.mod new file mode 100644 index 0000000..0108d78 --- /dev/null +++ b/libs/server/go.mod @@ -0,0 +1,5 @@ +module github.com/jobs-scraper/libs/server + +go 1.24.0 + +require github.com/gorilla/mux v1.8.1 diff --git a/libs/server/go.sum b/libs/server/go.sum new file mode 100644 index 0000000..7128337 --- /dev/null +++ b/libs/server/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= diff --git a/libs/server/server.go b/libs/server/server.go new file mode 100644 index 0000000..3f3ce34 --- /dev/null +++ b/libs/server/server.go @@ -0,0 +1,40 @@ +package server + +import ( + "context" + http "net/http" + "time" + + "github.com/gorilla/mux" +) + +type AppServer struct { + router *mux.Router + httpServer *http.Server +} + +func NewServer(router *mux.Router) *AppServer { + port := "8080" + + httpServer := &http.Server{ + Addr: ":" + port, + Handler: router, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + } + server := &AppServer{ + router: router, + httpServer: httpServer, + } + + return server +} + +func (s *AppServer) Start() error { + return s.httpServer.ListenAndServe() +} + +func (s *AppServer) Stop(ctx context.Context) { + s.httpServer.Shutdown(ctx) +} diff --git a/scripts/docker/docker-compose.yml b/scripts/docker/docker-compose.yml new file mode 100644 index 0000000..5b2f033 --- /dev/null +++ b/scripts/docker/docker-compose.yml @@ -0,0 +1,37 @@ +version: '3.8' + +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_DB: jobs_scraper + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + rabbitmq: + image: rabbitmq:3-management-alpine + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + ports: + - "5672:5672" + - "15672:15672" + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "ping"] + interval: 10s + timeout: 5s + retries: 5 +volumes: + postgres_data: + rabbitmq_data: diff --git a/services/api/.env b/services/api/.env new file mode 100644 index 0000000..ade07d5 --- /dev/null +++ b/services/api/.env @@ -0,0 +1,12 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/services/api/cmd/server/main.go b/services/api/cmd/server/main.go new file mode 100644 index 0000000..b44955a --- /dev/null +++ b/services/api/cmd/server/main.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gorilla/mux" + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/services/api/internal/app" + httpHandler "github.com/jobs-scraper/services/api/pkg/http" + _ "github.com/jobs-scraper/services/api/pkg/swagger" + "github.com/joho/godotenv" + httpSwagger "github.com/swaggo/http-swagger" +) + +// @title Jobs Scraper API +// @version 1.0 +// @description API for managing job scraping operations +// @BasePath / +func main() { + // Try to load .local.env first, then fallback to .env + if err := godotenv.Load(".local.env"); err != nil { + log.Println("No .local.env file found, trying .env") + if err := godotenv.Load(".env"); err != nil { + log.Println("No .env file found, using system environment variables") + } + } + + dbConfig := infrastructure.LoadConfigFromEnv() + + db, err := infrastructure.NewConnection(dbConfig) + if err != nil { + log.Fatal("Error connecting to db") + } + + err = db.Ping() + if err != nil { + log.Fatal("Error pinging db") + } + + log.Println("Successfully connected to db") + + rmq, err := rabbitmq.NewRabbitMQClient() + + if err != nil { + log.Fatal("Failed to create RabbitMQ client") + } + + defer rmq.Close() + + // Run database migrations + if err := infrastructure.RunMigrations(db); err != nil { + log.Fatalf("Failed to run migrations: %v", err) + } + + router := mux.NewRouter() + + // Swagger endpoint + router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) + + router.Use(httpHandler.CORSMiddleware) + router.Use(httpHandler.LogsMiddleware) + + app := app.NewApplication(router, db, rmq) + + // Create job analysis result repository + jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db) + + jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo) + + jobHandler.RegisterRoutes(router) + + go func() { + log.Printf("Server running on http %s\n", "8080") + if err := app.Server.Start(); err != nil && err != http.ErrServerClosed { + log.Fatalf("Could not listen on http %s: %v\n", "8080", err) + } + }() + + stopChan := make(chan os.Signal, 1) + signal.Notify( + stopChan, + os.Interrupt, + syscall.SIGTERM, + ) + + <-stopChan + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + app.Server.Stop(ctx) + + log.Println("Server shutting down") + os.Exit(0) +} diff --git a/services/api/go.mod b/services/api/go.mod new file mode 100644 index 0000000..1c207ab --- /dev/null +++ b/services/api/go.mod @@ -0,0 +1,47 @@ +module github.com/jobs-scraper/services/api + +go 1.24.0 + +require ( + github.com/gorilla/mux v1.8.1 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 + github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/jobs-scraper/libs/ports v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000 + github.com/joho/godotenv v1.5.1 + github.com/swaggo/http-swagger v1.3.4 + github.com/swaggo/swag v1.16.3 +) + +require ( + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/spec v0.20.6 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/mailru/easyjson v0.7.6 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/tools v0.24.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure + +replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils + +replace github.com/jobs-scraper/libs/ports => ../../libs/ports + +replace github.com/jobs-scraper/libs/repo => ../../libs/repo + +replace github.com/jobs-scraper/libs/server => ../../libs/server diff --git a/services/api/go.sum b/services/api/go.sum new file mode 100644 index 0000000..8512564 --- /dev/null +++ b/services/api/go.sum @@ -0,0 +1,134 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/spec v0.20.6 h1:ich1RQ3WDbfoeTqTAb+5EIxNmpKVJZWBNah9RAT0jIQ= +github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc= +github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w= +github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64a5ww= +github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/api/internal/app/app.go b/services/api/internal/app/app.go new file mode 100644 index 0000000..4b7e9ed --- /dev/null +++ b/services/api/internal/app/app.go @@ -0,0 +1,34 @@ +package app + +import ( + "database/sql" + + "github.com/gorilla/mux" + "github.com/jobs-scraper/services/api/internal/commands/job" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/libs/server" +) + +type Application struct { + Router *mux.Router + DB *sql.DB + Server *server.AppServer + JobCommands *ports.JobCommands +} + +func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient) *Application { + s := server.NewServer(router) + + jobRepo := repo.NewJobRepository(db) + + return &Application{ + Server: s, + Router: router, + DB: db, + JobCommands: &ports.JobCommands{ + CreateJob: job.NewCreateJobHandler(jobRepo, rmq), + }, + } +} diff --git a/services/api/internal/commands/job/create-job.go b/services/api/internal/commands/job/create-job.go new file mode 100644 index 0000000..d0a1bb7 --- /dev/null +++ b/services/api/internal/commands/job/create-job.go @@ -0,0 +1,61 @@ +package job + +import ( + "encoding/json" + "errors" + "log" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" +) + +type CreateJob struct { + jobRepo *repo.JobRepository + rmq *rabbitmq.RabbitMQClient +} + +func NewCreateJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *CreateJob { + return &CreateJob{ + jobRepo: jobRepo, + rmq: rmq, + } +} + +func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error { + jobRequest := domain.SearchQuery{ + Keywords: cmd.Keywords, + Location: cmd.Location, + FWT: cmd.FWT, + NumPages: cmd.NumPages, + } + + jsonData, err := json.Marshal(jobRequest) + if err != nil { + log.Printf("Error marshaling JSON: %v", err) + return err + } + + switch cmd.Provider { + case domain.LinkedIn: + if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil { + log.Printf("Error publishing LinkedIn message: %v", err) + return err + } + log.Printf("Published LinkedIn job request: %s", string(jsonData)) + case domain.Glassdoor: + if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil { + log.Printf("Error publishing Glassdoor message: %v", err) + return err + } + log.Printf("Published Glassdoor job request: %s", string(jsonData)) + default: + log.Printf("Unsupported job provider: %v", cmd.Provider) + return errors.New("unsupported job provider") + } + + return nil +} + +// sk-24a0fbede3a8464b85e22360165a3cc9 diff --git a/services/api/pkg/http/job.go b/services/api/pkg/http/job.go new file mode 100644 index 0000000..1dae58b --- /dev/null +++ b/services/api/pkg/http/job.go @@ -0,0 +1,148 @@ +package http + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "strconv" + + "github.com/jobs-scraper/libs/ports" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/internal/pkg/utils" + + "github.com/gorilla/mux" +) + +// JobHandler handles HTTP requests related to jobs +type JobHandler struct { + jobCommands *ports.JobCommands + jobAnalysisResultRepo *repo.JobAnalysisResultRepository +} + +// NewJobHandler creates a new instance of JobHandler +func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler { + return &JobHandler{ + jobCommands: jobCommands, + jobAnalysisResultRepo: jobAnalysisResultRepo, + } +} + +// RegisterRoutes registers all routes to the router +func (h *JobHandler) RegisterRoutes(router *mux.Router) { + jobs := router.PathPrefix("/jobs").Subrouter() + + //jobs.Use(AuthMiddleware) + + jobs.HandleFunc("", utils.Make(h.CreateJob)).Methods("POST") + jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET") + jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET") + jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET") +} + +// CreateJob handles the creation of a new job +// @Summary Create a new job +// @Description Creates a new job with the provided specifications +// @Tags jobs +// @Accept json +// @Produce json +// @Param job body ports.CreateJobCommand true "Job creation data" +// @Success 201 {object} map[string]string "Successfully created job" +// @Failure 400 {object} map[string]string "Invalid request data" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs [post] +func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error { + var cmd ports.CreateJobCommand + if err := json.NewDecoder(r.Body).Decode(&cmd); err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusBadRequest, utils.InvalidJSON()) + } + + err := h.jobCommands.CreateJob.Handle(cmd) + if err != nil { + slog.Error(err.Error()) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"}) + return nil +} + +// GetAllAnalysisResults handles retrieving all job analysis results +// @Summary Get all job analysis results +// @Description Retrieves all job analysis results from the database +// @Tags jobs +// @Produce json +// @Success 200 {array} domain.JobAnalysisResult "List of job analysis results" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/analysis [get] +func (h *JobHandler) GetAllAnalysisResults(w http.ResponseWriter, r *http.Request) error { + results, err := h.jobAnalysisResultRepo.GetAllAnalysisResults() + if err != nil { + slog.Error("Failed to get analysis results", "error", err) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusOK, results) + return nil +} + +// GetJobAnalysisResult handles retrieving analysis result for a specific job +// @Summary Get job analysis result by job ID +// @Description Retrieves the analysis result for a specific job +// @Tags jobs +// @Produce json +// @Param id path int true "Job ID" +// @Success 200 {object} domain.JobAnalysisResult "Job analysis result" +// @Failure 400 {object} map[string]string "Invalid job ID" +// @Failure 404 {object} map[string]string "Analysis result not found" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/{id}/analysis [get] +func (h *JobHandler) GetJobAnalysisResult(w http.ResponseWriter, r *http.Request) error { + vars := mux.Vars(r) + jobID, err := strconv.ParseInt(vars["id"], 10, 64) + if err != nil { + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID")) + } + + result, err := h.jobAnalysisResultRepo.GetAnalysisResultByJobID(jobID) + if err != nil { + slog.Error("Failed to get analysis result", "jobID", jobID, "error", err) + return utils.NewAPIError(http.StatusNotFound, fmt.Errorf("analysis result not found")) + } + + utils.WriteJSON(w, http.StatusOK, result) + return nil +} + +// GetTopMatches handles retrieving top matching jobs based on analysis score +// @Summary Get top matching jobs +// @Description Retrieves jobs with high analysis match scores +// @Tags jobs +// @Produce json +// @Param min_score query int false "Minimum match score (default: 70)" +// @Success 200 {array} domain.JobAnalysisResult "List of top matching job analysis results" +// @Failure 400 {object} map[string]string "Invalid minimum score" +// @Failure 500 {object} map[string]string "Internal server error" +// @Router /jobs/analysis/top-matches [get] +func (h *JobHandler) GetTopMatches(w http.ResponseWriter, r *http.Request) error { + minScoreStr := r.URL.Query().Get("min_score") + minScore := 70 // default minimum score + + if minScoreStr != "" { + var err error + minScore, err = strconv.Atoi(minScoreStr) + if err != nil { + return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid minimum score")) + } + } + + results, err := h.jobAnalysisResultRepo.GetAnalysisResultsByMatchScore(minScore) + if err != nil { + slog.Error("Failed to get top matches", "minScore", minScore, "error", err) + return utils.NewAPIError(http.StatusInternalServerError, err) + } + + utils.WriteJSON(w, http.StatusOK, results) + return nil +} diff --git a/services/api/pkg/http/middlewares.go b/services/api/pkg/http/middlewares.go new file mode 100644 index 0000000..3b6e0da --- /dev/null +++ b/services/api/pkg/http/middlewares.go @@ -0,0 +1,143 @@ +package http + +import ( + "context" + "encoding/json" + "log" + "net/http" + "strings" + "time" +) + +// CORSMiddleware handles Cross-Origin Resource Sharing +func CORSMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Set CORS headers + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With") + w.Header().Set("Access-Control-Allow-Credentials", "true") + w.Header().Set("Access-Control-Max-Age", "86400") + + // Handle preflight OPTIONS requests + if r.Method == "OPTIONS" { + w.WriteHeader(http.StatusOK) + return + } + + next.ServeHTTP(w, r) + }) +} + +// LogsMiddleware logs HTTP requests with timing information +func LogsMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + next.ServeHTTP(w, r) + log.Printf("%s %s %s", r.Method, r.RequestURI, time.Since(start)) + }) +} + +// AuthMiddleware handles JWT token validation, API keys, and basic auth +func AuthMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Skip auth for swagger endpoints and health checks + if strings.HasPrefix(r.URL.Path, "/swagger/") || r.URL.Path == "/health" { + next.ServeHTTP(w, r) + return + } + + // Try Bearer token authentication first + authHeader := r.Header.Get("Authorization") + if authHeader != "" { + if strings.HasPrefix(authHeader, "Bearer ") { + token := strings.TrimPrefix(authHeader, "Bearer ") + if token != "" { + userID, err := validateJWTToken(token) + if err == nil { + ctx := context.WithValue(r.Context(), "user_id", userID) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + } + } + + // Try API Key authentication + apiKey := r.Header.Get("X-API-Key") + if apiKey != "" { + if isValidAPIKey(apiKey) { + ctx := context.WithValue(r.Context(), "user_id", getUserIDFromAPIKey(apiKey)) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + + // Try Basic authentication + username, password, ok := r.BasicAuth() + if ok { + if isValidCredentials(username, password) { + ctx := context.WithValue(r.Context(), "username", username) + next.ServeHTTP(w, r.WithContext(ctx)) + return + } + } + + // Fallback to cookie-based auth (existing behavior) + token, err := r.Cookie("token") + if err == nil && token.Value == "123" { + next.ServeHTTP(w, r) + return + } + + // No valid authentication found + response := map[string]interface{}{ + "error": "Unauthorized", + "message": "Valid authentication required. Use Bearer token, API key, or basic auth.", + "code": http.StatusUnauthorized, + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + json.NewEncoder(w).Encode(response) + }) +} + +// Helper functions for authentication validation +func validateJWTToken(token string) (string, error) { + // TODO: Implement JWT validation logic with your preferred JWT library + // For demonstration, accept a demo token + if token == "valid-jwt-token" { + return "user-123", nil + } + return "", http.ErrAbortHandler +} + +func isValidAPIKey(apiKey string) bool { + // TODO: Implement API key validation logic + // For demonstration, accept a demo API key + return apiKey == "demo-api-key-123" +} + +func getUserIDFromAPIKey(apiKey string) string { + // TODO: Implement logic to get user ID from API key + // For demonstration, return a mock user ID + return "api-user-123" +} + +func isValidCredentials(username, password string) bool { + // TODO: Implement credential validation logic + // For demonstration, accept demo credentials + return username == "admin" && password == "password" +} + +// GetUserIDFromContext extracts user ID from request context +func GetUserIDFromContext(ctx context.Context) (string, bool) { + userID, ok := ctx.Value("user_id").(string) + return userID, ok +} + +// GetUsernameFromContext extracts username from request context +func GetUsernameFromContext(ctx context.Context) (string, bool) { + username, ok := ctx.Value("username").(string) + return username, ok +} diff --git a/services/api/pkg/swagger/docs.go b/services/api/pkg/swagger/docs.go new file mode 100644 index 0000000..1a64398 --- /dev/null +++ b/services/api/pkg/swagger/docs.go @@ -0,0 +1,307 @@ +// Package swagger Code generated by swaggo/swag. DO NOT EDIT +package swagger + +import "github.com/swaggo/swag" + +const docTemplate = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "contact": {}, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/jobs": { + "post": { + "description": "Creates a new job with the provided specifications", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Create a new job", + "parameters": [ + { + "description": "Job creation data", + "name": "job", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/ports.CreateJobCommand" + } + } + ], + "responses": { + "201": { + "description": "Successfully created job", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request data", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/jobs/analysis": { + "get": { + "description": "Retrieves all job analysis results from the database", + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Get all job analysis results", + "responses": { + "200": { + "description": "List of job analysis results", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/domain.JobAnalysisResult" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/jobs/analysis/top-matches": { + "get": { + "description": "Retrieves jobs with high analysis match scores", + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Get top matching jobs", + "parameters": [ + { + "type": "integer", + "description": "Minimum match score (default: 70)", + "name": "min_score", + "in": "query" + } + ], + "responses": { + "200": { + "description": "List of top matching job analysis results", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/domain.JobAnalysisResult" + } + } + }, + "400": { + "description": "Invalid minimum score", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/jobs/{id}/analysis": { + "get": { + "description": "Retrieves the analysis result for a specific job", + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Get job analysis result by job ID", + "parameters": [ + { + "type": "integer", + "description": "Job ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Job analysis result", + "schema": { + "$ref": "#/definitions/domain.JobAnalysisResult" + } + }, + "400": { + "description": "Invalid job ID", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Analysis result not found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + } + }, + "definitions": { + "domain.JobAnalysisResult": { + "type": "object", + "properties": { + "analysisResult": { + "type": "string" + }, + "analyzedAt": { + "type": "string" + }, + "id": { + "type": "integer", + "format": "int64" + }, + "jobID": { + "type": "integer", + "format": "int64" + }, + "keySkills": { + "type": "array", + "items": { + "type": "string" + } + }, + "matchScore": { + "type": "integer" + }, + "missingSkills": { + "type": "array", + "items": { + "type": "string" + } + }, + "recommendations": { + "type": "string" + } + } + }, + "domain.JobProvider": { + "type": "integer", + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "x-enum-varnames": [ + "LinkedIn", + "Indeed", + "Glassdoor", + "Bayt", + "TokyoDev", + "JapanDev" + ] + }, + "ports.CreateJobCommand": { + "type": "object", + "properties": { + "fwt": { + "type": "string" + }, + "keywords": { + "type": "string" + }, + "location": { + "type": "string" + }, + "numPages": { + "type": "integer" + }, + "provider": { + "$ref": "#/definitions/domain.JobProvider" + } + } + } + } +}` + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = &swag.Spec{ + Version: "1.0", + Host: "", + BasePath: "/", + Schemes: []string{}, + Title: "Jobs Scraper API", + Description: "API for managing job scraping operations", + InfoInstanceName: "swagger", + SwaggerTemplate: docTemplate, + LeftDelim: "{{", + RightDelim: "}}", +} + +func init() { + swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) +} diff --git a/services/api/pkg/swagger/swagger.json b/services/api/pkg/swagger/swagger.json new file mode 100644 index 0000000..c6c28fd --- /dev/null +++ b/services/api/pkg/swagger/swagger.json @@ -0,0 +1,282 @@ +{ + "swagger": "2.0", + "info": { + "description": "API for managing job scraping operations", + "title": "Jobs Scraper API", + "contact": {}, + "version": "1.0" + }, + "basePath": "/", + "paths": { + "/jobs": { + "post": { + "description": "Creates a new job with the provided specifications", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Create a new job", + "parameters": [ + { + "description": "Job creation data", + "name": "job", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/ports.CreateJobCommand" + } + } + ], + "responses": { + "201": { + "description": "Successfully created job", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "400": { + "description": "Invalid request data", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/jobs/analysis": { + "get": { + "description": "Retrieves all job analysis results from the database", + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Get all job analysis results", + "responses": { + "200": { + "description": "List of job analysis results", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/domain.JobAnalysisResult" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/jobs/analysis/top-matches": { + "get": { + "description": "Retrieves jobs with high analysis match scores", + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Get top matching jobs", + "parameters": [ + { + "type": "integer", + "description": "Minimum match score (default: 70)", + "name": "min_score", + "in": "query" + } + ], + "responses": { + "200": { + "description": "List of top matching job analysis results", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/domain.JobAnalysisResult" + } + } + }, + "400": { + "description": "Invalid minimum score", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + }, + "/jobs/{id}/analysis": { + "get": { + "description": "Retrieves the analysis result for a specific job", + "produces": [ + "application/json" + ], + "tags": [ + "jobs" + ], + "summary": "Get job analysis result by job ID", + "parameters": [ + { + "type": "integer", + "description": "Job ID", + "name": "id", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Job analysis result", + "schema": { + "$ref": "#/definitions/domain.JobAnalysisResult" + } + }, + "400": { + "description": "Invalid job ID", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "404": { + "description": "Analysis result not found", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "500": { + "description": "Internal server error", + "schema": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + } + } + } + }, + "definitions": { + "domain.JobAnalysisResult": { + "type": "object", + "properties": { + "analysisResult": { + "type": "string" + }, + "analyzedAt": { + "type": "string" + }, + "id": { + "type": "integer", + "format": "int64" + }, + "jobID": { + "type": "integer", + "format": "int64" + }, + "keySkills": { + "type": "array", + "items": { + "type": "string" + } + }, + "matchScore": { + "type": "integer" + }, + "missingSkills": { + "type": "array", + "items": { + "type": "string" + } + }, + "recommendations": { + "type": "string" + } + } + }, + "domain.JobProvider": { + "type": "integer", + "enum": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "x-enum-varnames": [ + "LinkedIn", + "Indeed", + "Glassdoor", + "Bayt", + "TokyoDev", + "JapanDev" + ] + }, + "ports.CreateJobCommand": { + "type": "object", + "properties": { + "fwt": { + "type": "string" + }, + "keywords": { + "type": "string" + }, + "location": { + "type": "string" + }, + "numPages": { + "type": "integer" + }, + "provider": { + "$ref": "#/definitions/domain.JobProvider" + } + } + } + } +} \ No newline at end of file diff --git a/services/api/pkg/swagger/swagger.yaml b/services/api/pkg/swagger/swagger.yaml new file mode 100644 index 0000000..7040064 --- /dev/null +++ b/services/api/pkg/swagger/swagger.yaml @@ -0,0 +1,189 @@ +basePath: / +definitions: + domain.JobAnalysisResult: + properties: + analysisResult: + type: string + analyzedAt: + type: string + id: + format: int64 + type: integer + jobID: + format: int64 + type: integer + keySkills: + items: + type: string + type: array + matchScore: + type: integer + missingSkills: + items: + type: string + type: array + recommendations: + type: string + type: object + domain.JobProvider: + enum: + - 0 + - 1 + - 2 + - 3 + - 4 + - 5 + type: integer + x-enum-varnames: + - LinkedIn + - Indeed + - Glassdoor + - Bayt + - TokyoDev + - JapanDev + ports.CreateJobCommand: + properties: + fwt: + type: string + keywords: + type: string + location: + type: string + numPages: + type: integer + provider: + $ref: '#/definitions/domain.JobProvider' + type: object +info: + contact: {} + description: API for managing job scraping operations + title: Jobs Scraper API + version: "1.0" +paths: + /jobs: + post: + consumes: + - application/json + description: Creates a new job with the provided specifications + parameters: + - description: Job creation data + in: body + name: job + required: true + schema: + $ref: '#/definitions/ports.CreateJobCommand' + produces: + - application/json + responses: + "201": + description: Successfully created job + schema: + additionalProperties: + type: string + type: object + "400": + description: Invalid request data + schema: + additionalProperties: + type: string + type: object + "500": + description: Internal server error + schema: + additionalProperties: + type: string + type: object + summary: Create a new job + tags: + - jobs + /jobs/{id}/analysis: + get: + description: Retrieves the analysis result for a specific job + parameters: + - description: Job ID + in: path + name: id + required: true + type: integer + produces: + - application/json + responses: + "200": + description: Job analysis result + schema: + $ref: '#/definitions/domain.JobAnalysisResult' + "400": + description: Invalid job ID + schema: + additionalProperties: + type: string + type: object + "404": + description: Analysis result not found + schema: + additionalProperties: + type: string + type: object + "500": + description: Internal server error + schema: + additionalProperties: + type: string + type: object + summary: Get job analysis result by job ID + tags: + - jobs + /jobs/analysis: + get: + description: Retrieves all job analysis results from the database + produces: + - application/json + responses: + "200": + description: List of job analysis results + schema: + items: + $ref: '#/definitions/domain.JobAnalysisResult' + type: array + "500": + description: Internal server error + schema: + additionalProperties: + type: string + type: object + summary: Get all job analysis results + tags: + - jobs + /jobs/analysis/top-matches: + get: + description: Retrieves jobs with high analysis match scores + parameters: + - description: 'Minimum match score (default: 70)' + in: query + name: min_score + type: integer + produces: + - application/json + responses: + "200": + description: List of top matching job analysis results + schema: + items: + $ref: '#/definitions/domain.JobAnalysisResult' + type: array + "400": + description: Invalid minimum score + schema: + additionalProperties: + type: string + type: object + "500": + description: Internal server error + schema: + additionalProperties: + type: string + type: object + summary: Get top matching jobs + tags: + - jobs +swagger: "2.0" diff --git a/services/scraper-glassdoor/.env b/services/scraper-glassdoor/.env new file mode 100644 index 0000000..2739b5b --- /dev/null +++ b/services/scraper-glassdoor/.env @@ -0,0 +1,12 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/services/scraper-glassdoor/db/db.ts b/services/scraper-glassdoor/db/db.ts new file mode 100644 index 0000000..c281440 --- /dev/null +++ b/services/scraper-glassdoor/db/db.ts @@ -0,0 +1,19 @@ +import postgres from "postgres"; +import dotenv from "dotenv"; + +if (dotenv.config({ path: ".local.env" }).error) { + console.log("No .local.env file found, trying .env"); + if (dotenv.config().error) { + console.log("No .env file found, using system environment variables"); + } +} + +const sql = postgres({ + host: process.env.DB_HOST, + port: Number(process.env.DB_PORT) || 5432, + database: process.env.DB_NAME, + username: process.env.DB_USER, + password: process.env.DB_PASSWORD, +}); + +export default sql; diff --git a/services/scraper-glassdoor/index.ts b/services/scraper-glassdoor/index.ts new file mode 100644 index 0000000..29e87e8 --- /dev/null +++ b/services/scraper-glassdoor/index.ts @@ -0,0 +1,184 @@ +import puppeteer, { Browser, Page } from "puppeteer-core"; +import { ChildProcess } from "child_process"; +import * as dotenv from "dotenv"; +import { + ChromeVersionResponse, + launchChromeWithDebugging, + randomDelay, + waitForChromeReady, +} from "./utils/browser"; +import { + RabbitMQClient, + SearchQuery, + createRabbitMQClient, +} from "./utils/rabbitmq"; +import { BrowserService } from "./services/browserService"; +import { saveJobs } from "./queries/save-jobs"; +import { saveJobDescriptions } from "./queries/save-job-description"; + +if (dotenv.config({ path: ".local.env" }).error) { + console.log("No .local.env file found, trying .env"); + if (dotenv.config().error) { + console.log("No .env file found, using system environment variables"); + } +} + +let globalBrowser: Browser | null = null; +let chrome: ChildProcess; + +async function cleanup() { + console.log("Starting cleanup process..."); + + if (rabbitMQClient) { + console.log("Closing RabbitMQ connection"); + await rabbitMQClient.close(); + rabbitMQClient = null; + } + + if (globalBrowser) { + console.log("Closing browser on app shutdown"); + await globalBrowser.close(); + globalBrowser = null; + } + + if (chrome) { + chrome.kill("SIGINT"); + } + + console.log("Cleanup completed"); +} + +async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { + chrome = launchChromeWithDebugging(); + await waitForChromeReady(); + + const response = await fetch("http://localhost:9222/json/version", { + headers: { + Origin: "", + }, + }); + + if (!response.ok) { + throw new Error("Failed to connect to Chrome debugging port"); + } + + const chromeInstance = (await response.json()) as ChromeVersionResponse; + const { webSocketDebuggerUrl } = chromeInstance; + + const browser = await puppeteer.connect({ + browserWSEndpoint: webSocketDebuggerUrl, + defaultViewport: null, + }); + + globalBrowser = browser; + console.log("Connected to browser successfully"); + + let pages = await browser.pages(); + console.log(`Found ${pages.length} pages`); + + let existingcookies = await browser.cookies(); + await browser.deleteCookie(...existingcookies); + + let page: Page; + if (pages.length === 0) { + console.log("No pages found, creating a new page"); + page = await browser.newPage(); + } else { + page = pages[0]; + } + + await page.setUserAgent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ); + + return { browser, page }; +} + +async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise { + try { + const { page } = await initializeBrowser(); + const browserService = new BrowserService(page); + + const searchUrl = `https://www.glassdoor.com/Job/jobs.htm`; + + await page.goto(searchUrl, { + waitUntil: ["networkidle2", "domcontentloaded"], + timeout: 60000, + }); + + await randomDelay(1500, 2500); + + await browserService.searchJobs(searchQuery.keywords, searchQuery.location); + const jobs = await browserService.extractJobListings(searchQuery.location); + + if (jobs.length > 0) { + await saveJobs(jobs); + await saveJobDescriptions(jobs); + + console.log( + `saved ${jobs.length} jobs from Glassdoor for query:`, + searchQuery + ); + } else { + console.log("No jobs found for this search query"); + } + + await randomDelay(1500, 2500); + } catch (error) { + console.error("Error scraping Glassdoor jobs:", error); + throw error; + } + + if (globalBrowser) { + console.log("Closing browser on app shutdown"); + await globalBrowser.close(); + globalBrowser = null; + } + + if (chrome) { + chrome.kill("SIGINT"); + } +} + +let rabbitMQClient: RabbitMQClient | null = null; + +async function subscribeToGlassdoorQueue(): Promise { + try { + rabbitMQClient = await createRabbitMQClient(); + + await rabbitMQClient.subscribe(async (searchQuery: SearchQuery) => { + await scrapeGlassdoorJobs(searchQuery); + }); + } catch (error) { + console.error("Failed to set up RabbitMQ subscription:", error); + process.exit(1); + } +} + +async function main() { + console.log("Starting Glassdoor scraper service..."); + await subscribeToGlassdoorQueue(); +} + +process.on("SIGINT", async () => { + console.log("Received SIGINT signal"); + await cleanup(); + if (chrome) { + chrome.kill("SIGINT"); + } + process.exit(0); +}); + +process.on("SIGTERM", async () => { + console.log("Received SIGTERM signal"); + await cleanup(); + if (chrome) { + chrome.kill("SIGTERM"); + } + process.exit(0); +}); + +main().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); diff --git a/services/scraper-glassdoor/package-lock.json b/services/scraper-glassdoor/package-lock.json new file mode 100644 index 0000000..b8db1e5 --- /dev/null +++ b/services/scraper-glassdoor/package-lock.json @@ -0,0 +1,1104 @@ +{ + "name": "glassdoor-scraper", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "glassdoor-scraper", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@types/amqplib": "^0.10.7", + "amqplib": "^0.10.9", + "dotenv": "^17.2.3", + "postgres": "^3.4.7", + "puppeteer-core": "^24.25.0" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.12", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.12.tgz", + "integrity": "sha512-mP9iLFZwH+FapKJLeA7/fLqOlSUwYpMwjR1P5J23qd4e7qGJwecJccJqHYrjw33jmIZYV4dtiTHPD/J+1e7cEw==", + "dependencies": { + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.3", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, + "node_modules/@types/amqplib": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz", + "integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "24.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz", + "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/amqplib": { + "version": "0.10.9", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz", + "integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==", + "dependencies": { + "buffer-more-ints": "~1.0.0", + "url-parse": "~1.5.10" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/b4a": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.7.3.tgz", + "integrity": "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/bare-events": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.1.tgz", + "integrity": "sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.0.tgz", + "integrity": "sha512-GljgCjeupKZJNetTqxKaQArLK10vpmK28or0+RwWjEl5Rk+/xG3wkpmkv+WrcBm3q1BwHKlnhXzR8O37kcvkXQ==", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz", + "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz", + "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz", + "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", + "optional": true, + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, + "node_modules/chromium-bidi": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-10.5.1.tgz", + "integrity": "sha512-rlj6OyhKhVTnk4aENcUme3Jl9h+cq4oXu4AzBcvr8RMmT6BR4a3zSNT9dbIfXr9/BS6ibzRyDhowuw4n2GgzsQ==", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1521046", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1521046.tgz", + "integrity": "sha512-vhE6eymDQSKWUXwwA37NtTTVEzjtGVfDr3pRbsWEQ5onH/Snp2c+2xZHWJJawG/0hCCJLRGt4xVtEVUVILol4w==" + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "node_modules/postgres": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.7.tgz", + "integrity": "sha512-Jtc2612XINuBjIl/QTWsV5UvE8UHuNblcO3vVADSrKsrc6RqGX6lOW1cEo3CM2v0XG4Nat8nI+YM7/f26VxXLw==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer-core": { + "version": "24.27.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.27.0.tgz", + "integrity": "sha512-yubwj2XXmTM3wRIpbhO5nCjbByPgpFHlgrsD4IK+gMPqO7/a5FfnoSXDKjmqi8A2M1Ewusz0rTI/r+IN0GU0MA==", + "dependencies": { + "@puppeteer/browsers": "2.10.12", + "chromium-bidi": "10.5.1", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1521046", + "typed-query-selector": "^2.12.0", + "webdriver-bidi-protocol": "0.3.8", + "ws": "^8.18.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/streamx": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==" + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.8.tgz", + "integrity": "sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ==" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/services/scraper-glassdoor/package.json b/services/scraper-glassdoor/package.json new file mode 100644 index 0000000..f24d6f4 --- /dev/null +++ b/services/scraper-glassdoor/package.json @@ -0,0 +1,27 @@ +{ + "name": "glassdoor-scraper", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node index.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@types/amqplib": "^0.10.7", + "amqplib": "^0.10.9", + "dotenv": "^17.2.3", + "postgres": "^3.4.7", + "puppeteer-core": "^24.25.0" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } +} diff --git a/services/scraper-glassdoor/queries/save-job-description.ts b/services/scraper-glassdoor/queries/save-job-description.ts new file mode 100644 index 0000000..7a2dc7d --- /dev/null +++ b/services/scraper-glassdoor/queries/save-job-description.ts @@ -0,0 +1,33 @@ +import sql from "../db/db"; +import { Job } from "../types/job"; + +export const saveJobDescriptions = async (jobs: Job[]) => { + if (jobs.length === 0) { + console.log("No jobs to save descriptions for."); + return; + } + + try { + const jobDescriptionsForDb = jobs.map((job) => ({ + job_id: job.id, + description: job.description, + job_criteria: job.criteria, + })); + + await sql` + INSERT INTO public.job_descriptions ${sql( + jobDescriptionsForDb, + "job_id", + "description", + "job_criteria" + )} + ON CONFLICT (job_id) DO UPDATE SET + description = EXCLUDED.description, + job_criteria = EXCLUDED.job_criteria, + updated_at = CURRENT_TIMESTAMP + `; + console.log(`Updated descriptions for ${jobs.length} jobs.`); + } catch (error) { + console.error("Error updating job descriptions:", error); + } +}; diff --git a/services/scraper-glassdoor/queries/save-jobs.ts b/services/scraper-glassdoor/queries/save-jobs.ts new file mode 100644 index 0000000..dc88deb --- /dev/null +++ b/services/scraper-glassdoor/queries/save-jobs.ts @@ -0,0 +1,52 @@ +import sql from "../db/db"; +import { Job } from "../types/job"; + +export const saveJobs = async (jobs: Job[]) => { + if (jobs.length === 0) { + console.log("No jobs to save."); + return; + } + + try { + const jobsForDb = jobs.map((job) => ({ + id: job.id, + title: job.title, + company: job.company, + location: job.location, + company_link: job.companyLink, + job_link: job.jobLink, + provider: job.provider, + status: job.status, + job_timestamp: + job.jobPostTime instanceof Date && !isNaN(job.jobPostTime.getTime()) + ? job.jobPostTime + : new Date(), + })); + + const result = await sql` + INSERT INTO public.jobs ${sql( + jobsForDb, + "id", + "title", + "company", + "location", + "company_link", + "job_link", + "provider", + "status", + "job_timestamp" + )} + ON CONFLICT (id) DO UPDATE SET + title = EXCLUDED.title, + company = EXCLUDED.company, + company_link = EXCLUDED.company_link, + location = EXCLUDED.location, + job_link = EXCLUDED.job_link, + job_timestamp = EXCLUDED.job_timestamp + `; + + console.log(`Saved ${jobs.length} jobs to the database.`); + } catch (error) { + console.error("Error saving jobs to the database:", error); + } +}; diff --git a/services/scraper-glassdoor/services/browserService.ts b/services/scraper-glassdoor/services/browserService.ts new file mode 100644 index 0000000..65b7bf6 --- /dev/null +++ b/services/scraper-glassdoor/services/browserService.ts @@ -0,0 +1,185 @@ +import { Page } from "puppeteer-core"; +import { randomDelay } from "../utils/browser"; +import { Job } from "../types/job"; + +export class BrowserService { + private page: Page; + + constructor(page: Page) { + this.page = page; + } + + async searchJobs(keywords: string, location: string) { + try { + const keywordInput = await this.page.waitForSelector( + "#searchBar-jobTitle, input[name='sc.keyword'], input[placeholder*='Job title']", + { timeout: 10000 } + ); + + if (keywordInput) { + await keywordInput.click({ clickCount: 3 }); + await keywordInput.type(keywords, { delay: 130 }); + await randomDelay(1000, 2000); + } + + const locationInput = await this.page.waitForSelector( + "#searchBar-location, input[name='locKeyword'], input[placeholder*='Location']", + { timeout: 5000 } + ); + + if (locationInput) { + await locationInput.click({ clickCount: 3 }); + await locationInput.type(location, { delay: 130 }); + await randomDelay(1000, 2000); + await locationInput.press("Enter"); + + await this.page.waitForNavigation({ + waitUntil: "networkidle2", + timeout: 30000, + }); + } + } catch (error) { + console.log( + "Search form not found, attempting to extract jobs from current page..." + ); + } + } + + async extractJobListings(location: string): Promise { + try { + await this.page.waitForSelector( + 'li[data-test="jobListing"], .JobsList_jobListItem__wjTHv, [data-jobid]', + { timeout: 10000 } + ); + } catch (error) { + console.log("No job listings found on the page"); + return []; + } + + const jobs = await this.page.evaluate((location) => { + const jobElements = document.querySelectorAll(` + li[data-test="jobListing"], + .JobsList_jobListItem__wjTHv, + li[data-jobid], + .react-job-listing + `) as NodeListOf; + + const jobs: any[] = []; + console.log("Found job elements:", jobElements.length); + + jobElements.forEach((element, index) => { + try { + const jobId = + element.getAttribute("data-jobid") || + element.getAttribute("data-brandviews")?.match(/jlid=(\d+)/)?.[1] || + `job-${index + 1}`; + + const titleElement = element.querySelector(` + a[data-test="job-title"], + .JobCard_jobTitle__GLyJ1, + .jobLink, + a[id*="job-title"] + `) as HTMLAnchorElement; + + const companyElement = element.querySelector(` + .EmployerProfile_compactEmployerName__9MGcV, + [data-test="employer-name"] a, + .employerName, + .EmployerProfile_employerNameContainer__ptolz span + `) as HTMLElement; + + const descriptionElement = element.querySelector(` + .JobDetails_jobDescription__uW_fK, + .JobDetails_showHidden__C_FOA, + [class*="jobDescription"] + `) as HTMLElement; + + const jobPostingTimeElement = element.querySelector(` + .JobCard_listingAge__jJsuc, + [data-test="job-age"], + .listingAge + `) as HTMLElement; + + const title = titleElement?.textContent?.trim() || ""; + const company = companyElement?.textContent?.trim() || ""; + const description = descriptionElement?.textContent?.trim() || ""; + const jobPosting = jobPostingTimeElement?.textContent?.trim() || ""; + + const jobLink = titleElement?.href + ? titleElement.href.startsWith("http") + ? titleElement.href + : `https://www.glassdoor.com${titleElement.href}` + : ""; + + const companyLink = + companyElement?.tagName === "A" + ? (companyElement as HTMLAnchorElement).href?.startsWith("http") + ? (companyElement as HTMLAnchorElement).href + : `https://www.glassdoor.com${ + (companyElement as HTMLAnchorElement).href + }` + : ""; + + let jobPostingDate = new Date(); + + try { + if (jobPosting.includes("d")) { + const match = jobPosting.match(/(\d+)\s*d/); + if (match) { + const daysAgo = parseInt(match[1], 10); + if (!isNaN(daysAgo) && daysAgo >= 0) { + jobPostingDate.setDate(jobPostingDate.getDate() - daysAgo); + } + } + } else if (jobPosting.includes("h")) { + const match = jobPosting.match(/(\d+)\s*h/); + if (match) { + const hoursAgo = parseInt(match[1], 10); + if (!isNaN(hoursAgo) && hoursAgo >= 0) { + jobPostingDate.setHours(jobPostingDate.getHours() - hoursAgo); + } + } + } + + if (isNaN(jobPostingDate.getTime())) { + jobPostingDate = new Date(); + } + } catch (error) { + console.warn( + `Failed to parse job posting date: "${jobPosting}", using current date` + ); + jobPostingDate = new Date(); + } + + const parsedJobId = Number.parseInt(jobId, 10); + + jobs.push({ + title, + company, + location, + description: description, + criteria: {}, + jobLink, + companyLink, + provider: 2, + id: parsedJobId, + jobPostTime: jobPostingDate.toISOString(), + status: 0, + }); + } catch (error) { + console.error(`Error extracting job ${index}:`, error); + } + }); + + console.log("Extracted jobs:", jobs.length); + return jobs; + }, location); + + console.log(`Found ${jobs.length} jobs on Glassdoor`); + + return jobs.map((job) => ({ + ...job, + jobPostTime: new Date(job.jobPostTime), + })); + } +} diff --git a/services/scraper-glassdoor/tsconfig.json b/services/scraper-glassdoor/tsconfig.json new file mode 100644 index 0000000..651cacf --- /dev/null +++ b/services/scraper-glassdoor/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020", "DOM"], + "outDir": "./dist", + "rootDir": "./", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "removeComments": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "exactOptionalPropertyTypes": false, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": false, + "noUncheckedIndexedAccess": false + }, + "include": [ + "**/*.ts", + "**/*.tsx" + ], + "exclude": [ + "node_modules", + "dist" + ] +} diff --git a/services/scraper-glassdoor/types/job.ts b/services/scraper-glassdoor/types/job.ts new file mode 100644 index 0000000..8cd59c9 --- /dev/null +++ b/services/scraper-glassdoor/types/job.ts @@ -0,0 +1,13 @@ +export interface Job { + id: number; + title: string; + company: string; + location: string; + description: string; + criteria: Record; + jobLink: string; + companyLink: string; + provider: 2; + jobPostTime: string; + status: number; +} diff --git a/services/scraper-glassdoor/utils/browser.ts b/services/scraper-glassdoor/utils/browser.ts new file mode 100644 index 0000000..df532d7 --- /dev/null +++ b/services/scraper-glassdoor/utils/browser.ts @@ -0,0 +1,139 @@ +import fs from "fs"; +import path from "path"; +import os from "os"; +import { spawn } from "child_process"; +import net from "net"; + +export interface ChromeVersionResponse { + webSocketDebuggerUrl: string; +} + +const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile"); + +export const getLocalISOString = (date = new Date()) => { + const offset = date.getTimezoneOffset(); + const offsetAbs = Math.abs(offset); + const hours = Math.floor(offsetAbs / 60) + .toString() + .padStart(2, "0"); + const minutes = (offsetAbs % 60).toString().padStart(2, "0"); + const sign = offset > 0 ? "-" : "+"; + + return new Date(date.getTime() - offset * 60000) + .toISOString() + .replace("Z", `${sign}${hours}:${minutes}`); +}; + +function getChromePath(): string { + const platform = os.platform(); + + const paths: Record = { + darwin: [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ], + win32: [ + process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe", + process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe", + process.env["PROGRAMFILES(X86)"] + + "\\Google\\Chrome\\Application\\chrome.exe", + ], + linux: [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + ], + aix: [], + freebsd: [], + openbsd: [], + sunos: [], + android: [], + haiku: [], + cygwin: [], + netbsd: [], + }; + + const platformPaths = paths[platform] || []; + + for (const path of platformPaths) { + try { + if (fs.existsSync(path)) { + return path; + } + } catch (err) { + continue; + } + } + + throw new Error("Could not find Chrome/Chromium installation"); +} + +export function launchChromeWithDebugging() { + try { + const chromePath = getChromePath(); + const args = [ + "--remote-debugging-port=9222", + `--user-data-dir=${userDataDir}`, + "--remote-allow-origins=*", + "--incognito", + // "--headless", + // `--proxy-server=46.3.202.240:2120`, + // `--proxy-server=http://jN8AqiMh:Pd9FEs6N@212.193.184.228:64616`, + ]; + + console.log(`Launching Chrome at: ${chromePath}`); + + const chromeProcess = spawn(chromePath, args, { + detached: true, + stdio: "ignore", + }); + + chromeProcess.unref(); + + chromeProcess.on("error", (err) => { + console.error("Failed to start Chrome:", err); + }); + + chromeProcess.on("exit", (code) => { + if (code !== 0) { + console.error(`Chrome process exited with code ${code}`); + } + }); + + console.log("Chrome launched successfully with debugging port 9222"); + return chromeProcess; + } catch (error) { + console.error("Failed to launch Chrome:", error); + process.exit(1); + } +} + +export async function waitForChromeReady(port = 9222, timeout = 30000) { + const startTime = Date.now(); + + return new Promise((resolve, reject) => { + function check() { + const client = net.createConnection({ port }, () => { + client.end(); + resolve(""); + }); + + client.on("error", () => { + if (Date.now() - startTime > timeout) { + reject(new Error("Timeout waiting for Chrome to start")); + } else { + setTimeout(check, 500); + } + }); + } + + check(); + }); +} + +export function randomDelay(min: number, max: number): Promise { + const delay = Math.floor(Math.random() * (max - min) + min); + return new Promise((resolve) => setTimeout(resolve, delay)); +} diff --git a/services/scraper-glassdoor/utils/rabbitmq.ts b/services/scraper-glassdoor/utils/rabbitmq.ts new file mode 100644 index 0000000..d021070 --- /dev/null +++ b/services/scraper-glassdoor/utils/rabbitmq.ts @@ -0,0 +1,165 @@ +import * as amqp from "amqplib"; + +export interface RabbitMQConfig { + url: string; + queueName: string; + exchangeName: string; + exchangeType: "direct" | "topic" | "fanout" | "headers"; + durable: boolean; +} + +export interface SearchQuery { + keywords: string; + location: string; + fwt?: string; + numPages: number; +} + +export class RabbitMQClient { + private connection: amqp.ChannelModel | null = null; + private channel: amqp.Channel | null = null; + private config: RabbitMQConfig; + + constructor(config: RabbitMQConfig) { + this.config = config; + } + + async connect(): Promise { + try { + console.log(`Connecting to RabbitMQ at: ${this.config.url}`); + this.connection = await amqp.connect(this.config.url); + console.log("Connected to RabbitMQ successfully"); + + if (!this.connection) { + throw new Error("Failed to establish RabbitMQ connection"); + } + + this.channel = await this.connection.createChannel(); + + if (!this.channel) { + throw new Error("Failed to create RabbitMQ channel"); + } + + // Ensure exchange and queue exist + await this.channel.assertExchange( + this.config.exchangeName, + this.config.exchangeType, + { durable: this.config.durable } + ); + + await this.channel.assertQueue(this.config.queueName, { + durable: this.config.durable, + arguments: { + "x-dead-letter-exchange": "scraper_dlx", + "x-dead-letter-routing-key": this.config.queueName, + "x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds + "x-max-retries": 3, + }, + }); + + await this.channel.bindQueue( + this.config.queueName, + this.config.exchangeName, + this.config.queueName + ); + + // Set prefetch to process one message at a time + await this.channel.prefetch(1); + + // Handle connection events + this.connection.on("close", () => { + console.log("RabbitMQ connection closed"); + this.connection = null; + this.channel = null; + }); + + this.connection.on("error", (error: Error) => { + console.error("RabbitMQ connection error:", error); + this.connection = null; + this.channel = null; + }); + } catch (error) { + console.error("Failed to connect to RabbitMQ:", error); + throw error; + } + } + + async subscribe( + messageHandler: (message: SearchQuery) => Promise + ): Promise { + if (!this.channel) { + throw new Error( + "RabbitMQ channel not initialized. Call connect() first." + ); + } + + console.log( + `Waiting for messages from ${this.config.queueName}. To exit press CTRL+C` + ); + + await this.channel.consume( + this.config.queueName, + async (msg: amqp.ConsumeMessage | null) => { + if (msg && this.channel) { + try { + const messageContent = msg.content.toString(); + console.log(`Received message: ${messageContent}`); + + const searchQuery: SearchQuery = JSON.parse(messageContent); + console.log(`Processing Glassdoor scrape request:`, searchQuery); + + // Process the message using the provided handler + await messageHandler(searchQuery); + + // Acknowledge the message on success + this.channel.ack(msg); + console.log("Message processed successfully"); + } catch (error) { + console.error("Error processing message:", error); + // Reject the message and don't requeue it + if (this.channel) { + this.channel.nack(msg, false, false); + } + } + } + } + ); + } + + async close(): Promise { + try { + if (this.channel) { + await this.channel.close(); + this.channel = null; + } + if (this.connection) { + await this.connection.close(); + this.connection = null; + } + console.log("RabbitMQ connection closed gracefully"); + } catch (error) { + console.error("Error closing RabbitMQ connection:", error); + } + } + + isConnected(): boolean { + return this.connection !== null && this.channel !== null; + } +} + +export function createRabbitMQConfig(): RabbitMQConfig { + return { + url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/", + queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor", + exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange", + exchangeType: "topic", + durable: true, + }; +} + +export async function createRabbitMQClient(): Promise { + const config = createRabbitMQConfig(); + const client = new RabbitMQClient(config); + await client.connect(); + return client; +} diff --git a/services/scraper-google/.dockerignore b/services/scraper-google/.dockerignore new file mode 100644 index 0000000..78a6e29 --- /dev/null +++ b/services/scraper-google/.dockerignore @@ -0,0 +1,47 @@ +# Git +.git +.gitignore + +# Environment files (will be mounted or set via docker-compose) +.local.env +*.env + +# Documentation +*.md +DOCKER_README.md +RECOMMENDED_MODELS.md + +# Development files +.vscode +.idea +*.swp +*.swo +*~ + +# Build artifacts +*.exe +*.dll +*.so +*.dylib +scraper-google + +# Test files +*_test.go +testdata/ + +# Logs +*.log +logs/ + +# Temporary files +tmp/ +temp/ + +# OS files +.DS_Store +Thumbs.db + +# Docker files (not needed in image) +Dockerfile +docker-compose.yml +.dockerignore diff --git a/services/scraper-google/.env b/services/scraper-google/.env new file mode 100644 index 0000000..1faafa7 --- /dev/null +++ b/services/scraper-google/.env @@ -0,0 +1,16 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA +OPENAI_BASE_URL=https://hubai.loe.gg/v1 +OPENAI_MODEL=gpt-4o-mini + +CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free +OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/services/scraper-google/.env.example b/services/scraper-google/.env.example new file mode 100644 index 0000000..35c9f7e --- /dev/null +++ b/services/scraper-google/.env.example @@ -0,0 +1,18 @@ +# Database Configuration +DB_HOST=postgres +DB_PORT=5432 +DB_NAME=jobs_scraper +DB_USER=postgres +DB_PASSWORD=postgres + +# OpenRouter Configuration (for job description extraction) +OPENROUTER_API_KEY=your_openrouter_api_key_here +CV_AI_MODEL=google/gemini-flash-1.5 + +# Optional: OpenAI Configuration (alternative to OpenRouter) +# OPENAI_API_KEY=your_openai_api_key_here +# OPENAI_BASE_URL=https://api.openai.com/v1 + +# Chrome/Chromedp Configuration (usually defaults are fine) +CHROMEDP_DISABLE_GPU=true +CHROMEDP_NO_SANDBOX=true diff --git a/services/scraper-google/Dockerfile b/services/scraper-google/Dockerfile new file mode 100644 index 0000000..3278239 --- /dev/null +++ b/services/scraper-google/Dockerfile @@ -0,0 +1,40 @@ +# Build stage +FROM golang:1.24-alpine AS builder + +# Install build dependencies +RUN apk add --no-cache git + +WORKDIR /build + +# Copy go mod files +COPY go.mod go.sum ./ +RUN go mod download + +# Copy the entire monorepo +COPY . . + +# Build the scraper-google service +WORKDIR /build/services/scraper-google +RUN CGO_ENABLED=0 GOOS=linux go build -o scraper-google . + +# Runtime stage +FROM chromedp/headless-shell:latest + +# Install ca-certificates for HTTPS requests +RUN apt-get update && apt-get install -y \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Copy the binary from builder +COPY --from=builder /build/services/scraper-google/scraper-google . + +# Copy environment files if they exist (optional) +COPY --from=builder /build/services/scraper-google/.env* ./ + +# Expose any ports if needed (not required for this service) +# EXPOSE 8080 + +# Run the scraper +CMD ["./scraper-google"] diff --git a/services/scraper-google/Makefile b/services/scraper-google/Makefile new file mode 100644 index 0000000..919f3b9 --- /dev/null +++ b/services/scraper-google/Makefile @@ -0,0 +1,67 @@ +.PHONY: help build up down logs restart clean rebuild + +help: ## Show this help message + @echo 'Usage: make [target]' + @echo '' + @echo 'Available targets:' + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf " %-15s %s\n", $$1, $$2}' $(MAKEFILE_LIST) + +build: ## Build the Docker images + docker-compose build + +up: ## Start all services + docker-compose up -d + +down: ## Stop all services + docker-compose down + +logs: ## View logs (use 'make logs service=scraper-google' for specific service) + @if [ -z "$(service)" ]; then \ + docker-compose logs -f; \ + else \ + docker-compose logs -f $(service); \ + fi + +restart: ## Restart all services + docker-compose restart + +restart-scraper: ## Restart just the scraper service + docker-compose restart scraper-google + +clean: ## Stop services and remove volumes (WARNING: deletes data) + docker-compose down -v + +rebuild: ## Rebuild and restart services + docker-compose down + docker-compose build --no-cache + docker-compose up -d + +ps: ## Show running containers + docker-compose ps + +stats: ## Show resource usage + docker stats + +exec-db: ## Connect to PostgreSQL database + docker-compose exec postgres psql -U postgres -d jobs_scraper + +backup-db: ## Backup the database + @mkdir -p backups + docker-compose exec postgres pg_dump -U postgres jobs_scraper > backups/backup_$(shell date +%Y%m%d_%H%M%S).sql + @echo "Database backed up to backups/backup_$(shell date +%Y%m%d_%H%M%S).sql" + +restore-db: ## Restore database from backup (use 'make restore-db file=backup.sql') + @if [ -z "$(file)" ]; then \ + echo "Error: Please specify backup file. Usage: make restore-db file=backup.sql"; \ + exit 1; \ + fi + docker-compose exec -T postgres psql -U postgres jobs_scraper < $(file) + +shell-scraper: ## Open shell in scraper container + docker-compose exec scraper-google sh + +dev: ## Run in development mode with live logs + docker-compose up --build + +test: ## Run tests (if you have any) + docker-compose exec scraper-google go test ./... diff --git a/services/scraper-google/analyzer/get-job-description.go b/services/scraper-google/analyzer/get-job-description.go new file mode 100644 index 0000000..d912191 --- /dev/null +++ b/services/scraper-google/analyzer/get-job-description.go @@ -0,0 +1,34 @@ +package analyzer + +import ( + "fmt" + "os" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/openai" + // "github.com/jobs-scraper/internal/pkg/openrouter" +) + +func GetJobDescription(htmlContent string) (*domain.JobDescription, error) { + // apiKey := os.Getenv("OPENROUTER_API_KEY") + // model := os.Getenv("CV_AI_MODEL") + openAiapiKey := os.Getenv("OPENAI_API_KEY") + if openAiapiKey == "" { + return nil, fmt.Errorf("OPENAI_API_KEY environment variable is required") + } + + openAiModel := os.Getenv("OPENAI_MODEL") + if openAiModel == "" { + openAiModel = "gpt-4o-mini" + } + + openAiService := openai.NewOpenAIService(openAiapiKey, openAiModel) + + // client := openrouter.NewOpenRouterService(model, apiKey) + result, err := openAiService.CreateJobDescription(htmlContent) + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/services/scraper-google/docker-compose.yml b/services/scraper-google/docker-compose.yml new file mode 100644 index 0000000..5310f1c --- /dev/null +++ b/services/scraper-google/docker-compose.yml @@ -0,0 +1,49 @@ +version: "3.8" + +services: + scraper-google: + build: + context: ../../ + dockerfile: services/scraper-google/Dockerfile + container_name: jobs-scraper-google + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: jobs_scraper + DB_USER: scraper_user + DB_PASSWORD: Cocowawa_12345 + + # OpenRouter Configuration + OPENROUTER_API_KEY: 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 + CV_AI_MODEL: alibaba/tongyi-deepresearch-30b-a3b:free + + # Optional: OpenAI Configuration + OPENAI_API_KEY: sk-GtdLX9YBOCsBEgBL4rBONA + OPENAI_BASE_URL: https://hubai.loe.gg/v1 + OPEN_AI_MODEL: gpt-4o-mini + + # Chrome/Chromedp Configuration + CHROMEDP_DISABLE_GPU: true + CHROMEDP_NO_SANDBOX: true + volumes: + - ./.env:/app/.env:ro + restart: unless-stopped + networks: + - scraper-network + # Add resource limits to prevent runaway chrome instances + deploy: + resources: + limits: + memory: 2G + cpus: "2" + reservations: + memory: 512M + cpus: "0.5" + +volumes: + postgres_data: + driver: local + +networks: + scraper-network: + driver: bridge diff --git a/services/scraper-google/go.mod b/services/scraper-google/go.mod new file mode 100644 index 0000000..4563969 --- /dev/null +++ b/services/scraper-google/go.mod @@ -0,0 +1,22 @@ +module github.com/jobs-scraper/services/scraper-google + +go 1.24.0 + +require ( + github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d + github.com/chromedp/chromedp v0.14.2 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 +) + +require ( + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + github.com/levmv/sked v0.2.1 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + golang.org/x/sys v0.36.0 // indirect +) + +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure diff --git a/services/scraper-google/go.sum b/services/scraper-google/go.sum new file mode 100644 index 0000000..54c07b7 --- /dev/null +++ b/services/scraper-google/go.sum @@ -0,0 +1,27 @@ +github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU= +github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/levmv/sked v0.2.1 h1:XMdLkbuajPM5LSzoBvNzalj6oYHkEfeA7/VPvkIHqs0= +github.com/levmv/sked v0.2.1/go.mod h1:w9pFUIpZLPu0Jr2+2JqJtaEIHmwXcvjPHWm6pvbqHVQ= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/services/scraper-google/main.go b/services/scraper-google/main.go new file mode 100644 index 0000000..8e1e530 --- /dev/null +++ b/services/scraper-google/main.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/services/scraper-google/analyzer" + si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure" + "github.com/jobs-scraper/services/scraper-google/utils" + "github.com/levmv/sked" +) + +type SearchResult struct { + Title string + Description string + Link string + CompanyName string +} + +func main() { + db, err := si.SetupInfrastructure() + if err != nil { + log.Fatalf("Failed to set up infrastructure: %v", err) + } + + jobRepo := repo.NewJobRepository(db) + jobDescRepo := repo.NewJobDescriptionRepository(db) + + ctx, cancel := context.WithCancel(context.Background()) + + sched := sked.New(ctx) + googleLinkStream := make(chan domain.GoogleLink, 100) + // pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + pages := []int{90} + + log.Println("Starting scraper with scheduler...") + + for _, page := range pages { + // Work(ctx)(page, googleLinkStream) + sched.Schedule(func(ctx context.Context) { + log.Printf("Running scrape for page offset: %d", page) + Work(ctx)(page, googleLinkStream) + }).Every(24 * time.Hour) + } + + if err := sched.Run(); err != nil { + log.Fatalf("Failed to start scheduler: %v", err) + } + + go func() { + for googleLink := range googleLinkStream { + htmlRaw, url, err := utils.GetHTMLRaw(ctx, googleLink) + + if err != nil { + log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err) + continue + } + + result, err := analyzer.GetJobDescription(htmlRaw) + + if err != nil { + log.Printf("Error analyzing job description for link %s: %v", googleLink.Link, err) + continue + } + + now := time.Now() + + job := domain.Job{ + ID: googleLink.ID, + Title: googleLink.Title, + Company: googleLink.CompanyName, + CompanyLink: googleLink.Link, + Location: result.Criteria["location"], + JobLink: url, + Provider: domain.Google, + JobPostTime: &now, + Status: domain.JobStatusCreated, + } + + jobDescription := domain.JobDescription{ + JobID: googleLink.ID, + Description: result.Description, + Criteria: result.Criteria, + } + + if err := jobRepo.SaveJobs([]domain.Job{job}); err != nil { + log.Printf("Failed to save job %d: %v", googleLink.ID, err) + continue + } + + if err := jobDescRepo.SaveJobDescriptions([]domain.JobDescription{jobDescription}); err != nil { + log.Printf("Failed to save job description for job %d: %v", googleLink.ID, err) + continue + } + + log.Printf("Successfully saved job: %s from %s (ID: %d)", job.Title, job.Company, job.ID) + } + }() + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + <-c + cancel() + log.Println("Shutting down scraper...") + +} diff --git a/services/scraper-google/setup-infrastructure/setup-infrastructure.go b/services/scraper-google/setup-infrastructure/setup-infrastructure.go new file mode 100644 index 0000000..269f121 --- /dev/null +++ b/services/scraper-google/setup-infrastructure/setup-infrastructure.go @@ -0,0 +1,38 @@ +package setupinfrastructure + +import ( + "database/sql" + "log" + + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/joho/godotenv" +) + +func SetupInfrastructure() (*sql.DB, error) { + // Load .env from the scraper-google service directory + if err := godotenv.Load(".local.env"); err != nil { + log.Printf("No .local.env file found: %v, trying .env", err) + if err := godotenv.Load(".env"); err != nil { + log.Println("No .env file found, using system environment variables") + } + } + + dbConfig := infrastructure.LoadConfigFromEnv() + + db, err := infrastructure.NewConnection(dbConfig) + + if err != nil { + log.Fatalf("Error connecting to db: %v", err) + return nil, err + } + + err = db.Ping() + + if err != nil { + log.Fatal("Error pinging db") + return nil, err + } + + log.Println("Successfully connected to db") + return db, err +} diff --git a/services/scraper-google/utils/build-url.go b/services/scraper-google/utils/build-url.go new file mode 100644 index 0000000..fac437c --- /dev/null +++ b/services/scraper-google/utils/build-url.go @@ -0,0 +1,15 @@ +package utils + +import ( + "strconv" + "strings" +) + +func BuildUrl(query string, page int) string { + var url strings.Builder + url.WriteString("https://www.google.com/search?q=") + url.WriteString(query) + url.WriteString("&start=") + url.WriteString(strconv.Itoa(page)) + return url.String() +} diff --git a/services/scraper-google/utils/extract-company-name.go b/services/scraper-google/utils/extract-company-name.go new file mode 100644 index 0000000..ab183a6 --- /dev/null +++ b/services/scraper-google/utils/extract-company-name.go @@ -0,0 +1,24 @@ +package utils + +import ( + "fmt" + "net/url" + "strings" +) + +func ExtractCompanyName(urlStr string) (string, error) { + parsedURL, err := url.Parse(urlStr) + + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + + path := parsedURL.Path + segments := strings.Split(strings.Trim(path, "/"), "/") + + if len(segments) > 0 && segments[0] != "" { + return segments[0], nil + } + + return "", fmt.Errorf("no path segments found") +} diff --git a/services/scraper-google/utils/extract-job-id.go b/services/scraper-google/utils/extract-job-id.go new file mode 100644 index 0000000..b1cf96b --- /dev/null +++ b/services/scraper-google/utils/extract-job-id.go @@ -0,0 +1,30 @@ +package utils + +import ( + "fmt" + "net/url" + "strconv" + "strings" +) + +func ExtractJobId(urlStr string) (int64, error) { + parsedURL, err := url.Parse(urlStr) + + if err != nil { + return 0, fmt.Errorf("failed to parse URL: %w", err) + } + + path := parsedURL.Path + segments := strings.Split(strings.Trim(path, "/"), "/") + + for i := len(segments) - 1; i >= 0; i-- { + if segments[i] != "" { + jobId, err := strconv.ParseInt(segments[i], 10, 64) + if err == nil { + return jobId, nil + } + } + } + + return 0, fmt.Errorf("no numeric job ID found in URL path: %s", path) +} diff --git a/services/scraper-google/utils/get-html-raw.go b/services/scraper-google/utils/get-html-raw.go new file mode 100644 index 0000000..d98945a --- /dev/null +++ b/services/scraper-google/utils/get-html-raw.go @@ -0,0 +1,92 @@ +package utils + +import ( + "context" + "fmt" + "log" + "net/url" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/chromedp/chromedp" + "github.com/jobs-scraper/internal/pkg/domain" +) + +func GetHTMLRaw(ctx context.Context, googleLink domain.GoogleLink) (string, string, error) { + // Set up chromedp options + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", false), + chromedp.Flag("disable-blink-features", "AutomationControlled"), + chromedp.Flag("excludeSwitches", "enable-automation"), + chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + chromedp.WindowSize(1920, 1080), + ) + + allocCtx, allocCancel := chromedp.NewExecAllocator(ctx, opts...) + defer allocCancel() + + browserCtx, browserCancel := chromedp.NewContext(allocCtx) + defer browserCancel() + + // Add timeout for the entire operation + timeoutCtx, timeoutCancel := context.WithTimeout(browserCtx, 30*time.Second) + defer timeoutCancel() + + var htmlContent string + var finalURL string + + err := chromedp.Run(timeoutCtx, + chromedp.Navigate(googleLink.Link), + chromedp.WaitVisible("body", chromedp.ByQuery), + chromedp.Sleep(2*time.Second), // Wait for JS to render + chromedp.Location(&finalURL), + chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery), + ) + + if err != nil { + log.Printf("Failed to fetch job description with chromedp for link %s: %v", googleLink.Link, err) + return "", "", err + } + + // Parse the HTML with goquery + doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent)) + if err != nil { + log.Printf("Failed to parse HTML for link %s: %v", googleLink.Link, err) + return "", "", err + } + + // Remove unwanted elements + doc.Find("script, style, nav, footer, header, .ads, .sidebar").Remove() + + log.Printf(" Received: %s from %s\n, ID: %d", googleLink.Title, googleLink.CompanyName, googleLink.ID) + + // Extract text content + var content strings.Builder + doc.Find("body").Each(func(i int, s *goquery.Selection) { + content.WriteString(s.Text()) + }) + + // Extract hostname from the final URL + hostname, err := extractHostname(finalURL) + if err != nil { + log.Printf("Failed to extract hostname from final URL %s: %v", finalURL, err) + // Fall back to the original link if extraction fails + return cleanText(content.String()), googleLink.Link, nil + } + + return cleanText(content.String()), hostname, nil +} + +func cleanText(text string) string { + text = strings.Join(strings.Fields(text), " ") + return text +} + +func extractHostname(urlStr string) (string, error) { + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "", fmt.Errorf("failed to parse URL: %w", err) + } + return parsedURL.Hostname(), nil +} diff --git a/services/scraper-google/work.go b/services/scraper-google/work.go new file mode 100644 index 0000000..64b60ee --- /dev/null +++ b/services/scraper-google/work.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "log" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/chromedp" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/services/scraper-google/utils" +) + +func Work(mainCtx context.Context) func(page int, stream chan<- domain.GoogleLink) { + + return func(page int, stream chan<- domain.GoogleLink) { + query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer" + + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", false), + chromedp.Flag("disable-blink-features", "AutomationControlled"), + chromedp.Flag("excludeSwitches", "enable-automation"), + chromedp.Flag("disable-web-security", false), + chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + chromedp.WindowSize(1920, 1080), + ) + allocCtx, cancel := chromedp.NewExecAllocator(mainCtx, opts...) + defer cancel() + + ctx, cancel := chromedp.NewContext(allocCtx) + defer cancel() + + var nodes []*cdp.Node + + // start = 0, start = 10, start = 20 and so on for offset pagination + err := chromedp.Run(ctx, + chromedp.Navigate(utils.BuildUrl(query, page)), + ) + + if err != nil { + log.Fatal(err) + } + + err = chromedp.Run(ctx, + chromedp.WaitVisible("#search", chromedp.ByID), + ) + + if err != nil { + log.Fatal(err) + } + + err = chromedp.Run(ctx, + chromedp.Nodes("[data-snc]", &nodes, chromedp.ByQueryAll), + ) + + if err != nil { + log.Fatal(err) + } + + for i, node := range nodes { + result := domain.GoogleLink{} + + err = chromedp.Run(ctx, + chromedp.Text("h3", &result.Title, chromedp.ByQuery, chromedp.FromNode(node)), + ) + + if err != nil { + log.Printf("Failed to get title for node %d: %v", i, err) + } + + err = chromedp.Run(ctx, + chromedp.AttributeValue("a", "href", &result.Link, nil, chromedp.ByQuery, chromedp.FromNode(node)), + ) + + if err != nil { + log.Printf("Failed to get link for node %d: %v", i, err) + } + + result.ID, err = utils.ExtractJobId(result.Link) + + if err != nil { + log.Printf("Failed to extract job ID for node %d: %v (link: %s)", i, err, result.Link) + continue + } + + result.CompanyName, err = utils.ExtractCompanyName(result.Link) + + if err != nil { + log.Printf("Failed to extract company name for node %d: %v (link: %s)", i, err, result.Link) + continue + } + + err = chromedp.Run(ctx, + chromedp.Text("div.VwiC3b", &result.Description, chromedp.ByQuery, chromedp.FromNode(node)), + ) + + if err != nil { + log.Printf("Failed to get description for node %d: %v", i, err) + } + + stream <- result + } + } + +} diff --git a/services/scraper-hiringcafe/go.mod b/services/scraper-hiringcafe/go.mod new file mode 100644 index 0000000..627339c --- /dev/null +++ b/services/scraper-hiringcafe/go.mod @@ -0,0 +1,3 @@ +module scraper-hiringcafe + +go 1.24.0 diff --git a/services/scraper-hiringcafe/main.go b/services/scraper-hiringcafe/main.go new file mode 100644 index 0000000..38dd16d --- /dev/null +++ b/services/scraper-hiringcafe/main.go @@ -0,0 +1,3 @@ +package main + +func main() {} diff --git a/services/scraper-linkedin/.env b/services/scraper-linkedin/.env new file mode 100644 index 0000000..ade07d5 --- /dev/null +++ b/services/scraper-linkedin/.env @@ -0,0 +1,12 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=your_db_user +DB_PASSWORD=your_db_password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/services/scraper-linkedin/go.mod b/services/scraper-linkedin/go.mod new file mode 100644 index 0000000..f9e8ef3 --- /dev/null +++ b/services/scraper-linkedin/go.mod @@ -0,0 +1,32 @@ +module github.com/jobs-scraper/services/scraper + +go 1.24.0 + +toolchain go1.24.7 + +require ( + github.com/PuerkitoBio/goquery v1.10.3 + github.com/jobs-scraper/internal/pkg/domain v0.0.0 + github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 + github.com/jobs-scraper/internal/pkg/utils v0.0.0 + github.com/jobs-scraper/libs/repo v0.0.0 + github.com/joho/godotenv v1.5.1 +) + +require ( + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/golang-migrate/migrate/v4 v4.19.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + golang.org/x/net v0.44.0 // indirect +) + +replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain + +replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure + +replace github.com/jobs-scraper/libs/repo => ../../libs/repo + +replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils diff --git a/services/scraper-linkedin/go.sum b/services/scraper-linkedin/go.sum new file mode 100644 index 0000000..6a20814 --- /dev/null +++ b/services/scraper-linkedin/go.sum @@ -0,0 +1,144 @@ +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= +github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= +github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/scraper-linkedin/main.go b/services/scraper-linkedin/main.go new file mode 100644 index 0000000..902c10b --- /dev/null +++ b/services/scraper-linkedin/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "encoding/json" + "log" + "os" + "os/signal" + "syscall" + + "time" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/infrastructure" + "github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq" + "github.com/jobs-scraper/libs/repo" + "github.com/jobs-scraper/services/scraper/pipeline" + "github.com/joho/godotenv" +) + +func main() { + // Try to load .local.env first, then fallback to .env + if err := godotenv.Load(".local.env"); err != nil { + log.Println("No .local.env file found, trying .env") + if err := godotenv.Load(); err != nil { + log.Println("No .env file found, using system environment variables") + } + } + + dbConfig := infrastructure.LoadConfigFromEnv() + db, err := infrastructure.NewConnection(dbConfig) + if err != nil { + log.Fatalf("Error connecting to db %v", err) + } + + err = db.Ping() + + if err != nil { + log.Fatalf("Error pinging db %v", err) + } + + log.Println("Successfully connected to db") + + rmq, err := rabbitmq.NewRabbitMQClient() + if err != nil { + log.Fatalf("Error connecting to RabbitMQ %v", err) + } + + scraper := pipeline.NewScraper(pipeline.Config{ + SortBy: "R", + MaxRetries: 3, + BaseDelay: 1 * time.Second, + MaxDelay: 30 * time.Second, + RequestTimeout: 30 * time.Second, + }) + + jobRepo := repo.NewJobRepository(db) + jobDescriptionRepo := repo.NewJobDescriptionRepository(db) + + jobPipeline := pipeline.NewJobPipeline(scraper, 3, 1*time.Second) // 3 workers, 1 second rate limit + + err = rmq.Subscribe(rabbitmq.LinkedInQueue, "scraper-consumer", func(data []byte) error { + ctx := context.Background() + + var searchParams domain.SearchQuery + + err := json.Unmarshal(data, &searchParams) + + if err != nil { + log.Printf("Error unmarshaling message: %v", err) + return err + } + + log.Printf("Received message: %s", string(data)) + err = jobPipeline.ProcessJobsStreaming(ctx, jobRepo, jobDescriptionRepo, searchParams) + + if err != nil { + log.Printf("Error processing jobs: %v", err) + return err + } + + log.Printf("Successfully processed job search for: %s in %s", searchParams.Keywords, searchParams.Location) + return nil + }) + if err != nil { + log.Printf("Error subscribing to LinkedIn queue: %v", err) + } + + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, syscall.SIGTERM) + <-c + rmq.Close() + log.Println("Shutting down scraper...") +} diff --git a/services/scraper-linkedin/pipeline/job_pipeline.go b/services/scraper-linkedin/pipeline/job_pipeline.go new file mode 100644 index 0000000..bdbb779 --- /dev/null +++ b/services/scraper-linkedin/pipeline/job_pipeline.go @@ -0,0 +1,65 @@ +package pipeline + +import ( + "context" + "fmt" + + // "log" + "sync" + + "time" + + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/libs/repo" +) + +// JobDescriptionResult represents the result of job description scraping +type JobDescriptionResult struct { + Job domain.Job + Description string + Criteria map[string]string + Error error +} + +// JobPipeline manages the job processing pipeline +type JobPipeline struct { + scraperService *Scraper + numWorkers int + rateLimit time.Duration +} + +// NewJobPipeline creates a new job processing pipeline +func NewJobPipeline(scraperService *Scraper, numWorkers int, rateLimit time.Duration) *JobPipeline { + return &JobPipeline{ + scraperService: scraperService, + numWorkers: numWorkers, + rateLimit: rateLimit, + } +} + +// ProcessJobsStreaming processes jobs and job descriptions concurrently +func (p *JobPipeline) ProcessJobsStreaming(ctx context.Context, jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository, searchQuery domain.SearchQuery) error { + allJobs := make([]domain.Job, 0, 100) + allJobDescriptions := make([]domain.JobDescription, 0, 100) + var jbMu sync.Mutex + jobsChan := GetJobs(ctx, p.scraperService, searchQuery) + jobWithDescriptionChan := GetJobDescription(ctx, p.scraperService, jobsChan, p.numWorkers) + + for jobWithDescription := range jobWithDescriptionChan { + fmt.Printf("Received job description for job : %d\n", jobWithDescription.Job.ID) + jbMu.Lock() + allJobs = append(allJobs, jobWithDescription.Job) + allJobDescriptions = append(allJobDescriptions, jobWithDescription.JobDescription) + jbMu.Unlock() + } + + if err := jobRepo.SaveJobs(allJobs); err != nil { + return fmt.Errorf("failed to save jobs to database: %w", err) + } + + if err := jobDescRepo.SaveJobDescriptions(allJobDescriptions); err != nil { + return fmt.Errorf("failed to save job descriptions: %w", err) + } + + return nil +} diff --git a/services/scraper-linkedin/pipeline/job_pipeline_workers.go b/services/scraper-linkedin/pipeline/job_pipeline_workers.go new file mode 100644 index 0000000..5b39bc0 --- /dev/null +++ b/services/scraper-linkedin/pipeline/job_pipeline_workers.go @@ -0,0 +1,92 @@ +package pipeline + +import ( + "context" + "log" + + "github.com/jobs-scraper/internal/pkg/domain" +) + +func GetJobs(context context.Context, scraperService *Scraper, searchQuery domain.SearchQuery) <-chan domain.Job { + jobChan := make(chan domain.Job) + + go func() { + defer close(jobChan) + if err := scraperService.ScrapeLinkedInJobsStreaming(context, searchQuery.NumPages, jobChan, searchQuery); err != nil { + log.Printf("Error scraping jobs: %v", err) + } + }() + + return jobChan +} + +func GetJobDescription(context context.Context, scraperService *Scraper, jobChan <-chan domain.Job, numWorkers int) <-chan domain.JobWithDescription { + jobDescriptionChan := make(chan domain.JobWithDescription) + + go func() { + defer close(jobDescriptionChan) + + for job := range jobChan { + select { + case <-context.Done(): + return + default: + } + + if jd, jc, err := scraperService.ScrapeJobDescriptionWithContext(context, job); err != nil { + log.Printf("Error scraping job description: %v", err) + } else { + jobDescriptionChan <- domain.JobWithDescription{ + Job: job, + JobDescription: domain.JobDescription{ + JobID: job.ID, + Description: jd, + Criteria: jc, + }, + } + } + } + }() + + return jobDescriptionChan + + // unless you have proxies, you will get rate limited by linkedin, + // but if you do have you can use this code below, + // it will work faster (if u pass more than one worker) + // + dont forget to use buffered channels + + // for range numWorkers { + // wg.Add(1) + // go func() { + // defer wg.Done() + // for job := range jobChan { + // select { + // case <-context.Done(): + // return + // default: + // } + + // // Scrape job description + // if jd, jc, err := scraperService.ScrapeJobDescriptionWithContext(context, job); err != nil { + // log.Printf("Error scraping job description: %v", err) + // } else { + // jobDescriptionChan <- domain.JobWithDescription{ + // Job: job, + // JobDescription: domain.JobDescription{ + // JobID: job.ID, + // Description: jd, + // Criteria: jc, + // }, + // } + // } + // } + // }() + // } + + // Close the output channel when all workers are done + // go func() { + // wg.Wait() + // close(jobDescriptionChan) + // }() + +} diff --git a/services/scraper-linkedin/pipeline/scraper.go b/services/scraper-linkedin/pipeline/scraper.go new file mode 100644 index 0000000..97f6b2f --- /dev/null +++ b/services/scraper-linkedin/pipeline/scraper.go @@ -0,0 +1,248 @@ +package pipeline + +import ( + "context" + "fmt" + "log" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" + "github.com/jobs-scraper/internal/pkg/domain" + "github.com/jobs-scraper/internal/pkg/utils" +) + +type Config struct { + Timespan string // Time filter for job postings (e.g., "r86400" for last 24 hours) + SortBy string // Sort results by (R for relevance, DD for date posted) + MaxRetries int // Maximum number of retries for failed requests + BaseDelay time.Duration // Base delay for exponential backoff + MaxDelay time.Duration // Maximum delay between retries + RequestTimeout time.Duration // Timeout for individual HTTP requests +} + +type Scraper struct { + config Config +} + +func NewScraper(config Config) *Scraper { + // r86400 last 24 hours + // r604800 last week + // r2592000 last month + // r7776000 last 3 months + // r31536000 last year + if config.Timespan == "" { + config.Timespan = "r604800" // Default to last week + } + + // Set default retry configuration + if config.MaxRetries == 0 { + config.MaxRetries = 3 + } + if config.BaseDelay == 0 { + config.BaseDelay = 1 * time.Second + } + if config.MaxDelay == 0 { + config.MaxDelay = 30 * time.Second + } + if config.RequestTimeout == 0 { + config.RequestTimeout = 30 * time.Second + } + + return &Scraper{ + config: config, + } +} + +// ScrapeLinkedInJobsStreaming scrapes jobs page by page and sends them to channel immediately +func (s *Scraper) ScrapeLinkedInJobsStreaming(ctx context.Context, numPages int, jobChan chan<- domain.Job, params domain.SearchQuery) error { + // Process pages sequentially to send jobs immediately + for i := range numPages { + select { + case <-ctx.Done(): + return ctx.Err() + default: + jobs, err := s.ScrapeJobsWithContext(ctx, i, params) + if err != nil { + return fmt.Errorf("error scraping page %d: %w", i, err) + } + + // Send jobs to channel immediately + for _, job := range jobs { + select { + case <-ctx.Done(): + return ctx.Err() + case jobChan <- job: + } + } + } + } + return nil +} + +func (s *Scraper) ScrapeJobsWithContext(ctx context.Context, page int, params domain.SearchQuery) ([]domain.Job, error) { + jobs := make([]domain.Job, 0, 10) + url := s.buildSearchURL(params, page) + log.Println("Parsing page: ", page+1) + retryableRequest := utils.NewRetryableHTTPRequest(utils.RetryConfig{ + BaseDelay: s.config.BaseDelay, + MaxDelay: s.config.MaxDelay, + MaxRetries: s.config.MaxRetries, + }) + + res, err := retryableRequest.RetryableHTTPRequest(ctx, url, "GET", nil, nil) + if err != nil { + fmt.Printf("Error fetching URL after retries: %v\n", err) + return jobs, err + } + + defer res.Body.Close() + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + return jobs, err + } + + doc.Find("li > div.base-card").Each(func(i int, s *goquery.Selection) { + job := domain.Job{} + title := s.Find("[class*=_title]").Text() + job.Title = strings.TrimSpace(title) + job.Company = strings.TrimSpace(s.Find(".hidden-nested-link").Text()) + job.CompanyLink = strings.TrimSpace(s.Find(".hidden-nested-link").AttrOr("href", "")) + job.Location = strings.TrimSpace(params.Location) + job.JobLink = strings.TrimSpace(s.Find("a.base-card__full-link").AttrOr("href", "")) + postTimeText := s.Find("[class*=listdate]").AttrOr("datetime", time.DateTime) + postTime, err := time.Parse(time.DateOnly, postTimeText) + // Handle any parsing errors by logging them + if err != nil { + fmt.Printf("Couldn't parse the job post time: %v", err) + } + + job.JobPostTime = &postTime + jobId, err := extractJobIDFromURL(job.JobLink) + if err != nil { + fmt.Printf("Error extracting job ID from URL %s: %v\n", job.JobLink, err) + } + jobIdInt, err := strconv.ParseInt(jobId, 10, 64) + + if err != nil { + fmt.Printf("Error extracting job ID from URL %s: %v\n", job.JobLink, err) + } + job.ID = jobIdInt + job.Provider = 0 + + if job.Title != "" && job.Company != "" && job.Location != "" && job.JobLink != "" { + jobs = append(jobs, job) + } + }) + + return jobs, nil +} + +func (s *Scraper) ScrapeJobDescriptionWithContext(ctx context.Context, job domain.Job) (string, map[string]string, error) { + var jobDescription string + jobLink := fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/%d", job.ID) + + url := s.buildJobDescriptionSearchURL(jobLink) + + retryableRequest := utils.NewRetryableHTTPRequest(utils.RetryConfig{ + MaxRetries: s.config.MaxRetries, + BaseDelay: s.config.BaseDelay, + MaxDelay: s.config.MaxDelay, + }) + + res, err := retryableRequest.RetryableHTTPRequest(ctx, url, "GET", nil, nil) + if err != nil { + fmt.Printf("Error fetching job description URL after retries: %v\n", err) + return "", map[string]string{}, err + } + + defer res.Body.Close() + + doc, err := goquery.NewDocumentFromReader(res.Body) + if err != nil { + return "", map[string]string{}, err + } + + // Find the job details section directly by ID and extract its text content + jobDescription, jobCriteria, err := s.parseJobDescription(doc) + if err != nil { + return "", map[string]string{}, err + } + + return strings.TrimSpace(jobDescription), jobCriteria, nil +} + +func (s *Scraper) buildSearchURL(query domain.SearchQuery, page int) string { + baseURL := "https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search" + params := url.Values{} + + params.Set("keywords", query.Keywords) + params.Set("location", query.Location) + + if query.FWT != "" { + params.Set("f_WT", query.FWT) + } + + params.Set("start", strconv.Itoa(25*page)) + + return fmt.Sprintf("%s?%s", baseURL, params.Encode()) +} + +func (s *Scraper) buildJobDescriptionSearchURL(jobLink string) string { + return strings.ReplaceAll(jobLink, "jp.linkedin.com", "linkedin.com") +} + +func (s *Scraper) parseJobDescription(doc *goquery.Document) (string, map[string]string, error) { + var jobDescription string + jobCriteria := make(map[string]string) + + jobDescription = strings.TrimSpace(doc.Find("[class*=description] > section > div").Text()) + + criteriaList := doc.Find("[class*=_job-criteria-list]") + + if criteriaList.Length() > 0 { + criteriaList.Find("li.description__job-criteria-item").Each(func(i int, li *goquery.Selection) { + header := strings.TrimSpace(li.Find("h3.description__job-criteria-subheader").Text()) + + value := strings.TrimSpace(li.Find("span.description__job-criteria-text").Text()) + + if header != "" && value != "" { + jobCriteria[header] = value + } + }) + } + + jobDescription = strings.TrimSpace(jobDescription) + + if jobDescription == "" { + return "", jobCriteria, fmt.Errorf("job description not found") + } + + return jobDescription, jobCriteria, nil +} + +func extractJobIDFromURL(jobURL string) (string, error) { + parsed, err := url.Parse(jobURL) + if err != nil { + return "", fmt.Errorf("invalid URL: %w", err) + } + + // Get the last segment of the path + lastSegment := path.Base(parsed.Path) + + // Extract job ID from the end of the segment using regex + // Look for a sequence of digits at the end of the string + re := regexp.MustCompile(`(\d+)$`) + matches := re.FindStringSubmatch(lastSegment) + + if len(matches) < 2 { + return "", fmt.Errorf("job ID not found in URL path: %s", jobURL) + } + + return matches[1], nil +}