Compare commits
10 commits
a8c253bdd5
...
af52085ea3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
af52085ea3 | ||
|
|
a281363b61 | ||
|
|
8a4ad07e46 | ||
|
|
13d10e9d11 | ||
|
|
cbcf9bdcc7 | ||
|
|
18842091a4 | ||
|
|
9ef1833073 | ||
|
|
bb55154a19 | ||
|
|
2a137d3b61 | ||
|
|
3187757776 |
124 changed files with 7391 additions and 965 deletions
|
|
@ -1,23 +1,6 @@
|
|||
stages:
|
||||
- deploy
|
||||
|
||||
deploy_production:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"
|
||||
include:
|
||||
- local: 'services/scraper-google/.gitlab-ci.yml'
|
||||
- local: 'services/api/.gitlab-ci.yml'
|
||||
|
|
|
|||
13
Makefile
13
Makefile
|
|
@ -6,6 +6,7 @@ API_DIR=./services/api
|
|||
SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin
|
||||
SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor
|
||||
SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright
|
||||
JOB_APPLIER_BROWSER_USE_DIR=./services/job-applier-browser-use
|
||||
CRON_ANALYZER_DIR=./apps/cron-analyzer
|
||||
CV_ANALYZER_DIR=./apps/cv-analyzer
|
||||
DOCS_DIR=./services/api/pkg/swagger
|
||||
|
|
@ -71,6 +72,18 @@ docker-scraper-playwright: ## Build and run Playwright scraper in Docker
|
|||
@echo "Running Playwright scraper in Docker..."
|
||||
@docker run --rm --name playwright-scraper playwright-scraper
|
||||
|
||||
.PHONY: setup-job-applier-browser-use
|
||||
setup-job-applier-browser-use: ## Setup the browser-use job applier service
|
||||
@echo "Setting up browser-use job applier service..."
|
||||
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv venv && uv pip install -e .
|
||||
@echo "Installing Chromium browser..."
|
||||
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uvx browser-use install
|
||||
|
||||
.PHONY: run-job-applier-browser-use
|
||||
run-job-applier-browser-use: ## Run the browser-use job applier service
|
||||
@echo "Running browser-use job applier service..."
|
||||
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv run python -m src.main
|
||||
|
||||
# Apps
|
||||
.PHONY: run-cron-analyzer
|
||||
run-cron-analyzer: ## Run the job analysis cron service
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"log"
|
||||
|
||||
"github.com/jobs-scraper/apps/cron-analyzer/internal"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,9 +3,9 @@ module github.com/jobs-scraper/apps/cron-analyzer
|
|||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/openrouter v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
)
|
||||
|
|
@ -20,10 +20,10 @@ require (
|
|||
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter
|
||||
replace github.com/jobs-scraper/internal/openrouter => ../../internal/openrouter
|
||||
|
||||
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ import (
|
|||
|
||||
// "path/filepath"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/pkg/openai"
|
||||
"github.com/jobs-scraper/internal/pkg/openrouter"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/openai"
|
||||
"github.com/jobs-scraper/internal/openrouter"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
||||
|
|
|
|||
14
go.mod
14
go.mod
|
|
@ -3,3 +3,17 @@ module github.com/jobs-scraper
|
|||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
||||
require (
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
|
||||
github.com/chromedp/chromedp v0.14.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
)
|
||||
|
|
|
|||
10
go.sum
10
go.sum
|
|
@ -0,0 +1,10 @@
|
|||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
|
||||
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
12
go.work
12
go.work
|
|
@ -3,12 +3,12 @@ go 1.24.0
|
|||
use (
|
||||
.
|
||||
./apps/cron-analyzer
|
||||
./internal/pkg/domain
|
||||
./internal/pkg/gemini
|
||||
./internal/pkg/infrastructure
|
||||
./internal/pkg/openai
|
||||
./internal/pkg/openrouter
|
||||
./internal/pkg/utils
|
||||
./internal/domain
|
||||
./internal/gemini
|
||||
./internal/infrastructure
|
||||
./internal/openai
|
||||
./internal/openrouter
|
||||
./internal/utils
|
||||
./libs/ports
|
||||
./libs/repo
|
||||
./libs/server
|
||||
|
|
|
|||
14
go.work.sum
14
go.work.sum
|
|
@ -43,6 +43,9 @@ github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
|
|||
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
|
||||
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
|
||||
github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
|
||||
github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM=
|
||||
|
|
@ -57,9 +60,13 @@ github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9
|
|||
github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
|
||||
github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
|
||||
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
|
||||
|
|
@ -102,8 +109,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
|
|||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
||||
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
|
||||
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
|
|
@ -121,6 +128,7 @@ github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86w
|
|||
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI=
|
||||
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
|
||||
github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
|
||||
|
|
@ -156,7 +164,7 @@ golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
|
|||
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
|
|
@ -169,7 +177,6 @@ golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
|||
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
|
||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg=
|
||||
google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw=
|
||||
|
|
@ -183,7 +190,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.
|
|||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
|
||||
google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
|
||||
google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
|
||||
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
|
||||
modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
|
||||
|
|
|
|||
403
internal/browser/browser.go
Normal file
403
internal/browser/browser.go
Normal file
|
|
@ -0,0 +1,403 @@
|
|||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"math/rand"
|
||||
"os/exec"
|
||||
// "strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/dom"
|
||||
"github.com/chromedp/cdproto/page"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
type ChromeInstance struct {
|
||||
WSURL string
|
||||
Cmd *exec.Cmd
|
||||
}
|
||||
|
||||
type Browser struct {
|
||||
Instance *ChromeInstance
|
||||
allocCtx context.Context
|
||||
allocCancel context.CancelFunc
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// NewBrowser launches Chrome and creates a shared allocator.
|
||||
// Use NewTab() to create separate tabs for concurrent use.
|
||||
func NewBrowser(ctx context.Context) (*Browser, error) {
|
||||
wsURL, cmd, err := launchChrome()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
|
||||
|
||||
// Create initial browser context (first tab)
|
||||
browserCtx, cancel := chromedp.NewContext(allocCtx)
|
||||
|
||||
return &Browser{
|
||||
Instance: &ChromeInstance{
|
||||
WSURL: wsURL,
|
||||
Cmd: cmd,
|
||||
},
|
||||
allocCtx: allocCtx,
|
||||
allocCancel: allocCancel,
|
||||
ctx: browserCtx,
|
||||
cancel: cancel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewTab creates a new browser tab that can be used concurrently.
|
||||
// Returns a new Browser instance sharing the same Chrome process.
|
||||
func (b *Browser) NewTab() (*Browser, error) {
|
||||
tabCtx, cancel := chromedp.NewContext(b.allocCtx)
|
||||
|
||||
return &Browser{
|
||||
Instance: b.Instance,
|
||||
allocCtx: b.allocCtx,
|
||||
allocCancel: nil, // Don't close allocator when closing a tab
|
||||
ctx: tabCtx,
|
||||
cancel: cancel,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b *Browser) Close() error {
|
||||
if b.cancel != nil {
|
||||
b.cancel()
|
||||
}
|
||||
|
||||
if b.allocCancel != nil {
|
||||
b.allocCancel()
|
||||
}
|
||||
|
||||
// Only kill Chrome process if this is the main browser (has allocCancel),
|
||||
// not a tab created via NewTab()
|
||||
if b.allocCancel != nil && b.Instance != nil && b.Instance.Cmd != nil && b.Instance.Cmd.Process != nil {
|
||||
return b.Instance.Cmd.Process.Kill()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Browser) Navigate(url string) error {
|
||||
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36")
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.Navigate(url),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Browser) WaitVisible(selector string) error {
|
||||
ctx, cancel := context.WithTimeout(b.ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
err := chromedp.Run(ctx,
|
||||
chromedp.WaitVisible(selector, chromedp.ByQuery),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
|
||||
var nodes []*cdp.Node
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector, chromedp.ByQueryAll),
|
||||
chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nodes, err
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
// GetOuterHTML gets the outer HTML of the first element matching the selector
|
||||
func (b *Browser) GetOuterHTML(selector string) (string, error) {
|
||||
var html string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector, chromedp.ByQuery),
|
||||
chromedp.OuterHTML(selector, &html, chromedp.ByQuery),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return html, nil
|
||||
}
|
||||
|
||||
func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) {
|
||||
var text string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.Text(selector, &text, chromedp.ByQuery, chromedp.FromNode(node)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return text, nil
|
||||
}
|
||||
|
||||
func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr string) (string, error) {
|
||||
var value string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.AttributeValue(selector, attr, &value, nil, chromedp.ByQuery, chromedp.FromNode(node)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// value = strings.ReplaceAll(value, "/apply", "")
|
||||
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func Evaluate[T any](b *Browser, expression string) (T, error) {
|
||||
var result T
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.Evaluate(expression, &result),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (b *Browser) GetCurrentLocation() (string, error) {
|
||||
var finalURL string
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.Location(&finalURL),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Failed to get current location: %v", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return finalURL, nil
|
||||
}
|
||||
|
||||
func (b *Browser) GetHTML() (string, error) {
|
||||
var htmlContent string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get HTML content: %v", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return htmlContent, nil
|
||||
}
|
||||
|
||||
func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
return page.SetLifecycleEventsEnabled(true).Do(ctx)
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ch := make(chan struct{}, 1)
|
||||
chromedp.ListenTarget(b.ctx, func(ev interface{}) {
|
||||
if e, ok := ev.(*page.EventLifecycleEvent); ok {
|
||||
if e.Name == "networkIdle" {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
select {
|
||||
case <-ch:
|
||||
return nil
|
||||
case <-time.After(timeout):
|
||||
// Timeout reached, but don't error - page may still be usable
|
||||
return nil
|
||||
case <-b.ctx.Done():
|
||||
return b.ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Browser) NodeToString(node *cdp.Node) (string, error) {
|
||||
var outerHTML string
|
||||
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
h, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
outerHTML = h
|
||||
return nil
|
||||
}),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return outerHTML, err
|
||||
}
|
||||
|
||||
func (b *Browser) Type(selector string, value string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.SendKeys(selector, value),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Browser) Click(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Click(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Select chooses an option from a <select> element by value
|
||||
func (b *Browser) Select(selector string, value string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.SetValue(selector, value),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetCheckbox sets a checkbox to checked or unchecked state
|
||||
func (b *Browser) SetCheckbox(selector string, checked bool) error {
|
||||
var isChecked bool
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Evaluate(`document.querySelector('`+selector+`').checked`, &isChecked),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isChecked != checked {
|
||||
err = chromedp.Run(b.ctx,
|
||||
chromedp.Click(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetRadio clicks a radio button to select it
|
||||
func (b *Browser) SetRadio(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Click(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadFile sets a file input to the specified file path
|
||||
func (b *Browser) UploadFile(selector string, filePath string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.SetUploadFiles(selector, []string{filePath}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetValue sets the value of an input field directly (useful for hidden, date, color, etc.)
|
||||
func (b *Browser) SetValue(selector string, value string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitReady(selector),
|
||||
chromedp.SetValue(selector, value),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clear clears the value of an input field
|
||||
func (b *Browser) Clear(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Clear(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Focus focuses on an element
|
||||
func (b *Browser) Focus(selector string) error {
|
||||
err := chromedp.Run(b.ctx,
|
||||
chromedp.WaitVisible(selector),
|
||||
chromedp.Focus(selector),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RandomDelay waits for a random duration between min and max milliseconds
|
||||
func RandomDelay(minMs, maxMs int) {
|
||||
delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond
|
||||
time.Sleep(delay)
|
||||
}
|
||||
103
internal/browser/launch-chrome.go
Normal file
103
internal/browser/launch-chrome.go
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
package browser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"time"
|
||||
)
|
||||
|
||||
func launchChrome() (string, *exec.Cmd, error) {
|
||||
var chromePath string
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
||||
case "linux":
|
||||
chromePath = "google-chrome"
|
||||
case "windows":
|
||||
chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
|
||||
default:
|
||||
return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
// Kill any existing Chrome debug instances first
|
||||
exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
cmd := exec.Command(chromePath,
|
||||
"--remote-debugging-port=9222",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--user-data-dir=/tmp/chrome-debug-profile",
|
||||
|
||||
// Anti-detection flags
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-infobars",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-browser-side-navigation",
|
||||
"--disable-gpu",
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
|
||||
// Hide automation indicators
|
||||
"--disable-extensions",
|
||||
"--disable-plugins-discovery",
|
||||
"--disable-bundled-ppapi-flash",
|
||||
|
||||
// Reduce fingerprinting
|
||||
"--disable-background-networking",
|
||||
"--disable-sync",
|
||||
"--disable-translate",
|
||||
"--metrics-recording-only",
|
||||
"--safebrowsing-disable-auto-update",
|
||||
"--disable-client-side-phishing-detection",
|
||||
"--disable-default-apps",
|
||||
"--disable-hang-monitor",
|
||||
"--disable-popup-blocking",
|
||||
"--disable-prompt-on-repost",
|
||||
|
||||
// Window size to look like a real browser
|
||||
"--window-size=1920,1080",
|
||||
"--start-maximized",
|
||||
)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
|
||||
}
|
||||
|
||||
// Poll the debug endpoint to get the WebSocket URL
|
||||
var wsURL string
|
||||
for range 50 {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
resp, err := http.Get("http://127.0.0.1:9222/json/version")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var result struct {
|
||||
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
resp.Body.Close()
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if result.WebSocketDebuggerURL != "" {
|
||||
wsURL = result.WebSocketDebuggerURL
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if wsURL == "" {
|
||||
cmd.Process.Kill()
|
||||
return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
|
||||
}
|
||||
|
||||
// Wait for Chrome to be fully ready to accept connections
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
return wsURL, cmd, nil
|
||||
}
|
||||
5
internal/domain/go.mod
Normal file
5
internal/domain/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
module github.com/jobs-scraper/internal/domain
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
|
@ -1,6 +1,8 @@
|
|||
package domain
|
||||
|
||||
import "time"
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// JobProvider represents different job board providers
|
||||
type JobProvider int
|
||||
|
|
@ -15,6 +17,7 @@ const (
|
|||
TokyoDev
|
||||
JapanDev
|
||||
Google
|
||||
HiringCafe
|
||||
)
|
||||
|
||||
const (
|
||||
53
internal/dto/jobs.go
Normal file
53
internal/dto/jobs.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package dto
|
||||
|
||||
import "github.com/jobs-scraper/internal/domain"
|
||||
|
||||
type JobDTO struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Company string `json:"company"`
|
||||
CompanyLink string `json:"companyLink"`
|
||||
Location string `json:"location"`
|
||||
JobLink string `json:"jobLink"`
|
||||
JobPostTime string `json:"jobPostTime,omitempty"`
|
||||
Provider domain.JobProvider `json:"provider"`
|
||||
}
|
||||
|
||||
func JobFromDomain(j domain.Job) JobDTO {
|
||||
var postTime string
|
||||
if j.JobPostTime != nil {
|
||||
postTime = j.JobPostTime.Format("2006-01-02")
|
||||
}
|
||||
return JobDTO{
|
||||
ID: j.ID,
|
||||
Title: j.Title,
|
||||
Company: j.Company,
|
||||
CompanyLink: j.CompanyLink,
|
||||
Location: j.Location,
|
||||
JobLink: j.JobLink,
|
||||
JobPostTime: postTime,
|
||||
Provider: j.Provider,
|
||||
}
|
||||
}
|
||||
|
||||
func JobsFromDomain(jobs []domain.Job) []JobDTO {
|
||||
dtos := make([]JobDTO, len(jobs))
|
||||
for i, j := range jobs {
|
||||
dtos[i] = JobFromDomain(j)
|
||||
}
|
||||
return dtos
|
||||
}
|
||||
|
||||
type JobWithDescriptionDTO struct {
|
||||
JobDTO
|
||||
Description string `json:"description"`
|
||||
Criteria map[string]string `json:"criteria,omitempty"`
|
||||
}
|
||||
|
||||
func JobWithDescriptionFromDomain(j domain.JobWithDescription) JobWithDescriptionDTO {
|
||||
return JobWithDescriptionDTO{
|
||||
JobDTO: JobFromDomain(j.Job),
|
||||
Description: j.JobDescription.Description,
|
||||
Criteria: j.JobDescription.Criteria,
|
||||
}
|
||||
}
|
||||
|
|
@ -7,7 +7,7 @@ import (
|
|||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"google.golang.org/genai"
|
||||
)
|
||||
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module github.com/jobs-scraper/internal/pkg/gemini
|
||||
module github.com/jobs-scraper/internal/gemini
|
||||
|
||||
go 1.24.0
|
||||
|
||||
|
|
@ -152,22 +152,32 @@ func GetMigrationVersion(db *sql.DB) (uint, bool, error) {
|
|||
}
|
||||
|
||||
func getMigrationsPath() (string, error) {
|
||||
// Get the directory of the current source file
|
||||
_, filename, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("failed to get current file path")
|
||||
if envPath := os.Getenv("MIGRATIONS_PATH"); envPath != "" {
|
||||
if _, err := os.Stat(envPath); err == nil {
|
||||
return envPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Get the internal directory (go up from internal/pkg/infrastructure/ to internal/)
|
||||
// Current path: .../internal/pkg/infrastructure/db.go
|
||||
// Need to go up: infrastructure -> pkg -> internal
|
||||
paths := []string{
|
||||
"./internal/migrations",
|
||||
"../internal/migrations",
|
||||
"../../internal/migrations",
|
||||
}
|
||||
|
||||
for _, p := range paths {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
|
||||
_, filename, _, ok := runtime.Caller(0)
|
||||
if ok {
|
||||
internalDir := filepath.Dir(filepath.Dir(filepath.Dir(filename)))
|
||||
migrationsPath := filepath.Join(internalDir, "migrations")
|
||||
|
||||
// Verify the migrations directory exists
|
||||
if _, err := os.Stat(migrationsPath); err != nil {
|
||||
return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err)
|
||||
}
|
||||
|
||||
if _, err := os.Stat(migrationsPath); err == nil {
|
||||
return migrationsPath, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("migrations directory not found")
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
module github.com/jobs-scraper/internal/pkg/infrastructure
|
||||
module github.com/jobs-scraper/internal/infrastructure
|
||||
|
||||
go 1.24.0
|
||||
|
||||
|
|
@ -9,16 +9,16 @@ require (
|
|||
github.com/gorilla/mux v1.8.1
|
||||
github.com/jobs-scraper/libs/ports v0.0.0
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/utils v0.0.0
|
||||
github.com/jobs-scraper/internal/utils v0.0.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/rabbitmq/amqp091-go v1.10.0
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/libs/repo => ../../../libs/repo
|
||||
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/utils => ../utils
|
||||
replace github.com/jobs-scraper/internal/utils => ../utils
|
||||
|
||||
replace github.com/jobs-scraper/libs/ports => ../../../libs/ports
|
||||
replace github.com/jobs-scraper/libs/ports => ../../libs/ports
|
||||
|
||||
require (
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
|
|
@ -9,7 +9,7 @@ import (
|
|||
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/internal/pkg/utils"
|
||||
"github.com/jobs-scraper/internal/utils"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
|
@ -21,6 +21,7 @@ const (
|
|||
DeadLetterExchange = "scraper_dlx"
|
||||
CvAnalyzeExchange = "cv_exchange"
|
||||
CvAnalyzeQueue = "cv.analyze"
|
||||
JobApplierQueue = "job.applier"
|
||||
)
|
||||
|
||||
type RabbitMQClient struct {
|
||||
|
|
@ -31,10 +32,25 @@ type RabbitMQClient struct {
|
|||
type MessageHandler func([]byte) error
|
||||
|
||||
func NewRabbitMQClient() (*RabbitMQClient, error) {
|
||||
// Get RabbitMQ URL from environment or use default
|
||||
rabbitmqURL := os.Getenv("RABBITMQ_URL")
|
||||
if rabbitmqURL == "" {
|
||||
rabbitmqURL = "amqp://guest:guest@localhost:5672/"
|
||||
host := os.Getenv("RABBITMQ_HOST")
|
||||
if host == "" {
|
||||
host = "localhost"
|
||||
}
|
||||
port := os.Getenv("RABBITMQ_PORT")
|
||||
if port == "" {
|
||||
port = "5672"
|
||||
}
|
||||
user := os.Getenv("RABBITMQ_USER")
|
||||
if user == "" {
|
||||
user = "guest"
|
||||
}
|
||||
password := os.Getenv("RABBITMQ_PASSWORD")
|
||||
if password == "" {
|
||||
password = "guest"
|
||||
}
|
||||
rabbitmqURL = fmt.Sprintf("amqp://%s:%s@%s:%s/", user, password, host, port)
|
||||
}
|
||||
|
||||
// Connect to RabbitMQ with retry logic
|
||||
|
|
@ -114,6 +130,7 @@ func (r *RabbitMQClient) setupInfrastructure() error {
|
|||
JapanDevQueue: "scraper.japandev",
|
||||
GlassDoorQueue: "scraper.glassdoor",
|
||||
GoogleQueue: "scraper.google",
|
||||
JobApplierQueue: "job.applier",
|
||||
}
|
||||
|
||||
// Declare queues with dead letter exchange and TTL
|
||||
1
internal/migrations/009_job_indexes.down.sql
Normal file
1
internal/migrations/009_job_indexes.down.sql
Normal file
|
|
@ -0,0 +1 @@
|
|||
DROP INDEX IF EXISTS idx_jobs_provider_location;
|
||||
1
internal/migrations/009_job_indexes.up.sql
Normal file
1
internal/migrations/009_job_indexes.up.sql
Normal file
|
|
@ -0,0 +1 @@
|
|||
CREATE INDEX idx_jobs_provider_location ON jobs(provider, location);
|
||||
|
|
@ -36,8 +36,8 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/openai"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/openai"
|
||||
)
|
||||
|
||||
func main() {
|
||||
10
internal/openai/go.mod
Normal file
10
internal/openai/go.mod
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
module github.com/jobs-scraper/internal/openai
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/domain v0.0.0-00010101000000-000000000000
|
||||
github.com/sashabaranov/go-openai v1.41.2
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/domain => ../domain
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
package openai
|
||||
|
||||
import "github.com/jobs-scraper/internal/pkg/domain"
|
||||
import "github.com/jobs-scraper/internal/domain"
|
||||
|
||||
// JobAnalysisService defines the interface for job analysis services
|
||||
type JobAnalysisService interface {
|
||||
|
|
@ -8,7 +8,7 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/sashabaranov/go-openai"
|
||||
)
|
||||
|
||||
12
internal/openrouter/go.mod
Normal file
12
internal/openrouter/go.mod
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
module github.com/jobs-scraper/internal/openrouter
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/eduardolat/openroutergo v0.1.0
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
)
|
||||
|
||||
require github.com/orsinium-labs/enum v1.4.0 // indirect
|
||||
|
||||
replace github.com/jobs-scraper/internal/domain => ../domain
|
||||
|
|
@ -7,9 +7,56 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/eduardolat/openroutergo"
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
// FieldType represents HTML input field types
|
||||
type FieldType string
|
||||
|
||||
const (
|
||||
FieldTypeText FieldType = "text"
|
||||
FieldTypeEmail FieldType = "email"
|
||||
FieldTypeTel FieldType = "tel"
|
||||
FieldTypePassword FieldType = "password"
|
||||
FieldTypeNumber FieldType = "number"
|
||||
FieldTypeDate FieldType = "date"
|
||||
FieldTypeDatetimeLocal FieldType = "datetime-local"
|
||||
FieldTypeTime FieldType = "time"
|
||||
FieldTypeMonth FieldType = "month"
|
||||
FieldTypeWeek FieldType = "week"
|
||||
FieldTypeURL FieldType = "url"
|
||||
FieldTypeSearch FieldType = "search"
|
||||
FieldTypeColor FieldType = "color"
|
||||
FieldTypeRange FieldType = "range"
|
||||
FieldTypeFile FieldType = "file"
|
||||
FieldTypeHidden FieldType = "hidden"
|
||||
FieldTypeCheckbox FieldType = "checkbox"
|
||||
FieldTypeRadio FieldType = "radio"
|
||||
FieldTypeSelect FieldType = "select"
|
||||
FieldTypeTextarea FieldType = "textarea"
|
||||
FieldTypeButton FieldType = "button"
|
||||
FieldTypeSubmit FieldType = "submit"
|
||||
FieldTypeReset FieldType = "reset"
|
||||
)
|
||||
|
||||
// FormField represents a single form field extracted from HTML
|
||||
type FormField struct {
|
||||
Label string `json:"label"`
|
||||
FieldName string `json:"field_name"`
|
||||
FieldType FieldType `json:"field_type"`
|
||||
Value string `json:"value"`
|
||||
Placeholder string `json:"placeholder,omitempty"`
|
||||
Required bool `json:"required"`
|
||||
Selector string `json:"selector"`
|
||||
}
|
||||
|
||||
// FormExtractionResult represents the structured response from form extraction
|
||||
type FormExtractionResult struct {
|
||||
Fields []FormField `json:"fields"`
|
||||
ApplyButton string `json:"apply_button"`
|
||||
AdjustedCV string `json:"adjusted_cv"`
|
||||
}
|
||||
|
||||
// JobAnalysisResult represents the structured response from job analysis
|
||||
type JobAnalysisResult struct {
|
||||
Recommendation string `json:"recommendation"`
|
||||
|
|
@ -199,3 +246,117 @@ Extract from this HTML Content:
|
|||
|
||||
return jobDescription, nil
|
||||
}
|
||||
|
||||
// ExtractFormFields extracts form fields from HTML and populates them based on CV data.
|
||||
// It also adjusts the CV based on job type (frontend/backend/fullstack).
|
||||
func (s *OpenRouterService) CreateCV(htmlForm string, cvMarkdown string, jobDescription string) (*FormExtractionResult, error) {
|
||||
client, err := openroutergo.
|
||||
NewClient().
|
||||
WithAPIKey(s.apiKey).
|
||||
Create()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create client: %v", err)
|
||||
}
|
||||
|
||||
prompt := fmt.Sprintf(`You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
|
||||
1. Extract ALL form fields and populate them with appropriate values from the CV
|
||||
2. Adjust the CV based on the job type (frontend/backend/fullstack)
|
||||
|
||||
HTML FORM:
|
||||
%s
|
||||
|
||||
JOB DESCRIPTION:
|
||||
%s
|
||||
|
||||
ORIGINAL CV (Markdown):
|
||||
%s
|
||||
|
||||
CV ADJUSTMENT RULES:
|
||||
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
|
||||
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
|
||||
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
|
||||
- If FULLSTACK: Keep both frontend and backend content.
|
||||
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
|
||||
- Maintain the same markdown structure and formatting as the original CV.
|
||||
- Do NOT invent new skills or experience - only filter existing content.
|
||||
|
||||
CRITICAL RULES:
|
||||
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
|
||||
2. Extract ALL input fields, textareas, selects, and buttons from the form
|
||||
3. For each field, determine the appropriate value from the ADJUSTED CV
|
||||
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
|
||||
5. Identify the apply/submit button text
|
||||
6. Use the exact JSON schema below
|
||||
|
||||
JSON Schema:
|
||||
{
|
||||
"fields": [
|
||||
{
|
||||
"label": "Field label or name attribute",
|
||||
"field_name": "name or id attribute of the input",
|
||||
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
|
||||
"value": "Value to populate based on CV data",
|
||||
"placeholder": "Placeholder text if any",
|
||||
"required": true|false,
|
||||
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
|
||||
}
|
||||
],
|
||||
"apply_button": "Unique CSS selector for the form's submit button",
|
||||
"adjusted_cv": "The full adjusted CV in markdown format, tailored to the job type"
|
||||
}
|
||||
|
||||
FIELD MAPPING GUIDELINES:
|
||||
- Name fields: Extract full name, first name, last name from CV
|
||||
- Email: Use email from CV contact info
|
||||
- Phone: Use phone number from CV
|
||||
- LinkedIn/Portfolio/Website: Use URLs from CV
|
||||
- Experience/Years: Calculate from CV work history
|
||||
- Current company/title: Use most recent from CV
|
||||
- Skills: List relevant skills from ADJUSTED CV
|
||||
- Education: Use education details from CV
|
||||
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
|
||||
- Salary expectations: Leave empty unless specified in CV
|
||||
- Location/Address: Use from CV contact info
|
||||
- Resume/CV upload: Leave value empty (file upload)
|
||||
|
||||
Start your response with { and end with }`, htmlForm, jobDescription, cvMarkdown)
|
||||
|
||||
_, resp, err := client.
|
||||
NewChatCompletion().
|
||||
WithModel(s.model).
|
||||
WithSystemMessage("You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.").
|
||||
WithUserMessage(prompt).
|
||||
Execute()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to execute completion: %v", err)
|
||||
}
|
||||
|
||||
if len(resp.Choices) == 0 {
|
||||
return nil, fmt.Errorf("no response choices received from API")
|
||||
}
|
||||
|
||||
jsonContent := resp.Choices[0].Message.Content
|
||||
|
||||
log.Printf("Raw API response for form extraction: %s", jsonContent)
|
||||
|
||||
if len(jsonContent) > 0 && jsonContent[0] == '<' {
|
||||
return nil, fmt.Errorf("API returned HTML/XML instead of JSON. Response: %s", jsonContent[:min(200, len(jsonContent))])
|
||||
}
|
||||
|
||||
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
|
||||
|
||||
var result FormExtractionResult
|
||||
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
|
||||
if syntaxErr, ok := err.(*json.SyntaxError); ok {
|
||||
start := max(int(syntaxErr.Offset)-50, 0)
|
||||
end := int(syntaxErr.Offset) + 50
|
||||
if end > len(jsonContent) {
|
||||
end = len(jsonContent)
|
||||
}
|
||||
log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end])
|
||||
}
|
||||
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(500, len(jsonContent))])
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/domain
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/openai
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000
|
||||
github.com/sashabaranov/go-openai v1.41.2
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../domain
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/openrouter
|
||||
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/eduardolat/openroutergo v0.1.0
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
)
|
||||
|
||||
require github.com/orsinium-labs/enum v1.4.0 // indirect
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../domain
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
module github.com/jobs-scraper/internal/pkg/utils
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
|
|
@ -1,425 +0,0 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
// "crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
// "net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RetryConfig struct {
|
||||
MaxRetries int
|
||||
BaseDelay time.Duration
|
||||
MaxDelay time.Duration
|
||||
}
|
||||
|
||||
type RetryableHTTPRequestImpl struct {
|
||||
client *http.Client
|
||||
config RetryConfig
|
||||
}
|
||||
|
||||
func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl {
|
||||
return NewRetryableHTTPRequestWithProxy(config, "")
|
||||
}
|
||||
|
||||
func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl {
|
||||
// var transport *http.Transport
|
||||
|
||||
// if proxyURL != "" {
|
||||
// proxyUrl, err := url.Parse(proxyURL)
|
||||
// if err != nil {
|
||||
// panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err))
|
||||
// }
|
||||
|
||||
// transport = &http.Transport{
|
||||
// Proxy: http.ProxyURL(proxyUrl),
|
||||
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
// }
|
||||
// }
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
// Transport: transport,
|
||||
}
|
||||
|
||||
return &RetryableHTTPRequestImpl{
|
||||
client: client,
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
|
||||
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, body)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
||||
for _, header := range headers {
|
||||
req.Header.Set(header.Get("Key"), header.Get("Value"))
|
||||
}
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
|
||||
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
|
||||
// Success
|
||||
return resp, nil
|
||||
} else if resp.StatusCode == http.StatusTooManyRequests {
|
||||
// 429 Too Many Requests - retry with backoff
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
|
||||
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
|
||||
} else {
|
||||
// All other errors (4xx, 5xx) - don't retry
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
// max attempts for now = 3
|
||||
if attempt < s.config.MaxRetries {
|
||||
// Increment delay by 2 seconds for each attempt
|
||||
delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ...
|
||||
|
||||
// Cap the delay to MaxDelay if set
|
||||
if s.config.MaxDelay > 0 && delay > s.config.MaxDelay {
|
||||
delay = s.config.MaxDelay
|
||||
}
|
||||
|
||||
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(delay):
|
||||
// Continue to next attempt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
|
||||
}
|
||||
|
||||
// 5.10.245.81:80
|
||||
// 172.67.70.206:80
|
||||
// 190.93.245.33:80
|
||||
// 185.221.160.176:80
|
||||
// 141.101.123.156:80
|
||||
// 103.21.244.129:80
|
||||
// 103.169.142.59:80
|
||||
// 172.67.81.185:80
|
||||
// 37.18.73.60:5566
|
||||
// 209.38.83.56:1088
|
||||
// 89.169.36.109:1080
|
||||
// 103.21.244.248:80
|
||||
// 172.67.70.92:80
|
||||
// 172.67.88.34:80
|
||||
// 141.101.120.38:80
|
||||
// 172.67.75.255:80
|
||||
// 172.67.75.171:80
|
||||
// 45.131.4.157:80
|
||||
// 141.101.121.208:80
|
||||
// 141.101.120.70:80
|
||||
// 103.21.244.70:80
|
||||
// 103.21.244.140:80
|
||||
// 103.21.244.138:80
|
||||
// 154.16.146.43:80
|
||||
// 128.199.202.122:3128
|
||||
// 160.153.0.28:80
|
||||
// 185.170.166.31:80
|
||||
// 172.67.70.95:80
|
||||
// 141.101.123.91:80
|
||||
// 141.101.121.81:80
|
||||
// 141.101.121.71:80
|
||||
// 141.101.120.224:80
|
||||
// 192.111.137.35:4145
|
||||
// 45.12.31.20:80
|
||||
// 103.21.244.150:80
|
||||
// 103.21.244.214:80
|
||||
// 103.21.244.29:80
|
||||
// 195.85.23.88:80
|
||||
// 23.227.39.134:80
|
||||
// 66.42.224.229:41679
|
||||
// 172.67.73.123:80
|
||||
// 79.110.200.27:8000
|
||||
// 5.182.34.33:80
|
||||
// 103.21.244.174:80
|
||||
// 172.67.75.203:80
|
||||
// 141.101.122.150:80
|
||||
// 141.101.120.35:80
|
||||
// 175.47.237.95:6128
|
||||
// 103.21.244.185:80
|
||||
// 141.101.122.86:80
|
||||
// 141.101.121.89:80
|
||||
// 103.21.244.105:80
|
||||
// 103.21.244.234:80
|
||||
// 172.67.70.20:80
|
||||
// 163.223.172.27:1080
|
||||
// 172.67.84.128:80
|
||||
// 103.21.244.57:80
|
||||
// 31.43.179.191:80
|
||||
// 173.245.59.61:80
|
||||
// 141.101.120.48:80
|
||||
// 185.162.230.19:80
|
||||
// 172.67.172.150:80
|
||||
// 159.112.235.63:80
|
||||
// 172.64.40.184:80
|
||||
// 103.21.244.55:80
|
||||
// 103.21.244.49:80
|
||||
// 172.67.70.228:80
|
||||
// 103.21.244.8:80
|
||||
// 66.235.200.77:80
|
||||
// 45.85.119.147:80
|
||||
// 172.67.177.231:80
|
||||
// 173.245.49.28:80
|
||||
// 110.232.92.49:8080
|
||||
// 212.183.88.212:80
|
||||
// 108.162.193.73:80
|
||||
// 185.238.228.29:80
|
||||
// 172.67.127.8:80
|
||||
// 164.38.155.86:80
|
||||
// 160.153.0.18:80
|
||||
// 141.101.122.119:80
|
||||
// 103.21.244.186:80
|
||||
// 172.67.172.162:80
|
||||
// 8.218.39.40:10800
|
||||
// 47.237.132.101:60031
|
||||
// 222.59.173.105:44193
|
||||
// 143.110.190.60:1080
|
||||
// 152.53.194.55:32059
|
||||
// 115.187.50.40:5678
|
||||
// 171.254.219.180:1080
|
||||
// 103.165.157.155:8080
|
||||
// 45.233.169.57:999
|
||||
// 222.59.173.105:45035
|
||||
// 202.40.179.18:4145
|
||||
// 170.79.181.188:60606
|
||||
// 37.186.66.36:3629
|
||||
// 104.200.152.30:4145
|
||||
// 106.13.58.110:8888
|
||||
// 202.40.181.220:31247
|
||||
// 43.135.36.240:80
|
||||
// 103.82.27.107:10001
|
||||
// 192.252.209.158:4145
|
||||
// 142.54.236.97:4145
|
||||
// 142.54.239.1:4145
|
||||
// 184.170.245.148:4145
|
||||
// 68.71.247.130:4145
|
||||
// 72.195.34.58:4145
|
||||
// 172.65.90.2:80
|
||||
// 172.65.90.0:80
|
||||
// 36.138.53.26:10019
|
||||
// 183.215.23.242:9091
|
||||
// 45.131.7.34:80
|
||||
// 141.101.120.230:80
|
||||
// 172.64.78.193:80
|
||||
// 141.101.122.9:80
|
||||
// 103.21.244.80:80
|
||||
// 103.21.244.147:80
|
||||
// 103.21.244.106:80
|
||||
// 103.21.244.16:80
|
||||
// 199.34.229.210:80
|
||||
// 141.101.90.98:80
|
||||
// 172.67.70.127:80
|
||||
// 103.21.244.221:80
|
||||
// 172.67.171.203:80
|
||||
// 172.67.181.188:80
|
||||
// 72.195.101.99:4145
|
||||
// 141.101.120.106:80
|
||||
// 103.21.244.180:80
|
||||
// 103.21.244.102:80
|
||||
// 172.67.70.233:80
|
||||
// 108.162.193.107:80
|
||||
// 45.131.208.112:80
|
||||
// 154.194.12.195:80
|
||||
// 69.84.182.23:80
|
||||
// 141.101.120.128:80
|
||||
// 141.101.120.149:80
|
||||
// 141.101.120.232:80
|
||||
// 141.101.120.6:80
|
||||
// 172.67.98.18:80
|
||||
// 141.101.113.20:80
|
||||
// 208.65.90.21:4145
|
||||
// 198.177.254.131:4145
|
||||
// 142.54.237.38:4145
|
||||
// 192.252.220.89:4145
|
||||
// 199.187.210.54:4145
|
||||
// 199.102.105.242:4145
|
||||
// 199.102.107.145:4145
|
||||
// 198.8.84.3:4145
|
||||
// 98.182.171.161:4145
|
||||
// 192.111.129.150:4145
|
||||
// 184.170.248.5:4145
|
||||
// 98.188.47.150:4145
|
||||
// 125.228.94.232:4145
|
||||
// 125.228.94.153:4145
|
||||
// 185.162.229.219:80
|
||||
// 103.21.244.22:80
|
||||
// 172.64.89.97:80
|
||||
// 103.21.244.54:80
|
||||
// 103.21.244.37:80
|
||||
// 141.101.123.225:80
|
||||
// 103.21.244.168:80
|
||||
// 103.21.244.149:80
|
||||
// 103.21.244.100:80
|
||||
// 103.21.244.24:80
|
||||
// 23.227.39.209:80
|
||||
// 172.67.83.15:80
|
||||
// 45.131.4.241:80
|
||||
// 141.101.120.201:80
|
||||
// 172.67.191.237:80
|
||||
// 172.67.127.188:80
|
||||
// 103.21.244.92:80
|
||||
// 103.160.204.22:80
|
||||
// 220.197.44.36:3128
|
||||
// 39.185.41.193:5911
|
||||
// 23.227.39.121:80
|
||||
// 139.162.78.109:80
|
||||
// 188.114.99.144:80
|
||||
// 45.131.5.37:80
|
||||
// 45.131.4.250:80
|
||||
// 23.227.39.65:80
|
||||
// 141.101.121.191:80
|
||||
// 172.67.188.16:80
|
||||
// 172.67.254.148:80
|
||||
// 221.1.104.177:7302
|
||||
// 222.59.173.105:44008
|
||||
// 103.21.244.83:80
|
||||
// 172.64.149.1:80
|
||||
// 45.12.30.22:80
|
||||
// 222.59.173.105:44027
|
||||
// 172.64.149.26:80
|
||||
// 103.21.244.46:80
|
||||
// 45.12.31.242:80
|
||||
// 23.227.38.195:80
|
||||
// 172.67.177.162:80
|
||||
// 103.160.204.200:80
|
||||
// 172.67.70.129:80
|
||||
// 141.101.123.245:80
|
||||
// 185.162.230.117:80
|
||||
// 185.162.230.183:80
|
||||
// 69.61.200.104:36181
|
||||
// 208.65.90.3:4145
|
||||
// 72.223.188.92:4145
|
||||
// 192.252.214.17:4145
|
||||
// 68.71.242.118:4145
|
||||
// 192.252.210.233:4145
|
||||
// 107.181.161.81:4145
|
||||
// 107.181.168.145:4145
|
||||
// 206.220.175.2:4145
|
||||
// 72.37.217.3:4145
|
||||
// 192.252.208.70:14282
|
||||
// 70.166.167.55:57745
|
||||
// 192.111.137.37:18762
|
||||
// 98.188.47.132:4145
|
||||
// 62.99.138.162:80
|
||||
// 172.64.90.186:80
|
||||
// 172.67.74.57:80
|
||||
// 45.131.6.67:80
|
||||
// 141.101.122.37:80
|
||||
// 103.21.244.134:80
|
||||
// 172.64.155.71:80
|
||||
// 5.182.34.139:80
|
||||
// 45.131.6.31:80
|
||||
// 141.101.121.21:80
|
||||
// 141.101.120.190:80
|
||||
// 103.21.244.161:80
|
||||
// 103.21.244.156:80
|
||||
// 103.21.244.151:80
|
||||
// 172.64.89.0:80
|
||||
// 58.216.109.17:800
|
||||
// 58.241.88.18:800
|
||||
// 36.147.78.166:80
|
||||
// 170.244.26.36:8888
|
||||
// 170.244.25.52:8888
|
||||
// 170.244.26.206:8888
|
||||
// 36.138.53.26:10017
|
||||
// 201.148.32.162:80
|
||||
// 203.19.38.114:1080
|
||||
// 31.43.179.204:80
|
||||
// 45.12.31.50:80
|
||||
// 170.244.27.142:8888
|
||||
// 170.244.26.195:8888
|
||||
// 170.244.27.58:8888
|
||||
// 170.244.27.61:8888
|
||||
// 170.244.27.150:8888
|
||||
// 47.243.94.125:1080
|
||||
// 40.177.65.8:80
|
||||
// 47.57.13.107:80
|
||||
// 194.158.203.14:80
|
||||
// 194.219.134.234:80
|
||||
// 103.21.244.51:80
|
||||
// 103.21.244.160:80
|
||||
// 103.21.244.144:80
|
||||
// 213.33.126.130:80
|
||||
// 103.21.244.69:80
|
||||
// 103.21.244.192:80
|
||||
// 103.21.244.133:80
|
||||
// 141.193.213.189:80
|
||||
// 68.71.254.6:4145
|
||||
// 185.221.160.21:80
|
||||
// 188.114.99.97:80
|
||||
// 63.141.128.94:80
|
||||
// 141.193.213.213:80
|
||||
// 185.162.228.234:80
|
||||
// 213.143.113.82:80
|
||||
// 154.194.12.207:80
|
||||
// 189.203.181.34:1080
|
||||
// 202.144.134.150:5678
|
||||
// 190.242.157.215:8080
|
||||
// 72.49.49.11:31034
|
||||
// 142.54.237.34:4145
|
||||
// 68.71.249.153:48606
|
||||
// 192.252.209.155:14455
|
||||
// 192.252.216.86:4145
|
||||
// 142.54.229.249:4145
|
||||
// 209.97.150.167:8080
|
||||
// 192.252.220.92:17328
|
||||
// 98.178.72.21:10919
|
||||
// 144.124.228.87:1080
|
||||
// 183.240.46.42:80
|
||||
// 32.223.6.94:80
|
||||
// 192.252.214.20:15864
|
||||
// 198.177.252.24:4145
|
||||
// 192.252.208.67:14287
|
||||
// 198.8.94.170:4145
|
||||
// 68.71.241.33:4145
|
||||
// 98.181.137.83:4145
|
||||
// 45.12.30.139:80
|
||||
// 103.21.244.97:80
|
||||
// 65.1.148.157:80
|
||||
// 199.58.184.97:4145
|
||||
// 141.101.120.148:80
|
||||
// 185.238.228.77:80
|
||||
// 172.67.202.134:80
|
||||
// 172.67.182.49:80
|
||||
// 45.131.210.112:80
|
||||
// 141.101.120.100:80
|
||||
// 103.21.244.59:80
|
||||
// 103.21.244.189:80
|
||||
// 103.21.244.154:80
|
||||
// 103.21.244.23:80
|
||||
// 173.245.49.40:80
|
||||
// 82.200.235.134:38191
|
||||
// 43.224.118.89:2626
|
||||
// 119.148.47.226:16464
|
||||
// 211.230.49.122:3128
|
||||
// 62.171.159.232:8888
|
||||
// 115.127.112.34:1080
|
||||
// 185.191.236.162:3128
|
||||
// 41.223.119.156:3128
|
||||
// 103.138.123.242:8082
|
||||
// 121.169.46.116:1090
|
||||
// 185.145.185.218:8080
|
||||
// 45.166.93.113:999
|
||||
// 163.53.204.178:9813
|
||||
// 181.224.226.154:8080
|
||||
5
internal/utils/go.mod
Normal file
5
internal/utils/go.mod
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
module github.com/jobs-scraper/internal/utils
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.7
|
||||
26
internal/utils/pagination.go
Normal file
26
internal/utils/pagination.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
package utils
|
||||
|
||||
import "math"
|
||||
|
||||
type PageViewModel struct {
|
||||
Total int `json:"total"`
|
||||
PageNumber int `json:"pageNumber"`
|
||||
PageSize int `json:"pageSize"`
|
||||
TotalPages int `json:"totalPages"`
|
||||
HasPrevious bool `json:"hasPrevious"`
|
||||
HasNext bool `json:"hasNext"`
|
||||
}
|
||||
|
||||
// NewPageViewModel creates a new PageViewModel with calculated fields
|
||||
func NewPageViewModel(count, pageNumber, pageSize int) *PageViewModel {
|
||||
totalPages := int(math.Ceil(float64(count) / float64(pageSize)))
|
||||
|
||||
return &PageViewModel{
|
||||
Total: count,
|
||||
PageNumber: pageNumber,
|
||||
PageSize: pageSize,
|
||||
TotalPages: totalPages,
|
||||
HasPrevious: pageNumber > 1,
|
||||
HasNext: pageNumber < totalPages,
|
||||
}
|
||||
}
|
||||
110
internal/utils/retryable-http-request.go
Normal file
110
internal/utils/retryable-http-request.go
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
// "crypto/tls"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
// "net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RetryConfig struct {
|
||||
MaxRetries int
|
||||
BaseDelay time.Duration
|
||||
MaxDelay time.Duration
|
||||
}
|
||||
|
||||
type RetryableHTTPRequestImpl struct {
|
||||
client *http.Client
|
||||
config RetryConfig
|
||||
}
|
||||
|
||||
func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl {
|
||||
return NewRetryableHTTPRequestWithProxy(config, "")
|
||||
}
|
||||
|
||||
func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl {
|
||||
// var transport *http.Transport
|
||||
|
||||
// if proxyURL != "" {
|
||||
// proxyUrl, err := url.Parse(proxyURL)
|
||||
// if err != nil {
|
||||
// panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err))
|
||||
// }
|
||||
|
||||
// transport = &http.Transport{
|
||||
// Proxy: http.ProxyURL(proxyUrl),
|
||||
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
// }
|
||||
// }
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
// Transport: transport,
|
||||
}
|
||||
|
||||
return &RetryableHTTPRequestImpl{
|
||||
client: client,
|
||||
config: config,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, body)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
|
||||
for _, header := range headers {
|
||||
req.Header.Set(header.Get("Key"), header.Get("Value"))
|
||||
}
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
|
||||
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
|
||||
// Success
|
||||
return resp, nil
|
||||
} else if resp.StatusCode == http.StatusTooManyRequests {
|
||||
// 429 Too Many Requests - retry with backoff
|
||||
resp.Body.Close()
|
||||
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
|
||||
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
|
||||
} else {
|
||||
// All other errors (4xx, 5xx) - don't retry
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
|
||||
}
|
||||
|
||||
// max attempts for now = 3
|
||||
if attempt < s.config.MaxRetries {
|
||||
// Increment delay by 2 seconds for each attempt
|
||||
delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ...
|
||||
|
||||
// Cap the delay to MaxDelay if set
|
||||
if s.config.MaxDelay > 0 && delay > s.config.MaxDelay {
|
||||
delay = s.config.MaxDelay
|
||||
}
|
||||
|
||||
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(delay):
|
||||
// Continue to next attempt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
|
||||
}
|
||||
1146
libs/browser-automation/package-lock.json
generated
Normal file
1146
libs/browser-automation/package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load diff
25
libs/browser-automation/package.json
Normal file
25
libs/browser-automation/package.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "@jobs-scraper/browser-automation",
|
||||
"version": "1.0.0",
|
||||
"description": "Browser automation library using Puppeteer - ported from Go chromedp implementation",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "ts-node src/index.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": ["puppeteer", "browser", "automation", "scraping"],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"dotenv": "^17.2.3",
|
||||
"puppeteer-core": "^24.25.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
389
libs/browser-automation/src/browser.ts
Normal file
389
libs/browser-automation/src/browser.ts
Normal file
|
|
@ -0,0 +1,389 @@
|
|||
import puppeteer, {
|
||||
Browser as PuppeteerBrowser,
|
||||
Page,
|
||||
ElementHandle,
|
||||
} from "puppeteer-core";
|
||||
import { ChildProcess } from "child_process";
|
||||
import {
|
||||
launchChromeWithDebugging,
|
||||
waitForChromeReady,
|
||||
getChromeWebSocketUrl,
|
||||
} from "./utils/chrome-launcher";
|
||||
import { randomDelay } from "./utils/helpers";
|
||||
|
||||
export interface BrowserOptions {
|
||||
headless?: boolean;
|
||||
port?: number;
|
||||
userAgent?: string;
|
||||
}
|
||||
|
||||
export class Browser {
|
||||
private browser: PuppeteerBrowser | null = null;
|
||||
private page: Page | null = null;
|
||||
private chromeProcess: ChildProcess | null = null;
|
||||
private port: number;
|
||||
private userAgent: string;
|
||||
|
||||
constructor(options: BrowserOptions = {}) {
|
||||
this.port = options.port || 9222;
|
||||
this.userAgent =
|
||||
options.userAgent ||
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the browser by launching Chrome and connecting via Puppeteer
|
||||
*/
|
||||
async init(headless: boolean = false): Promise<void> {
|
||||
this.chromeProcess = launchChromeWithDebugging(this.port, headless);
|
||||
await waitForChromeReady(this.port);
|
||||
|
||||
const wsUrl = await getChromeWebSocketUrl(this.port);
|
||||
|
||||
this.browser = await puppeteer.connect({
|
||||
browserWSEndpoint: wsUrl,
|
||||
defaultViewport: null,
|
||||
});
|
||||
|
||||
console.log("Connected to browser successfully");
|
||||
|
||||
const pages = await this.browser.pages();
|
||||
if (pages.length === 0) {
|
||||
this.page = await this.browser.newPage();
|
||||
} else {
|
||||
this.page = pages[0];
|
||||
}
|
||||
|
||||
await this.page.setUserAgent(this.userAgent);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the browser and kill Chrome process
|
||||
*/
|
||||
async close(): Promise<void> {
|
||||
if (this.browser) {
|
||||
await this.browser.close();
|
||||
this.browser = null;
|
||||
}
|
||||
|
||||
if (this.chromeProcess) {
|
||||
this.chromeProcess.kill("SIGINT");
|
||||
this.chromeProcess = null;
|
||||
}
|
||||
|
||||
console.log("Browser closed");
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new tab
|
||||
*/
|
||||
async newTab(): Promise<Page> {
|
||||
if (!this.browser) {
|
||||
throw new Error("Browser not initialized");
|
||||
}
|
||||
return await this.browser.newPage();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current page
|
||||
*/
|
||||
getPage(): Page {
|
||||
if (!this.page) {
|
||||
throw new Error("Page not initialized");
|
||||
}
|
||||
return this.page;
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a URL
|
||||
*/
|
||||
async navigate(url: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await page.goto(url, {
|
||||
waitUntil: ["networkidle2", "domcontentloaded"],
|
||||
timeout: 60000,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an element to be visible
|
||||
*/
|
||||
async waitVisible(
|
||||
selector: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ElementHandle<Element> | null> {
|
||||
const page = this.getPage();
|
||||
return await page.waitForSelector(selector, {
|
||||
visible: true,
|
||||
timeout,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for an element to be ready in the DOM (not necessarily visible)
|
||||
*/
|
||||
async waitReady(
|
||||
selector: string,
|
||||
timeout: number = 30000
|
||||
): Promise<ElementHandle<Element> | null> {
|
||||
const page = this.getPage();
|
||||
return await page.waitForSelector(selector, {
|
||||
timeout,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all elements matching a selector
|
||||
*/
|
||||
async getNodes(selector: string): Promise<ElementHandle<Element>[]> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
return await page.$$(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the outer HTML of the first element matching the selector
|
||||
*/
|
||||
async getOuterHTML(selector: string): Promise<string> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
const html = await page.$eval(selector, (el) => el.outerHTML);
|
||||
return html;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get text content from an element
|
||||
*/
|
||||
async getText(selector: string): Promise<string> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
const text = await page.$eval(selector, (el) => el.textContent || "");
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an attribute value from an element
|
||||
*/
|
||||
async getAttribute(selector: string, attribute: string): Promise<string> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
const value = await page.$eval(
|
||||
selector,
|
||||
(el, attr) => el.getAttribute(attr) || "",
|
||||
attribute
|
||||
);
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate JavaScript in the browser context
|
||||
*/
|
||||
async evaluate<T>(expression: string): Promise<T> {
|
||||
const page = this.getPage();
|
||||
return (await page.evaluate(expression)) as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current page URL
|
||||
*/
|
||||
async getCurrentLocation(): Promise<string> {
|
||||
const page = this.getPage();
|
||||
return page.url();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the full HTML of the page
|
||||
*/
|
||||
async getHTML(): Promise<string> {
|
||||
const page = this.getPage();
|
||||
return await page.content();
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for network to be idle
|
||||
*/
|
||||
async waitForNetworkIdle(
|
||||
idleTime: number = 500,
|
||||
timeout: number = 30000
|
||||
): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await page.waitForNetworkIdle({ idleTime, timeout });
|
||||
}
|
||||
|
||||
/**
|
||||
* Type text into an input field (simulates keystrokes)
|
||||
*/
|
||||
async type(selector: string, value: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.type(selector, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Click on an element
|
||||
*/
|
||||
async click(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.click(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Select an option from a <select> element by value
|
||||
*/
|
||||
async select(selector: string, value: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.select(selector, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set a checkbox to checked or unchecked state
|
||||
*/
|
||||
async setCheckbox(selector: string, checked: boolean): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
|
||||
const isChecked = await page.$eval(
|
||||
selector,
|
||||
(el) => (el as HTMLInputElement).checked
|
||||
);
|
||||
|
||||
if (isChecked !== checked) {
|
||||
await page.click(selector);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Click a radio button to select it
|
||||
*/
|
||||
async setRadio(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.click(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload a file to a file input
|
||||
*/
|
||||
async uploadFile(selector: string, filePath: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitReady(selector);
|
||||
|
||||
const input = (await page.$(selector)) as ElementHandle<HTMLInputElement> | null;
|
||||
if (!input) {
|
||||
throw new Error(`File input not found: ${selector}`);
|
||||
}
|
||||
|
||||
await input.uploadFile(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the value of an input field directly (useful for hidden, date, color, etc.)
|
||||
*/
|
||||
async setValue(selector: string, value: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitReady(selector);
|
||||
|
||||
await page.$eval(
|
||||
selector,
|
||||
(el, val) => {
|
||||
(el as HTMLInputElement).value = val;
|
||||
el.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
el.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
},
|
||||
value
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the value of an input field
|
||||
*/
|
||||
async clear(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
|
||||
// Triple-click to select all text, then delete
|
||||
await page.click(selector, { clickCount: 3 });
|
||||
await page.keyboard.press("Backspace");
|
||||
}
|
||||
|
||||
/**
|
||||
* Focus on an element
|
||||
*/
|
||||
async focus(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.focus(selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Take a screenshot
|
||||
*/
|
||||
async screenshot(path?: string): Promise<Buffer> {
|
||||
const page = this.getPage();
|
||||
const buffer = await page.screenshot({ path });
|
||||
return buffer as Buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Random delay between min and max milliseconds
|
||||
*/
|
||||
async randomDelay(minMs: number, maxMs: number): Promise<void> {
|
||||
await randomDelay(minMs, maxMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Scroll to an element
|
||||
*/
|
||||
async scrollToElement(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.$eval(selector, (el) => {
|
||||
el.scrollIntoView({ behavior: "smooth", block: "center" });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an element exists
|
||||
*/
|
||||
async exists(selector: string): Promise<boolean> {
|
||||
const page = this.getPage();
|
||||
const element = await page.$(selector);
|
||||
return element !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the count of elements matching a selector
|
||||
*/
|
||||
async count(selector: string): Promise<number> {
|
||||
const page = this.getPage();
|
||||
const elements = await page.$$(selector);
|
||||
return elements.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Press a keyboard key
|
||||
*/
|
||||
async pressKey(key: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await page.keyboard.press(key as any);
|
||||
}
|
||||
|
||||
/**
|
||||
* Triple-click to select all text in an input
|
||||
*/
|
||||
async selectAllText(selector: string): Promise<void> {
|
||||
const page = this.getPage();
|
||||
await this.waitVisible(selector);
|
||||
await page.click(selector, { clickCount: 3 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear and type new value (common pattern)
|
||||
*/
|
||||
async clearAndType(selector: string, value: string): Promise<void> {
|
||||
await this.clear(selector);
|
||||
await this.type(selector, value);
|
||||
}
|
||||
}
|
||||
18
libs/browser-automation/src/index.ts
Normal file
18
libs/browser-automation/src/index.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
export { Browser, BrowserOptions } from "./browser";
|
||||
export {
|
||||
launchChromeWithDebugging,
|
||||
waitForChromeReady,
|
||||
getChromeWebSocketUrl,
|
||||
ChromeVersionResponse,
|
||||
} from "./utils/chrome-launcher";
|
||||
export { randomDelay, sleep } from "./utils/helpers";
|
||||
export {
|
||||
OpenRouterService,
|
||||
OpenRouterServiceOptions,
|
||||
FieldType,
|
||||
FormField,
|
||||
FormExtractionResult,
|
||||
JobAnalysisResult,
|
||||
JobDescription,
|
||||
getMockFormData,
|
||||
} from "./openrouter";
|
||||
12
libs/browser-automation/src/openrouter/index.ts
Normal file
12
libs/browser-automation/src/openrouter/index.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
export { OpenRouterService, OpenRouterServiceOptions } from "./openrouter";
|
||||
export {
|
||||
FieldType,
|
||||
FormField,
|
||||
FormExtractionResult,
|
||||
JobAnalysisResult,
|
||||
JobDescription,
|
||||
OpenRouterMessage,
|
||||
OpenRouterRequest,
|
||||
OpenRouterResponse,
|
||||
} from "./types";
|
||||
export { getMockFormData } from "./mock-data";
|
||||
102
libs/browser-automation/src/openrouter/mock-data.ts
Normal file
102
libs/browser-automation/src/openrouter/mock-data.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
import { FormExtractionResult } from "./types";
|
||||
|
||||
/**
|
||||
* Returns mock form extraction result for testing
|
||||
*/
|
||||
export function getMockFormData(): FormExtractionResult {
|
||||
return {
|
||||
fields: [
|
||||
{
|
||||
label: "Resume",
|
||||
field_name: "resume",
|
||||
field_type: "file",
|
||||
value: "",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[data-ui='resume']",
|
||||
},
|
||||
{
|
||||
label: "Cover letter",
|
||||
field_name: "cover_letter",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"Dear KOMOJU Hiring Team,\n\nI am excited to apply for the Fullstack Developer position at KOMOJU. With over 3 years of experience building web applications and leading development projects, I am confident I can contribute to your Fraud Prevention Team's mission of protecting merchants and customers across borders.\n\nMy experience includes leading the development of online shopping platforms, mentoring junior developers, and coordinating complex projects. I have hands-on experience with modern web technologies including React, Node.js, and microservices architecture. I am a motivated self-starter who can work independently while collaborating effectively with cross-functional teams.\n\nI am eager to bring my technical skills and passion for product-focused development to KOMOJU's innovative payment gateway solutions.\n\nThank you for your consideration.\n\nBest regards,\nZiad Elshimy",
|
||||
placeholder: "",
|
||||
required: false,
|
||||
selector: "#cover_letter",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"Do you have at least 2 years of personal or professional experience with Ruby?",
|
||||
field_name: "QA_10609776",
|
||||
field_type: "radio",
|
||||
value: "false",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609776']",
|
||||
},
|
||||
{
|
||||
label: "Could you tell us more about your Ruby experience?",
|
||||
field_name: "QA_10609777",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"I do not have professional Ruby experience, but I have extensive experience with JavaScript (React, Node.js), TypeScript, and Go. I am a fast learner with strong fundamentals in software development best practices, data structures, and algorithms. I am excited about the opportunity to learn Ruby and contribute to KOMOJU's fraud prevention systems.",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "#QA_10609777",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"Have you ever investigated and resolved a production issue that wasn't reproducible in a development or staging environment?",
|
||||
field_name: "QA_10609885",
|
||||
field_type: "radio",
|
||||
value: "true",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609885']",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"If the answer for above question is yes, can you tell us how did you approach it?",
|
||||
field_name: "QA_10609886",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"Yes, I have experience debugging production issues. My approach includes: 1) Analyzing production logs and error reports to identify patterns, 2) Using monitoring tools to track system behavior and performance metrics, 3) Creating targeted tests to reproduce the issue in isolated environments, 4) Collaborating with team members to validate hypotheses, and 5) Implementing and deploying fixes with proper testing and monitoring. I understand the importance of being systematic and thorough when diagnosing complex production issues.",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "#QA_10609886",
|
||||
},
|
||||
{
|
||||
label: "Do you currently reside in Japan?",
|
||||
field_name: "QA_10609774",
|
||||
field_type: "radio",
|
||||
value: "false",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609774']",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"This role requires at least 5 hours of overlap with Japan Standard Time (JST) business hours. Are you able to accommodate this?",
|
||||
field_name: "QA_10609887",
|
||||
field_type: "radio",
|
||||
value: "true",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "input[name='QA_10609887']",
|
||||
},
|
||||
{
|
||||
label:
|
||||
"Are you willing to relocate to Japan? If so, what are your motivations for doing so?",
|
||||
field_name: "QA_10609888",
|
||||
field_type: "textarea",
|
||||
value:
|
||||
"Yes, I am willing to relocate to Japan. My motivations include: 1) Joining KOMOJU's innovative team working on cutting-edge payment technology, 2) Experiencing Japan's world-class technology culture and work ethic, 3) Contributing to a product that powers payments for major platforms like Steam and TikTok, 4) Growing my career in an international environment with diverse perspectives, and 5) Embracing the opportunity to learn Japanese culture and language while working on globally impactful projects.",
|
||||
placeholder: "",
|
||||
required: true,
|
||||
selector: "#QA_10609888",
|
||||
},
|
||||
],
|
||||
apply_button: `button[data-ui="apply-button"]`,
|
||||
};
|
||||
}
|
||||
256
libs/browser-automation/src/openrouter/openrouter.ts
Normal file
256
libs/browser-automation/src/openrouter/openrouter.ts
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
import {
|
||||
FormExtractionResult,
|
||||
JobAnalysisResult,
|
||||
JobDescription,
|
||||
OpenRouterRequest,
|
||||
OpenRouterResponse,
|
||||
} from "./types";
|
||||
|
||||
export interface OpenRouterServiceOptions {
|
||||
model: string;
|
||||
apiKey: string;
|
||||
}
|
||||
|
||||
export class OpenRouterService {
|
||||
private model: string;
|
||||
private apiKey: string;
|
||||
private baseUrl = "https://openrouter.ai/api/v1/chat/completions";
|
||||
|
||||
constructor(options: OpenRouterServiceOptions) {
|
||||
this.model = options.model;
|
||||
this.apiKey = options.apiKey;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean markdown code blocks from response
|
||||
*/
|
||||
private cleanMarkdownCodeBlocks(content: string): string {
|
||||
content = content.trim();
|
||||
|
||||
if (content.startsWith("```json")) {
|
||||
content = content.slice(7);
|
||||
} else if (content.startsWith("```")) {
|
||||
content = content.slice(3);
|
||||
}
|
||||
|
||||
if (content.endsWith("```")) {
|
||||
content = content.slice(0, -3);
|
||||
}
|
||||
|
||||
return content.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a request to OpenRouter API
|
||||
*/
|
||||
private async makeRequest(
|
||||
systemMessage: string,
|
||||
userMessage: string
|
||||
): Promise<string> {
|
||||
const request: OpenRouterRequest = {
|
||||
model: this.model,
|
||||
messages: [
|
||||
{ role: "system", content: systemMessage },
|
||||
{ role: "user", content: userMessage },
|
||||
],
|
||||
};
|
||||
|
||||
const response = await fetch(this.baseUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
throw new Error(
|
||||
`OpenRouter API error: ${response.status} - ${errorText}`
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await response.json()) as OpenRouterResponse;
|
||||
|
||||
if (!data.choices || data.choices.length === 0) {
|
||||
throw new Error("No response choices received from API");
|
||||
}
|
||||
|
||||
const content = data.choices[0].message.content;
|
||||
|
||||
if (content.startsWith("<")) {
|
||||
throw new Error(
|
||||
`API returned HTML/XML instead of JSON: ${content.slice(0, 200)}`
|
||||
);
|
||||
}
|
||||
|
||||
return this.cleanMarkdownCodeBlocks(content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a job description against a CV
|
||||
*/
|
||||
async analyzeJobDescription(
|
||||
cv: string,
|
||||
jobDesc: JobDescription
|
||||
): Promise<JobAnalysisResult> {
|
||||
const userMessage = `Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
|
||||
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
|
||||
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
|
||||
|
||||
2) The job shouldn't require any language skills, preferbly only english.
|
||||
|
||||
3) The job should be remote, or provide relocation to the country.
|
||||
|
||||
CV:
|
||||
${cv}
|
||||
|
||||
Job Description:
|
||||
${jobDesc.description}
|
||||
|
||||
Job Criteria (key-value):
|
||||
${JSON.stringify(jobDesc.criteria, null, 2)}
|
||||
|
||||
CRITICAL OUTPUT REQUIREMENTS:
|
||||
- Return ONLY raw JSON - NO markdown formatting whatsoever
|
||||
- NO backticks, NO code blocks, NO json prefix
|
||||
- NO additional text before or after the JSON
|
||||
- Start your response directly with { and end with }
|
||||
- Use this exact schema and key names:
|
||||
{
|
||||
"recommendation": "apply" | "do_not_apply",
|
||||
"confidence_score": number, // integer 0-100
|
||||
"matching_skills": [string],
|
||||
"missing_skills": [string],
|
||||
"experience_match": "excellent" | "good" | "fair" | "poor",
|
||||
"summary": string,
|
||||
"improvement_suggestions": [string]
|
||||
}`;
|
||||
|
||||
const systemMessage =
|
||||
"You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
|
||||
|
||||
const jsonContent = await this.makeRequest(systemMessage, userMessage);
|
||||
|
||||
console.log("Raw API response:", jsonContent);
|
||||
|
||||
const result = JSON.parse(jsonContent) as JobAnalysisResult;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a job description from HTML content
|
||||
*/
|
||||
async createJobDescription(htmlContent: string): Promise<JobDescription> {
|
||||
const userMessage = `You are a job posting data extractor. Analyze the HTML content and extract data according to this JSON structure:
|
||||
{"description":"job description text","criteria":{"title":"Job Title","company":"Company","location":"Location","salary":"Salary","skills":"Skills","experience":"Experience","job_type":"Job Type","remote":"Remote"}}
|
||||
|
||||
CRITICAL RULES:
|
||||
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
|
||||
2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON)
|
||||
3. All criteria values MUST be strings (never use booleans, numbers, or arrays)
|
||||
4. Properly escape all quotes inside strings using \\"
|
||||
5. Use only standard ASCII quotes ("), never smart quotes (" " ' ')
|
||||
7. If no job description is found, use empty string for "description": ""
|
||||
8. All criteria fields must be present with empty string "" if not found
|
||||
|
||||
Example output:
|
||||
{"description":"Develop software applications...","criteria":{"title":"Backend Engineer","company":"Tech Corp","location":"Remote","salary":"$120k","skills":"Go, Docker","experience":"3+ years","job_type":"Full-time","remote":"Yes"}}
|
||||
|
||||
Extract from this HTML Content:
|
||||
${htmlContent}`;
|
||||
|
||||
const systemMessage =
|
||||
"You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
|
||||
|
||||
const jsonContent = await this.makeRequest(systemMessage, userMessage);
|
||||
|
||||
console.log("Raw API response for job description:", jsonContent);
|
||||
|
||||
const result = JSON.parse(jsonContent) as JobDescription;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract form fields from HTML and populate them based on CV data.
|
||||
* Also adjusts the CV based on job type (frontend/backend/fullstack).
|
||||
*/
|
||||
async applyForJob(
|
||||
htmlForm: string,
|
||||
cvMarkdown: string,
|
||||
jobDescription: string
|
||||
): Promise<FormExtractionResult> {
|
||||
const userMessage = `You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
|
||||
1. Extract ALL form fields and populate them with appropriate values from the CV
|
||||
|
||||
HTML FORM:
|
||||
${htmlForm}
|
||||
|
||||
JOB DESCRIPTION:
|
||||
${jobDescription}
|
||||
|
||||
ORIGINAL CV (Markdown):
|
||||
${cvMarkdown}
|
||||
|
||||
CV ADJUSTMENT RULES:
|
||||
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
|
||||
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
|
||||
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
|
||||
- If FULLSTACK: Keep both frontend and backend content.
|
||||
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
|
||||
- Maintain the same markdown structure and formatting as the original CV.
|
||||
- Do NOT invent new skills or experience - only filter existing content.
|
||||
|
||||
CRITICAL RULES:
|
||||
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
|
||||
2. Extract ALL input fields, textareas, selects, and buttons from the form
|
||||
3. For each field, determine the appropriate value from the ADJUSTED CV
|
||||
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
|
||||
5. Identify the apply/submit button text
|
||||
6. Use the exact JSON schema below
|
||||
7. If a field is already populated, ignore it (don't send it in the results)
|
||||
|
||||
JSON Schema:
|
||||
{
|
||||
"fields": [
|
||||
{
|
||||
"label": "Field label or name attribute",
|
||||
"field_name": "name or id attribute of the input",
|
||||
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
|
||||
"value": "Value to populate based on CV data",
|
||||
"placeholder": "Placeholder text if any",
|
||||
"required": true|false,
|
||||
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
|
||||
}
|
||||
],
|
||||
"apply_button": "Unique CSS selector for the form's submit button",
|
||||
}
|
||||
|
||||
FIELD MAPPING GUIDELINES:
|
||||
- Name fields: Extract full name, first name, last name from CV
|
||||
- Email: Use email from CV contact info
|
||||
- Phone: Use phone number from CV
|
||||
- LinkedIn/Portfolio/Website: Use URLs from CV
|
||||
- Experience/Years: Calculate from CV work history
|
||||
- Current company/title: Use most recent from CV
|
||||
- Skills: List relevant skills from ADJUSTED CV
|
||||
- Education: Use education details from CV
|
||||
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
|
||||
- Salary expectations: Leave empty unless specified in CV
|
||||
- Location/Address: Use from CV contact info
|
||||
- Resume/CV upload: Leave value empty (file upload)
|
||||
|
||||
Start your response with { and end with }`;
|
||||
|
||||
const systemMessage =
|
||||
"You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
|
||||
|
||||
const jsonContent = await this.makeRequest(systemMessage, userMessage);
|
||||
|
||||
console.log("Raw API response for form extraction:", jsonContent);
|
||||
|
||||
const result = JSON.parse(jsonContent) as FormExtractionResult;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
104
libs/browser-automation/src/openrouter/types.ts
Normal file
104
libs/browser-automation/src/openrouter/types.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
/**
|
||||
* FieldType represents HTML input field types
|
||||
*/
|
||||
export type FieldType =
|
||||
| "text"
|
||||
| "email"
|
||||
| "tel"
|
||||
| "password"
|
||||
| "number"
|
||||
| "date"
|
||||
| "datetime-local"
|
||||
| "time"
|
||||
| "month"
|
||||
| "week"
|
||||
| "url"
|
||||
| "search"
|
||||
| "color"
|
||||
| "range"
|
||||
| "file"
|
||||
| "hidden"
|
||||
| "checkbox"
|
||||
| "radio"
|
||||
| "select"
|
||||
| "textarea"
|
||||
| "button"
|
||||
| "submit"
|
||||
| "reset";
|
||||
|
||||
/**
|
||||
* FormField represents a single form field extracted from HTML
|
||||
*/
|
||||
export interface FormField {
|
||||
label: string;
|
||||
field_name: string;
|
||||
field_type: FieldType;
|
||||
value: string;
|
||||
placeholder?: string;
|
||||
required: boolean;
|
||||
selector: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* FormExtractionResult represents the structured response from form extraction
|
||||
*/
|
||||
export interface FormExtractionResult {
|
||||
fields: FormField[];
|
||||
apply_button: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* JobAnalysisResult represents the structured response from job analysis
|
||||
*/
|
||||
export interface JobAnalysisResult {
|
||||
recommendation: "apply" | "do_not_apply";
|
||||
confidence_score: number;
|
||||
matching_skills: string[];
|
||||
missing_skills: string[];
|
||||
experience_match: "excellent" | "good" | "fair" | "poor";
|
||||
summary: string;
|
||||
improvement_suggestions: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* JobDescription represents a parsed job description
|
||||
*/
|
||||
export interface JobDescription {
|
||||
description: string;
|
||||
criteria: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter API message format
|
||||
*/
|
||||
export interface OpenRouterMessage {
|
||||
role: "system" | "user" | "assistant";
|
||||
content: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter API request body
|
||||
*/
|
||||
export interface OpenRouterRequest {
|
||||
model: string;
|
||||
messages: OpenRouterMessage[];
|
||||
}
|
||||
|
||||
/**
|
||||
* OpenRouter API response
|
||||
*/
|
||||
export interface OpenRouterResponse {
|
||||
id: string;
|
||||
choices: {
|
||||
message: {
|
||||
role: string;
|
||||
content: string;
|
||||
};
|
||||
finish_reason: string;
|
||||
}[];
|
||||
usage?: {
|
||||
prompt_tokens: number;
|
||||
completion_tokens: number;
|
||||
total_tokens: number;
|
||||
};
|
||||
}
|
||||
144
libs/browser-automation/src/utils/chrome-launcher.ts
Normal file
144
libs/browser-automation/src/utils/chrome-launcher.ts
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
import os from "os";
|
||||
import { spawn, ChildProcess } from "child_process";
|
||||
import net from "net";
|
||||
|
||||
export interface ChromeVersionResponse {
|
||||
webSocketDebuggerUrl: string;
|
||||
}
|
||||
|
||||
const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile");
|
||||
|
||||
function getChromePath(): string {
|
||||
const platform = os.platform();
|
||||
|
||||
const paths: Record<NodeJS.Platform, string[]> = {
|
||||
darwin: [
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
||||
],
|
||||
win32: [
|
||||
process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
|
||||
process.env["PROGRAMFILES(X86)"] +
|
||||
"\\Google\\Chrome\\Application\\chrome.exe",
|
||||
],
|
||||
linux: [
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
"/usr/bin/chromium",
|
||||
"/usr/bin/chromium-browser",
|
||||
],
|
||||
aix: [],
|
||||
freebsd: [],
|
||||
openbsd: [],
|
||||
sunos: [],
|
||||
android: [],
|
||||
haiku: [],
|
||||
cygwin: [],
|
||||
netbsd: [],
|
||||
};
|
||||
|
||||
const platformPaths = paths[platform] || [];
|
||||
|
||||
for (const chromePath of platformPaths) {
|
||||
try {
|
||||
if (fs.existsSync(chromePath)) {
|
||||
return chromePath;
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error("Could not find Chrome/Chromium installation");
|
||||
}
|
||||
|
||||
export function launchChromeWithDebugging(
|
||||
port: number = 9222,
|
||||
headless: boolean = false
|
||||
): ChildProcess {
|
||||
try {
|
||||
const chromePath = getChromePath();
|
||||
const args = [
|
||||
`--remote-debugging-port=${port}`,
|
||||
`--user-data-dir=${userDataDir}`,
|
||||
"--remote-allow-origins=*",
|
||||
"--incognito",
|
||||
];
|
||||
|
||||
if (headless) {
|
||||
args.push("--headless=new");
|
||||
}
|
||||
|
||||
console.log(`Launching Chrome at: ${chromePath}`);
|
||||
|
||||
const chromeProcess = spawn(chromePath, args, {
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
});
|
||||
|
||||
chromeProcess.unref();
|
||||
|
||||
chromeProcess.on("error", (err) => {
|
||||
console.error("Failed to start Chrome:", err);
|
||||
});
|
||||
|
||||
chromeProcess.on("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
console.error(`Chrome process exited with code ${code}`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`Chrome launched successfully with debugging port ${port}`);
|
||||
return chromeProcess;
|
||||
} catch (error) {
|
||||
console.error("Failed to launch Chrome:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function waitForChromeReady(
|
||||
port: number = 9222,
|
||||
timeout: number = 30000
|
||||
): Promise<void> {
|
||||
const startTime = Date.now();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
function check() {
|
||||
const client = net.createConnection({ port }, () => {
|
||||
client.end();
|
||||
resolve();
|
||||
});
|
||||
|
||||
client.on("error", () => {
|
||||
if (Date.now() - startTime > timeout) {
|
||||
reject(new Error("Timeout waiting for Chrome to start"));
|
||||
} else {
|
||||
setTimeout(check, 500);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
check();
|
||||
});
|
||||
}
|
||||
|
||||
export async function getChromeWebSocketUrl(
|
||||
port: number = 9222
|
||||
): Promise<string> {
|
||||
const response = await fetch(`http://localhost:${port}/json/version`, {
|
||||
headers: {
|
||||
Origin: "",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to connect to Chrome debugging port");
|
||||
}
|
||||
|
||||
const chromeInstance = (await response.json()) as ChromeVersionResponse;
|
||||
return chromeInstance.webSocketDebuggerUrl;
|
||||
}
|
||||
14
libs/browser-automation/src/utils/helpers.ts
Normal file
14
libs/browser-automation/src/utils/helpers.ts
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
/**
|
||||
* Random delay between min and max milliseconds
|
||||
*/
|
||||
export function randomDelay(minMs: number, maxMs: number): Promise<void> {
|
||||
const delay = Math.floor(Math.random() * (maxMs - minMs) + minMs);
|
||||
return new Promise((resolve) => setTimeout(resolve, delay));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for a specified number of milliseconds
|
||||
*/
|
||||
export function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
34
libs/browser-automation/tsconfig.json
Normal file
34
libs/browser-automation/tsconfig.json
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
]
|
||||
}
|
||||
|
|
@ -4,6 +4,6 @@ go 1.24.0
|
|||
|
||||
toolchain go1.24.7
|
||||
|
||||
require github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
require github.com/jobs-scraper/internal/domain v0.0.0
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
package ports
|
||||
|
||||
import "github.com/jobs-scraper/internal/pkg/domain"
|
||||
import "github.com/jobs-scraper/internal/domain"
|
||||
|
||||
type JobCommands struct {
|
||||
CreateJob JobCommandHandler
|
||||
ApplyForJob ApplyForJobCommandHandler
|
||||
}
|
||||
|
||||
// CreateJobCommand represents the command to create a job
|
||||
|
|
@ -19,3 +20,7 @@ type CreateJobCommand struct {
|
|||
type JobCommandHandler interface {
|
||||
Handle(cmd CreateJobCommand) error
|
||||
}
|
||||
|
||||
type ApplyForJobCommandHandler interface {
|
||||
Handle(jobId int) error
|
||||
}
|
||||
|
|
|
|||
25
libs/ports/job-queries.go
Normal file
25
libs/ports/job-queries.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
package ports
|
||||
|
||||
import (
|
||||
"github.com/jobs-scraper/internal/dto"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
type JobQueries struct {
|
||||
GetJobs JobQueryHandler
|
||||
}
|
||||
|
||||
// GetJobQuery represents the command to create a job
|
||||
type GetJobQuery struct {
|
||||
Location string `schema:"location"`
|
||||
Keywords string `schema:"keywords"`
|
||||
FWT string `schema:"fwt"`
|
||||
Provider domain.JobProvider `schema:"provider"`
|
||||
PageNumber int `schema:"pageNumber"`
|
||||
PageSize int `schema:"pageSize"`
|
||||
}
|
||||
|
||||
// JobQueryHandler defines the interface for handling job queries
|
||||
type JobQueryHandler interface {
|
||||
Handle(query GetJobQuery) ([]dto.JobDTO, int, error)
|
||||
}
|
||||
102
libs/rabbitmq-ts/package-lock.json
generated
Normal file
102
libs/rabbitmq-ts/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
{
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"amqplib": "^0.10.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/amqplib": "^0.10.8",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/amqplib": {
|
||||
"version": "0.10.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz",
|
||||
"integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.3.tgz",
|
||||
"integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/amqplib": {
|
||||
"version": "0.10.9",
|
||||
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz",
|
||||
"integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"buffer-more-ints": "~1.0.0",
|
||||
"url-parse": "~1.5.10"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
}
|
||||
},
|
||||
"node_modules/buffer-more-ints": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz",
|
||||
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/querystringify": {
|
||||
"version": "2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
|
||||
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/requires-port": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
|
||||
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
|
||||
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/url-parse": {
|
||||
"version": "1.5.10",
|
||||
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
|
||||
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"querystringify": "^2.1.1",
|
||||
"requires-port": "^1.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
21
libs/rabbitmq-ts/package.json
Normal file
21
libs/rabbitmq-ts/package.json
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"dependencies": {
|
||||
"amqplib": "^0.10.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/amqplib": "^0.10.8",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
173
libs/rabbitmq-ts/src/index.ts
Normal file
173
libs/rabbitmq-ts/src/index.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
import * as amqp from "amqplib";
|
||||
|
||||
export interface RabbitMQConfig {
|
||||
url: string;
|
||||
queueName: string;
|
||||
exchangeName: string;
|
||||
exchangeType: "direct" | "topic" | "fanout" | "headers";
|
||||
durable: boolean;
|
||||
}
|
||||
|
||||
export interface SearchQuery {
|
||||
keywords: string;
|
||||
location: string;
|
||||
fwt?: string;
|
||||
numPages: number;
|
||||
}
|
||||
|
||||
export class RabbitMQClient {
|
||||
private connection: amqp.ChannelModel | null = null;
|
||||
private channel: amqp.Channel | null = null;
|
||||
private config: RabbitMQConfig;
|
||||
|
||||
constructor(config: RabbitMQConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
try {
|
||||
console.log(`Connecting to RabbitMQ at: ${this.config.url}`);
|
||||
this.connection = await amqp.connect(this.config.url);
|
||||
console.log("Connected to RabbitMQ successfully");
|
||||
|
||||
if (!this.connection) {
|
||||
throw new Error("Failed to establish RabbitMQ connection");
|
||||
}
|
||||
|
||||
this.channel = await this.connection.createChannel();
|
||||
|
||||
if (!this.channel) {
|
||||
throw new Error("Failed to create RabbitMQ channel");
|
||||
}
|
||||
|
||||
// Ensure exchange and queue exist
|
||||
await this.channel.assertExchange(
|
||||
this.config.exchangeName,
|
||||
this.config.exchangeType,
|
||||
{ durable: this.config.durable }
|
||||
);
|
||||
|
||||
await this.channel.assertQueue(this.config.queueName, {
|
||||
durable: this.config.durable,
|
||||
arguments: {
|
||||
"x-dead-letter-exchange": "scraper_dlx",
|
||||
"x-dead-letter-routing-key": this.config.queueName,
|
||||
"x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds
|
||||
"x-max-retries": 3,
|
||||
},
|
||||
});
|
||||
|
||||
await this.channel.bindQueue(
|
||||
this.config.queueName,
|
||||
this.config.exchangeName,
|
||||
this.config.queueName
|
||||
);
|
||||
|
||||
// Set prefetch to process one message at a time
|
||||
await this.channel.prefetch(1);
|
||||
|
||||
// Handle connection events
|
||||
this.connection.on("close", () => {
|
||||
console.log("RabbitMQ connection closed");
|
||||
this.connection = null;
|
||||
this.channel = null;
|
||||
});
|
||||
|
||||
this.connection.on("error", (error: Error) => {
|
||||
console.error("RabbitMQ connection error:", error);
|
||||
this.connection = null;
|
||||
this.channel = null;
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to connect to RabbitMQ:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async subscribe<T = SearchQuery>(
|
||||
messageHandler: (message: T) => Promise<void>
|
||||
): Promise<void> {
|
||||
if (!this.channel) {
|
||||
throw new Error(
|
||||
"RabbitMQ channel not initialized. Call connect() first."
|
||||
);
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Waiting for messages from ${this.config.queueName}. To exit press CTRL+C`
|
||||
);
|
||||
|
||||
await this.channel.consume(
|
||||
this.config.queueName,
|
||||
async (msg: amqp.ConsumeMessage | null) => {
|
||||
if (msg && this.channel) {
|
||||
try {
|
||||
const messageContent = msg.content.toString();
|
||||
console.log(`Received message: ${messageContent}`);
|
||||
|
||||
const message: T = JSON.parse(messageContent);
|
||||
console.log(`Processing message:`, message);
|
||||
|
||||
// Process the message using the provided handler
|
||||
await messageHandler(message);
|
||||
|
||||
// Acknowledge the message on success
|
||||
this.channel.ack(msg);
|
||||
console.log("Message processed successfully");
|
||||
} catch (error) {
|
||||
console.error("Error processing message:", error);
|
||||
// Reject the message and don't requeue it
|
||||
if (this.channel) {
|
||||
this.channel.nack(msg, false, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
if (this.channel) {
|
||||
try {
|
||||
await this.channel.close();
|
||||
} catch {
|
||||
// Channel may already be closing
|
||||
}
|
||||
this.channel = null;
|
||||
}
|
||||
if (this.connection) {
|
||||
try {
|
||||
await this.connection.close();
|
||||
} catch {
|
||||
// Connection may already be closing
|
||||
}
|
||||
this.connection = null;
|
||||
}
|
||||
console.log("RabbitMQ connection closed gracefully");
|
||||
} catch (error) {
|
||||
console.error("Error closing RabbitMQ connection:", error);
|
||||
}
|
||||
}
|
||||
|
||||
isConnected(): boolean {
|
||||
return this.connection !== null && this.channel !== null;
|
||||
}
|
||||
}
|
||||
|
||||
export function createRabbitMQConfig(): RabbitMQConfig {
|
||||
return {
|
||||
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
|
||||
queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor",
|
||||
exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange",
|
||||
exchangeType: "topic",
|
||||
durable: true,
|
||||
};
|
||||
}
|
||||
|
||||
export async function createRabbitMQClient(): Promise<RabbitMQClient> {
|
||||
const config = createRabbitMQConfig();
|
||||
const client = new RabbitMQClient(config);
|
||||
await client.connect();
|
||||
return client;
|
||||
}
|
||||
29
libs/rabbitmq-ts/tsconfig.json
Normal file
29
libs/rabbitmq-ts/tsconfig.json
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
|
|
@ -5,8 +5,8 @@ go 1.24.0
|
|||
toolchain go1.24.7
|
||||
|
||||
require (
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/lib/pq v1.10.9
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
type JobDescriptionRepository struct {
|
||||
|
|
|
|||
123
libs/repo/job.go
123
libs/repo/job.go
|
|
@ -3,10 +3,12 @@ package repo
|
|||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
)
|
||||
|
||||
type JobRepository struct {
|
||||
|
|
@ -94,33 +96,92 @@ func (r *JobRepository) SaveJobs(jobs []domain.Job) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (r *JobRepository) GetAllJobs() ([]domain.Job, error) {
|
||||
rows, err := r.db.Query("SELECT * FROM jobs")
|
||||
type JobFilter struct {
|
||||
Location string
|
||||
Keywords string
|
||||
Provider domain.JobProvider
|
||||
PageSize int
|
||||
PageNumber int
|
||||
}
|
||||
|
||||
func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) {
|
||||
query := "SELECT id, title, company, company_link, location, job_link, job_timestamp, provider FROM jobs"
|
||||
countQuery := "SELECT COUNT(*) FROM jobs"
|
||||
var conditions []string
|
||||
var args []interface{}
|
||||
paramIdx := 1
|
||||
|
||||
if filter.Location != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("location ILIKE $%d", paramIdx))
|
||||
args = append(args, "%"+filter.Location+"%")
|
||||
paramIdx++
|
||||
}
|
||||
|
||||
if filter.Keywords != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("(title ILIKE $%d OR company ILIKE $%d)", paramIdx, paramIdx))
|
||||
args = append(args, "%"+filter.Keywords+"%")
|
||||
paramIdx++
|
||||
}
|
||||
|
||||
if filter.Provider > 0 {
|
||||
conditions = append(conditions, fmt.Sprintf("provider = $%d", paramIdx))
|
||||
args = append(args, filter.Provider)
|
||||
paramIdx++
|
||||
}
|
||||
|
||||
whereClause := ""
|
||||
if len(conditions) > 0 {
|
||||
whereClause = " WHERE " + strings.Join(conditions, " AND ")
|
||||
query += whereClause
|
||||
countQuery += whereClause
|
||||
}
|
||||
|
||||
var totalCount int
|
||||
countArgs := make([]interface{}, len(args))
|
||||
copy(countArgs, args)
|
||||
if err := r.db.QueryRow(countQuery, countArgs...).Scan(&totalCount); err != nil {
|
||||
return nil, 0, fmt.Errorf("error counting jobs: %v", err)
|
||||
}
|
||||
|
||||
if filter.PageSize > 0 {
|
||||
query += fmt.Sprintf(" LIMIT $%d", paramIdx)
|
||||
args = append(args, filter.PageSize)
|
||||
paramIdx++
|
||||
|
||||
if filter.PageNumber > 0 {
|
||||
offset := (filter.PageNumber - 1) * filter.PageSize
|
||||
query += fmt.Sprintf(" OFFSET $%d", paramIdx)
|
||||
args = append(args, offset)
|
||||
paramIdx++
|
||||
}
|
||||
}
|
||||
|
||||
rows, err := r.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error querying jobs: %v", err)
|
||||
return nil, 0, fmt.Errorf("error querying jobs: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var jobs []domain.Job
|
||||
for rows.Next() {
|
||||
var job domain.Job
|
||||
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil {
|
||||
return nil, fmt.Errorf("error scanning job row: %v", err)
|
||||
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime, &job.Provider); err != nil {
|
||||
return nil, 0, fmt.Errorf("error scanning job row: %v", err)
|
||||
}
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("error iterating over job rows: %v", err)
|
||||
return nil, 0, fmt.Errorf("error iterating over job rows: %v", err)
|
||||
}
|
||||
|
||||
return jobs, nil
|
||||
return jobs, totalCount, nil
|
||||
}
|
||||
func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
||||
var job domain.Job
|
||||
|
||||
sqlStatement := `
|
||||
SELECT id, title, company, company_link, location, job_link, job_timestamp
|
||||
SELECT id, title, company, company_link, location, job_link, job_timestamp, provider
|
||||
FROM jobs
|
||||
WHERE id = $1
|
||||
`
|
||||
|
|
@ -133,6 +194,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
|||
&job.Location,
|
||||
&job.JobLink,
|
||||
&job.JobPostTime,
|
||||
&job.Provider,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
|
|
@ -146,6 +208,49 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
|
|||
return &job, nil
|
||||
}
|
||||
|
||||
func (r *JobRepository) GetJobWithDescription(id int) (*domain.JobWithDescription, error) {
|
||||
var job domain.JobWithDescription
|
||||
var criteriaJSON []byte
|
||||
|
||||
sqlStatement := `
|
||||
SELECT j.id, j.title, j.company, j.company_link, j.location, j.job_link, j.job_timestamp, j.provider,
|
||||
COALESCE(jd.description, ''), COALESCE(jd.job_criteria, '{}'::jsonb)
|
||||
FROM jobs j
|
||||
LEFT JOIN job_descriptions jd ON j.id = jd.job_id
|
||||
WHERE j.id = $1
|
||||
`
|
||||
|
||||
err := r.db.QueryRow(sqlStatement, id).Scan(
|
||||
&job.Job.ID,
|
||||
&job.Job.Title,
|
||||
&job.Job.Company,
|
||||
&job.Job.CompanyLink,
|
||||
&job.Job.Location,
|
||||
&job.Job.JobLink,
|
||||
&job.Job.JobPostTime,
|
||||
&job.Job.Provider,
|
||||
&job.JobDescription.Description,
|
||||
&criteriaJSON,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("job with ID %d not found", id)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error querying job: %v", err)
|
||||
}
|
||||
|
||||
// Parse criteria JSON
|
||||
if err := json.Unmarshal(criteriaJSON, &job.JobDescription.Criteria); err != nil {
|
||||
job.JobDescription.Criteria = make(map[string]string)
|
||||
}
|
||||
|
||||
job.JobDescription.JobID = job.Job.ID
|
||||
|
||||
return &job, nil
|
||||
}
|
||||
|
||||
func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) {
|
||||
bf := bytes.Buffer{}
|
||||
values := make([]interface{}, 0, len(models)*7)
|
||||
|
|
|
|||
20
services/api/.gitlab-ci.yml
Normal file
20
services/api/.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
deploy_api:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/api && docker compose up -d --build"
|
||||
36
services/api/Dockerfile
Normal file
36
services/api/Dockerfile
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
# Build stage
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
# Install build dependencies
|
||||
RUN apk add --no-cache git
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy go mod files
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy the entire monorepo
|
||||
COPY . .
|
||||
|
||||
# Build the API service
|
||||
WORKDIR /build/services/api
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o api ./cmd/server
|
||||
|
||||
# Runtime stage
|
||||
FROM alpine:latest
|
||||
|
||||
RUN apk add --no-cache ca-certificates
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the binary from builder
|
||||
COPY --from=builder /build/services/api/api .
|
||||
|
||||
# Copy migrations
|
||||
COPY --from=builder /build/internal/migrations ./internal/migrations
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Run the API
|
||||
ENTRYPOINT ["./api"]
|
||||
|
|
@ -10,8 +10,8 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/api/internal/app"
|
||||
httpHandler "github.com/jobs-scraper/services/api/pkg/http"
|
||||
|
|
@ -60,20 +60,22 @@ func main() {
|
|||
log.Fatalf("Failed to run migrations: %v", err)
|
||||
}
|
||||
|
||||
router := mux.NewRouter()
|
||||
log.Println("Successfully ran migrations")
|
||||
|
||||
// Swagger endpoint
|
||||
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
|
||||
router := mux.NewRouter()
|
||||
|
||||
router.Use(httpHandler.CORSMiddleware)
|
||||
router.Use(httpHandler.LogsMiddleware)
|
||||
|
||||
// Swagger endpoint
|
||||
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
|
||||
|
||||
app := app.NewApplication(router, db, rmq)
|
||||
|
||||
// Create job analysis result repository
|
||||
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
|
||||
|
||||
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo)
|
||||
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo, app.JobQueries)
|
||||
|
||||
jobHandler.RegisterRoutes(router)
|
||||
|
||||
|
|
|
|||
41
services/api/docker-compose.yml
Normal file
41
services/api/docker-compose.yml
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
services:
|
||||
api:
|
||||
build:
|
||||
context: ../../
|
||||
dockerfile: services/api/Dockerfile
|
||||
container_name: jobs-api
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
DB_HOST: shared_postgres
|
||||
DB_PORT: 5432
|
||||
DB_NAME: jobs_scraper
|
||||
DB_USER: scraper_user
|
||||
DB_PASSWORD: Cocowawa_12345
|
||||
|
||||
# RabbitMQ Configuration
|
||||
RABBITMQ_HOST: 172.18.0.9
|
||||
RABBITMQ_PORT: 5672
|
||||
RABBITMQ_USER: admin
|
||||
RABBITMQ_PASSWORD: 123Kari123!
|
||||
networks:
|
||||
- proxy
|
||||
restart: unless-stopped
|
||||
labels:
|
||||
- traefik.enable=true
|
||||
- traefik.docker.network=proxy
|
||||
- traefik.http.routers.api-jobs.rule=Host(`api-jobs.ai-assistant-bot.xyz`)
|
||||
- traefik.http.routers.api-jobs.entrypoints=web,websecure
|
||||
- traefik.http.routers.api-jobs.tls.certresolver=le
|
||||
- traefik.http.services.api-jobs.loadbalancer.server.port=8080
|
||||
# deploy:
|
||||
# resources:
|
||||
# limits:
|
||||
# memory: 512M
|
||||
# cpus: "1"
|
||||
# reservations:
|
||||
# memory: 128M
|
||||
# cpus: "0.25"
|
||||
networks:
|
||||
proxy:
|
||||
external: true
|
||||
|
|
@ -4,9 +4,10 @@ go 1.24.0
|
|||
|
||||
require (
|
||||
github.com/gorilla/mux v1.8.1
|
||||
github.com/jobs-scraper/internal/pkg/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/pkg/utils v0.0.0
|
||||
github.com/gorilla/schema v1.4.1
|
||||
github.com/jobs-scraper/internal/domain v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
|
||||
github.com/jobs-scraper/internal/utils v0.0.0
|
||||
github.com/jobs-scraper/libs/ports v0.0.0
|
||||
github.com/jobs-scraper/libs/repo v0.0.0
|
||||
github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000
|
||||
|
|
@ -38,11 +39,11 @@ require (
|
|||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
|
||||
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils
|
||||
replace github.com/jobs-scraper/internal/utils => ../../internal/utils
|
||||
|
||||
replace github.com/jobs-scraper/libs/ports => ../../libs/ports
|
||||
|
||||
|
|
|
|||
|
|
@ -44,6 +44,8 @@ github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9Knoi
|
|||
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
|
||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||
github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E=
|
||||
github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
|
|
@ -56,6 +58,7 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF
|
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
|
|
@ -113,7 +116,9 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
|||
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
|
|
@ -122,10 +127,12 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
|
|||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
|
||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
|
|
|
|||
|
|
@ -4,11 +4,12 @@ import (
|
|||
"database/sql"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/jobs-scraper/services/api/internal/commands/job"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/libs/server"
|
||||
jobCommands "github.com/jobs-scraper/services/api/internal/commands/job"
|
||||
jobQueries "github.com/jobs-scraper/services/api/internal/queries/job"
|
||||
)
|
||||
|
||||
type Application struct {
|
||||
|
|
@ -16,6 +17,7 @@ type Application struct {
|
|||
DB *sql.DB
|
||||
Server *server.AppServer
|
||||
JobCommands *ports.JobCommands
|
||||
JobQueries *ports.JobQueries
|
||||
}
|
||||
|
||||
func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient) *Application {
|
||||
|
|
@ -28,7 +30,11 @@ func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient
|
|||
Router: router,
|
||||
DB: db,
|
||||
JobCommands: &ports.JobCommands{
|
||||
CreateJob: job.NewCreateJobHandler(jobRepo, rmq),
|
||||
CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq),
|
||||
ApplyForJob: jobCommands.NewApplyForJobHandler(jobRepo, rmq),
|
||||
},
|
||||
JobQueries: &ports.JobQueries{
|
||||
GetJobs: jobQueries.NewGetJobsHandler(jobRepo),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
|
|||
44
services/api/internal/commands/job/apply-for-job.go
Normal file
44
services/api/internal/commands/job/apply-for-job.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package job
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/jobs-scraper/internal/dto"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
||||
type ApplyForJob struct {
|
||||
jobRepo *repo.JobRepository
|
||||
rmq *rabbitmq.RabbitMQClient
|
||||
}
|
||||
|
||||
func NewApplyForJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *ApplyForJob {
|
||||
return &ApplyForJob{
|
||||
jobRepo: jobRepo,
|
||||
rmq: rmq,
|
||||
}
|
||||
}
|
||||
|
||||
func (aj *ApplyForJob) Handle(jobId int) error {
|
||||
job, err := aj.jobRepo.GetJobWithDescription(jobId)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
jobDTO := dto.JobWithDescriptionFromDomain(*job)
|
||||
jsonData, err := json.Marshal(jobDTO)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = aj.rmq.Publish(rabbitmq.JobApplierQueue, rabbitmq.ScraperExchange, jsonData)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -5,8 +5,8 @@ import (
|
|||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
|
@ -33,25 +33,21 @@ func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error {
|
|||
|
||||
jsonData, err := json.Marshal(jobRequest)
|
||||
if err != nil {
|
||||
log.Printf("Error marshaling JSON: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
switch cmd.Provider {
|
||||
case domain.LinkedIn:
|
||||
if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
|
||||
log.Printf("Error publishing LinkedIn message: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Published LinkedIn job request: %s", string(jsonData))
|
||||
case domain.Glassdoor:
|
||||
if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
|
||||
log.Printf("Error publishing Glassdoor message: %v", err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Published Glassdoor job request: %s", string(jsonData))
|
||||
default:
|
||||
log.Printf("Unsupported job provider: %v", cmd.Provider)
|
||||
return errors.New("unsupported job provider")
|
||||
}
|
||||
|
||||
|
|
|
|||
34
services/api/internal/queries/job/get-jobs.go
Normal file
34
services/api/internal/queries/job/get-jobs.go
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
package job
|
||||
|
||||
import (
|
||||
"github.com/jobs-scraper/internal/dto"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
)
|
||||
|
||||
type GetJobs struct {
|
||||
jobRepo *repo.JobRepository
|
||||
}
|
||||
|
||||
func NewGetJobsHandler(jobRepo *repo.JobRepository) *GetJobs {
|
||||
return &GetJobs{
|
||||
jobRepo: jobRepo,
|
||||
}
|
||||
}
|
||||
|
||||
func (cj *GetJobs) Handle(query ports.GetJobQuery) ([]dto.JobDTO, int, error) {
|
||||
filter := repo.JobFilter{
|
||||
Location: query.Location,
|
||||
Keywords: query.Keywords,
|
||||
Provider: query.Provider,
|
||||
PageSize: query.PageSize,
|
||||
PageNumber: query.PageNumber,
|
||||
}
|
||||
|
||||
results, totalCount, err := cj.jobRepo.GetAllJobs(filter)
|
||||
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return dto.JobsFromDomain(results), totalCount, nil
|
||||
}
|
||||
|
|
@ -7,27 +7,31 @@ import (
|
|||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/gorilla/schema"
|
||||
"github.com/jobs-scraper/internal/utils"
|
||||
"github.com/jobs-scraper/libs/ports"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/internal/pkg/utils"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// JobHandler handles HTTP requests related to jobs
|
||||
type JobHandler struct {
|
||||
jobCommands *ports.JobCommands
|
||||
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
|
||||
jobQueries *ports.JobQueries
|
||||
}
|
||||
|
||||
// NewJobHandler creates a new instance of JobHandler
|
||||
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler {
|
||||
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository, jobQueries *ports.JobQueries) *JobHandler {
|
||||
return &JobHandler{
|
||||
jobCommands: jobCommands,
|
||||
jobAnalysisResultRepo: jobAnalysisResultRepo,
|
||||
jobQueries: jobQueries,
|
||||
}
|
||||
}
|
||||
|
||||
var decoder = schema.NewDecoder()
|
||||
|
||||
// RegisterRoutes registers all routes to the router
|
||||
func (h *JobHandler) RegisterRoutes(router *mux.Router) {
|
||||
jobs := router.PathPrefix("/jobs").Subrouter()
|
||||
|
|
@ -38,6 +42,8 @@ func (h *JobHandler) RegisterRoutes(router *mux.Router) {
|
|||
jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET")
|
||||
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
|
||||
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
|
||||
jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET")
|
||||
jobs.HandleFunc("/apply/{id}", utils.Make(h.ApplyForJob)).Methods("POST")
|
||||
}
|
||||
|
||||
// CreateJob handles the creation of a new job
|
||||
|
|
@ -68,6 +74,100 @@ func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Apply applies for a job by id
|
||||
// @Summary applies for a job
|
||||
// @Description Applies for a job with the provided id
|
||||
// @Tags jobs
|
||||
// @Param id query string false "Job ID"
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Success 201 {object} map[string]string "Successfully applied for job"
|
||||
// @Failure 400 {object} map[string]string "Invalid request data"
|
||||
// @Failure 500 {object} map[string]string "Internal server error"
|
||||
// @Router /jobs/apply/{id} [post]
|
||||
func (h *JobHandler) ApplyForJob(w http.ResponseWriter, r *http.Request) error {
|
||||
jobId := r.URL.Query().Get("id")
|
||||
|
||||
if jobId == "" {
|
||||
slog.Error("Job ID is required")
|
||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("job ID is required"))
|
||||
}
|
||||
|
||||
jobIdInt, err := strconv.Atoi(jobId)
|
||||
|
||||
if err != nil {
|
||||
slog.Error("Invalid Job ID", "error", err)
|
||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
|
||||
}
|
||||
|
||||
err = h.jobCommands.ApplyForJob.Handle(jobIdInt)
|
||||
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
return utils.NewAPIError(http.StatusInternalServerError, fmt.Errorf("Internal server error"))
|
||||
}
|
||||
|
||||
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetJobs handles the retrieval of all jobs
|
||||
// @Summary Gets all jobs
|
||||
// @Description Gets all jobs with the provided specifications
|
||||
// @Tags jobs
|
||||
// @Param location query string false "Location filter"
|
||||
// @Param keywords query string false "Keywords filter (searches title and company)"
|
||||
// @Param fwt query string false "Full/Part time filter"
|
||||
// @Param provider query int false "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)" Enums(0,1,2,3,4,5,6,7)
|
||||
// @Param pageNumber query int false "Page number"
|
||||
// @Param pageSize query int false "Number of elements"
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Success 200 {object} map[string]any "Successfully retrieved jobs"
|
||||
// @Failure 400 {object} map[string]any "Invalid request data"
|
||||
// @Failure 500 {object} map[string]any "Internal server error"
|
||||
// @Router /jobs [get]
|
||||
func (h *JobHandler) GetJobs(w http.ResponseWriter, r *http.Request) error {
|
||||
query := ports.GetJobQuery{
|
||||
PageNumber: 1,
|
||||
PageSize: 10,
|
||||
}
|
||||
|
||||
if pageNumberStr := r.URL.Query().Get("pageNumber"); pageNumberStr != "" {
|
||||
if page, err := strconv.Atoi(pageNumberStr); err == nil && page > 0 {
|
||||
query.PageNumber = page
|
||||
}
|
||||
}
|
||||
|
||||
if pageSizeStr := r.URL.Query().Get("pageSize"); pageSizeStr != "" {
|
||||
if pageSize, err := strconv.Atoi(pageSizeStr); err == nil && pageSize > 0 {
|
||||
query.PageSize = pageSize
|
||||
}
|
||||
}
|
||||
|
||||
if err := decoder.Decode(&query, r.URL.Query()); err != nil {
|
||||
slog.Error("Failed to decode query params", "error", err)
|
||||
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid query parameters: %w", err))
|
||||
}
|
||||
|
||||
result, totalCount, err := h.jobQueries.GetJobs.Handle(query)
|
||||
|
||||
if err != nil {
|
||||
slog.Error(err.Error())
|
||||
return utils.NewAPIError(http.StatusInternalServerError, err)
|
||||
}
|
||||
|
||||
pagination := utils.NewPageViewModel(totalCount, query.PageNumber, query.PageSize)
|
||||
|
||||
utils.WriteJSON(w, http.StatusOK, map[string]any{
|
||||
"jobs": result,
|
||||
"pagination": pagination,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAllAnalysisResults handles retrieving all job analysis results
|
||||
// @Summary Get all job analysis results
|
||||
// @Description Retrieves all job analysis results from the database
|
||||
|
|
|
|||
|
|
@ -13,11 +13,17 @@ import (
|
|||
func CORSMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Set CORS headers
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With")
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
origin = "*"
|
||||
}
|
||||
|
||||
w.Header().Set("Access-Control-Allow-Origin", origin)
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With, X-API-Key")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
w.Header().Set("Access-Control-Max-Age", "86400")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range")
|
||||
|
||||
// Handle preflight OPTIONS requests
|
||||
if r.Method == "OPTIONS" {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,90 @@ const docTemplate = `{
|
|||
"basePath": "{{.BasePath}}",
|
||||
"paths": {
|
||||
"/jobs": {
|
||||
"get": {
|
||||
"description": "Gets all jobs with the provided specifications",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"jobs"
|
||||
],
|
||||
"summary": "Gets all jobs",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Location filter",
|
||||
"name": "location",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Keywords filter (searches title and company)",
|
||||
"name": "keywords",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Full/Part time filter",
|
||||
"name": "fwt",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"enum": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7
|
||||
],
|
||||
"type": "integer",
|
||||
"description": "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)",
|
||||
"name": "provider",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"description": "Page number",
|
||||
"name": "pageNumber",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"description": "Number of elements",
|
||||
"name": "pageSize",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successfully retrieved jobs",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"description": "Creates a new job with the provided specifications",
|
||||
"consumes": [
|
||||
|
|
@ -151,6 +235,58 @@ const docTemplate = `{
|
|||
}
|
||||
}
|
||||
},
|
||||
"/jobs/apply/{id}": {
|
||||
"post": {
|
||||
"description": "Applies for a job with the provided id",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"jobs"
|
||||
],
|
||||
"summary": "applies for a job",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Job ID",
|
||||
"name": "id",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Successfully applied for job",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/jobs/{id}/analysis": {
|
||||
"get": {
|
||||
"description": "Retrieves the analysis result for a specific job",
|
||||
|
|
@ -254,7 +390,9 @@ const docTemplate = `{
|
|||
2,
|
||||
3,
|
||||
4,
|
||||
5
|
||||
5,
|
||||
6,
|
||||
7
|
||||
],
|
||||
"x-enum-varnames": [
|
||||
"LinkedIn",
|
||||
|
|
@ -262,7 +400,9 @@ const docTemplate = `{
|
|||
"Glassdoor",
|
||||
"Bayt",
|
||||
"TokyoDev",
|
||||
"JapanDev"
|
||||
"JapanDev",
|
||||
"Google",
|
||||
"HiringCafe"
|
||||
]
|
||||
},
|
||||
"ports.CreateJobCommand": {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,90 @@
|
|||
"basePath": "/",
|
||||
"paths": {
|
||||
"/jobs": {
|
||||
"get": {
|
||||
"description": "Gets all jobs with the provided specifications",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"jobs"
|
||||
],
|
||||
"summary": "Gets all jobs",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Location filter",
|
||||
"name": "location",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Keywords filter (searches title and company)",
|
||||
"name": "keywords",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Full/Part time filter",
|
||||
"name": "fwt",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"enum": [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7
|
||||
],
|
||||
"type": "integer",
|
||||
"description": "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)",
|
||||
"name": "provider",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"description": "Page number",
|
||||
"name": "pageNumber",
|
||||
"in": "query"
|
||||
},
|
||||
{
|
||||
"type": "integer",
|
||||
"description": "Number of elements",
|
||||
"name": "pageSize",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successfully retrieved jobs",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"description": "Creates a new job with the provided specifications",
|
||||
"consumes": [
|
||||
|
|
@ -144,6 +228,58 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"/jobs/apply/{id}": {
|
||||
"post": {
|
||||
"description": "Applies for a job with the provided id",
|
||||
"consumes": [
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"jobs"
|
||||
],
|
||||
"summary": "applies for a job",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Job ID",
|
||||
"name": "id",
|
||||
"in": "query"
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Successfully applied for job",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Invalid request data",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal server error",
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/jobs/{id}/analysis": {
|
||||
"get": {
|
||||
"description": "Retrieves the analysis result for a specific job",
|
||||
|
|
@ -247,7 +383,9 @@
|
|||
2,
|
||||
3,
|
||||
4,
|
||||
5
|
||||
5,
|
||||
6,
|
||||
7
|
||||
],
|
||||
"x-enum-varnames": [
|
||||
"LinkedIn",
|
||||
|
|
@ -255,7 +393,9 @@
|
|||
"Glassdoor",
|
||||
"Bayt",
|
||||
"TokyoDev",
|
||||
"JapanDev"
|
||||
"JapanDev",
|
||||
"Google",
|
||||
"HiringCafe"
|
||||
]
|
||||
},
|
||||
"ports.CreateJobCommand": {
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ definitions:
|
|||
- 3
|
||||
- 4
|
||||
- 5
|
||||
- 6
|
||||
- 7
|
||||
type: integer
|
||||
x-enum-varnames:
|
||||
- LinkedIn
|
||||
|
|
@ -41,6 +43,8 @@ definitions:
|
|||
- Bayt
|
||||
- TokyoDev
|
||||
- JapanDev
|
||||
- Google
|
||||
- HiringCafe
|
||||
ports.CreateJobCommand:
|
||||
properties:
|
||||
fwt:
|
||||
|
|
@ -61,6 +65,66 @@ info:
|
|||
version: "1.0"
|
||||
paths:
|
||||
/jobs:
|
||||
get:
|
||||
consumes:
|
||||
- application/json
|
||||
description: Gets all jobs with the provided specifications
|
||||
parameters:
|
||||
- description: Location filter
|
||||
in: query
|
||||
name: location
|
||||
type: string
|
||||
- description: Keywords filter (searches title and company)
|
||||
in: query
|
||||
name: keywords
|
||||
type: string
|
||||
- description: Full/Part time filter
|
||||
in: query
|
||||
name: fwt
|
||||
type: string
|
||||
- description: Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4,
|
||||
JapanDev=5, Google=6, HiringCafe=7)
|
||||
enum:
|
||||
- 0
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- 4
|
||||
- 5
|
||||
- 6
|
||||
- 7
|
||||
in: query
|
||||
name: provider
|
||||
type: integer
|
||||
- description: Page number
|
||||
in: query
|
||||
name: pageNumber
|
||||
type: integer
|
||||
- description: Number of elements
|
||||
in: query
|
||||
name: pageSize
|
||||
type: integer
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: Successfully retrieved jobs
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request data
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
"500":
|
||||
description: Internal server error
|
||||
schema:
|
||||
additionalProperties: true
|
||||
type: object
|
||||
summary: Gets all jobs
|
||||
tags:
|
||||
- jobs
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
|
|
@ -186,4 +250,38 @@ paths:
|
|||
summary: Get top matching jobs
|
||||
tags:
|
||||
- jobs
|
||||
/jobs/apply/{id}:
|
||||
post:
|
||||
consumes:
|
||||
- application/json
|
||||
description: Applies for a job with the provided id
|
||||
parameters:
|
||||
- description: Job ID
|
||||
in: query
|
||||
name: id
|
||||
type: string
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"201":
|
||||
description: Successfully applied for job
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
"400":
|
||||
description: Invalid request data
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
"500":
|
||||
description: Internal server error
|
||||
schema:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
summary: applies for a job
|
||||
tags:
|
||||
- jobs
|
||||
swagger: "2.0"
|
||||
|
|
|
|||
17
services/job-applier-ts/.env
Normal file
17
services/job-applier-ts/.env
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
DB_HOST=localhost
|
||||
DB_PORT=5432
|
||||
DB_USER=your_db_user
|
||||
DB_PASSWORD=your_db_password
|
||||
DB_NAME=linkedin_jobs
|
||||
DB_SSLMODE=disable
|
||||
|
||||
SERVER_PORT=8080
|
||||
SERVER_HOST=localhost
|
||||
|
||||
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
|
||||
OPENAI_BASE_URL=https://hubai.loe.gg/v1
|
||||
OPENAI_MODEL=gpt-4o-mini
|
||||
|
||||
CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
|
||||
OPENROUTER_API_KEY=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
|
||||
GEMINI_API_KEY=AIzaSyBkGcsPdqhIrjg-wbLOuhAYmjOCt3A_zxM
|
||||
BIN
services/job-applier-ts/CV.pdf
Normal file
BIN
services/job-applier-ts/CV.pdf
Normal file
Binary file not shown.
65
services/job-applier-ts/cv.txt
Normal file
65
services/job-applier-ts/cv.txt
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
# Ziad Elshimy
|
||||
## Frontend Developer
|
||||
|
||||
Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects.
|
||||
|
||||
`ziadshimy7@gmail.com`
|
||||
`01223381370`
|
||||
`Alexandria`
|
||||
[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6)
|
||||
[GitHub](https://github.com/ziadshimy7)
|
||||
|
||||
---
|
||||
|
||||
## Work Experience
|
||||
|
||||
### kari - Fullstack Developer
|
||||
**Jun 2022 - current**
|
||||
- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates.
|
||||
- Led the development of multiple internal projects.
|
||||
- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills.
|
||||
- Led task planning and coordinated project timelines, reducing overall project duration by 20%.
|
||||
|
||||
### Callibri - Frontend Developer
|
||||
**Jan 2022 - Mar 2022**
|
||||
- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement.
|
||||
- Collaborated closely with senior developers to manage a complex design project, increasing efficiency.
|
||||
|
||||
### EJADA - Frontend Developer
|
||||
**Mar 2021 - Dec 2021**
|
||||
- Troubleshooted the website's problems and stay up to date on technology.
|
||||
|
||||
---
|
||||
|
||||
## Education
|
||||
|
||||
### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General
|
||||
**Jan 2017 - Dec 2021**
|
||||
|
||||
---
|
||||
|
||||
## Skills
|
||||
- HTML5
|
||||
- Cascading Style Sheets (CSS)
|
||||
- SCSS
|
||||
- Tailwind css
|
||||
- JavaScript
|
||||
- TypeScript
|
||||
- React.js
|
||||
- Redux.js
|
||||
- Next.js
|
||||
- Node.js
|
||||
- SSR
|
||||
- Webpack
|
||||
- Vite
|
||||
- Go (Golang)
|
||||
- Microservices
|
||||
- docker
|
||||
- docker-compose
|
||||
|
||||
---
|
||||
|
||||
## Certifications
|
||||
- CCNA
|
||||
- The Complete JavaScript Course 2023- Udemy
|
||||
- React - The Complete Guide - Udemy
|
||||
287
services/job-applier-ts/package-lock.json
generated
Normal file
287
services/job-applier-ts/package-lock.json
generated
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
{
|
||||
"name": "job-applier",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "job-applier",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
|
||||
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
|
||||
"dotenv": "^17.2.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"../../libs/browser-automation": {
|
||||
"name": "@jobs-scraper/browser-automation",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"dotenv": "^17.2.3",
|
||||
"puppeteer-core": "^24.25.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"../../libs/rabbitmq-ts": {
|
||||
"name": "@jobs-scraper/rabbitmq-ts",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"amqplib": "^0.10.9"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/amqplib": "^0.10.8",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
},
|
||||
"node_modules/@cspotcode/source-map-support": {
|
||||
"version": "0.8.1",
|
||||
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
|
||||
"integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jridgewell/trace-mapping": "0.3.9"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/@jobs-scraper/browser-automation": {
|
||||
"resolved": "../../libs/browser-automation",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@jobs-scraper/rabbitmq-ts": {
|
||||
"resolved": "../../libs/rabbitmq-ts",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/@jridgewell/resolve-uri": {
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
|
||||
"integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@jridgewell/sourcemap-codec": {
|
||||
"version": "1.5.5",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
|
||||
"integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@jridgewell/trace-mapping": {
|
||||
"version": "0.3.9",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
|
||||
"integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jridgewell/resolve-uri": "^3.0.3",
|
||||
"@jridgewell/sourcemap-codec": "^1.4.10"
|
||||
}
|
||||
},
|
||||
"node_modules/@tsconfig/node10": {
|
||||
"version": "1.0.12",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz",
|
||||
"integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@tsconfig/node12": {
|
||||
"version": "1.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
|
||||
"integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@tsconfig/node14": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
|
||||
"integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@tsconfig/node16": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
|
||||
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "24.10.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz",
|
||||
"integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/acorn": {
|
||||
"version": "8.15.0",
|
||||
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
|
||||
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"acorn": "bin/acorn"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/acorn-walk": {
|
||||
"version": "8.3.4",
|
||||
"resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
|
||||
"integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"acorn": "^8.11.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/arg": {
|
||||
"version": "4.1.3",
|
||||
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
|
||||
"integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/create-require": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
|
||||
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/diff": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
|
||||
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">=0.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "17.2.3",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
|
||||
"integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
|
||||
"license": "BSD-2-Clause",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://dotenvx.com"
|
||||
}
|
||||
},
|
||||
"node_modules/make-error": {
|
||||
"version": "1.3.6",
|
||||
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
|
||||
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ts-node": {
|
||||
"version": "10.9.2",
|
||||
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
|
||||
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@cspotcode/source-map-support": "^0.8.0",
|
||||
"@tsconfig/node10": "^1.0.7",
|
||||
"@tsconfig/node12": "^1.0.7",
|
||||
"@tsconfig/node14": "^1.0.0",
|
||||
"@tsconfig/node16": "^1.0.2",
|
||||
"acorn": "^8.4.1",
|
||||
"acorn-walk": "^8.1.1",
|
||||
"arg": "^4.1.0",
|
||||
"create-require": "^1.1.0",
|
||||
"diff": "^4.0.1",
|
||||
"make-error": "^1.1.1",
|
||||
"v8-compile-cache-lib": "^3.0.1",
|
||||
"yn": "3.1.1"
|
||||
},
|
||||
"bin": {
|
||||
"ts-node": "dist/bin.js",
|
||||
"ts-node-cwd": "dist/bin-cwd.js",
|
||||
"ts-node-esm": "dist/bin-esm.js",
|
||||
"ts-node-script": "dist/bin-script.js",
|
||||
"ts-node-transpile-only": "dist/bin-transpile.js",
|
||||
"ts-script": "dist/bin-script-deprecated.js"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@swc/core": ">=1.2.50",
|
||||
"@swc/wasm": ">=1.2.50",
|
||||
"@types/node": "*",
|
||||
"typescript": ">=2.7"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@swc/core": {
|
||||
"optional": true
|
||||
},
|
||||
"@swc/wasm": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
|
||||
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/v8-compile-cache-lib": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
|
||||
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/yn": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
|
||||
"integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
25
services/job-applier-ts/package.json
Normal file
25
services/job-applier-ts/package.json
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
{
|
||||
"name": "job-applier",
|
||||
"version": "1.0.0",
|
||||
"description": "Job application automation service using Puppeteer",
|
||||
"main": "dist/index.js",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"dev": "ts-node src/index.ts",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": ["job", "automation", "puppeteer", "application"],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
|
||||
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
|
||||
"dotenv": "^17.2.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.8.1",
|
||||
"ts-node": "^10.9.2",
|
||||
"typescript": "^5.9.3"
|
||||
}
|
||||
}
|
||||
347
services/job-applier-ts/src/applier/work.ts
Normal file
347
services/job-applier-ts/src/applier/work.ts
Normal file
|
|
@ -0,0 +1,347 @@
|
|||
import fs from "fs";
|
||||
import path from "path";
|
||||
import {
|
||||
Browser,
|
||||
randomDelay,
|
||||
OpenRouterService,
|
||||
FormField,
|
||||
getMockFormData,
|
||||
FormExtractionResult,
|
||||
} from "@jobs-scraper/browser-automation";
|
||||
|
||||
/**
|
||||
* Job represents a job posting
|
||||
*/
|
||||
export interface Job {
|
||||
id: number;
|
||||
title: string;
|
||||
company: string;
|
||||
companyLink: string;
|
||||
location: string;
|
||||
jobLink: string;
|
||||
provider: number;
|
||||
jobPostTime: string;
|
||||
}
|
||||
|
||||
export interface JobWithDescription {
|
||||
job: Job;
|
||||
jobDescription: JobDescription;
|
||||
}
|
||||
|
||||
export interface JobDescription {
|
||||
description: string;
|
||||
criteria: Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read CV from file
|
||||
*/
|
||||
export function readCV(cvPath: string = "cv.txt"): string {
|
||||
const absolutePath = path.isAbsolute(cvPath)
|
||||
? cvPath
|
||||
: path.join(process.cwd(), cvPath);
|
||||
|
||||
return fs.readFileSync(absolutePath, "utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Read CV from file
|
||||
*/
|
||||
export function getCVPath(cvPath: string = "CV.pdf"): string {
|
||||
return path.isAbsolute(cvPath) ? cvPath : path.join(process.cwd(), cvPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle text, email, tel, url, search, password inputs
|
||||
*/
|
||||
async function handleTextInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.clear(field.selector);
|
||||
await browser.type(field.selector, "");
|
||||
await browser.type(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle textarea inputs
|
||||
*/
|
||||
async function handleTextarea(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.clear(field.selector);
|
||||
await browser.type(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle number and range inputs
|
||||
*/
|
||||
async function handleNumberInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle date, datetime-local, time, month, week inputs
|
||||
*/
|
||||
async function handleDateInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle select dropdown inputs
|
||||
*/
|
||||
async function handleSelect(browser: Browser, field: FormField): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.select(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle radio button inputs
|
||||
*/
|
||||
async function handleRadio(browser: Browser, field: FormField): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setRadio(field.selector);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle checkbox inputs
|
||||
*/
|
||||
async function handleCheckbox(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
const checked =
|
||||
field.value === "true" || field.value === "1" || field.value === "yes";
|
||||
await browser.setCheckbox(field.selector, checked);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle file input fields
|
||||
*/
|
||||
async function handleFileUpload(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.uploadFile(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle color picker inputs
|
||||
*/
|
||||
async function handleColorInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle hidden input fields
|
||||
*/
|
||||
async function handleHiddenInput(
|
||||
browser: Browser,
|
||||
field: FormField
|
||||
): Promise<void> {
|
||||
if (field.value === "") {
|
||||
return;
|
||||
}
|
||||
await browser.setValue(field.selector, field.value);
|
||||
}
|
||||
|
||||
export interface WorkOptions {
|
||||
useMockData?: boolean;
|
||||
openRouterApiKey?: string;
|
||||
openRouterModel?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main work function that fills out a job application form
|
||||
*/
|
||||
export async function work(
|
||||
browser: Browser,
|
||||
job: JobWithDescription,
|
||||
options: WorkOptions = {}
|
||||
): Promise<void> {
|
||||
const { openRouterApiKey = "", openRouterModel = "", useMockData } = options;
|
||||
|
||||
try {
|
||||
await browser.init();
|
||||
|
||||
await browser.navigate(`${job.job.jobLink}apply`);
|
||||
|
||||
await browser.evaluate<boolean>(
|
||||
`
|
||||
(() => {
|
||||
const acceptBtn = document.querySelector('[data-ui="cookie-consent-accept"]');
|
||||
if (acceptBtn) {
|
||||
acceptBtn.click();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
})()
|
||||
`
|
||||
);
|
||||
|
||||
let result: FormExtractionResult;
|
||||
|
||||
const formHtml = await browser.getOuterHTML("form");
|
||||
|
||||
// Read the CV
|
||||
const cv = readCV();
|
||||
|
||||
const cvPDFPath = getCVPath();
|
||||
|
||||
console.log({ cvPDFPath });
|
||||
|
||||
// let resumeButtonClicked: boolean = false;
|
||||
|
||||
// while (!resumeButtonClicked) {
|
||||
// resumeButtonClicked = await browser.evaluate<boolean>(
|
||||
// `
|
||||
// (() => {
|
||||
// const buttons = document.querySelectorAll('button');
|
||||
// const resumeBtn = Array.from(buttons).find(btn =>
|
||||
// btn.innerText.toLowerCase().includes('resume') ||
|
||||
// btn.innerText.toLowerCase().includes('import cv')
|
||||
// );
|
||||
// if (resumeBtn) {
|
||||
// resumeBtn.click();
|
||||
// return true;
|
||||
// }
|
||||
// return false;
|
||||
// })()
|
||||
// `
|
||||
// );
|
||||
// }
|
||||
|
||||
// console.log({ resumeButtonClicked });
|
||||
// await browser.uploadFile("input#file-upload", cvPDFPath);
|
||||
await randomDelay(5000, 7000);
|
||||
// await browser.uploadFile("input#resume-upload-input", cvPDFPath);
|
||||
|
||||
if (!useMockData) {
|
||||
const openRouterService = new OpenRouterService({
|
||||
apiKey: openRouterApiKey,
|
||||
model: openRouterModel,
|
||||
});
|
||||
console.log("Calling OpenRouter service...");
|
||||
result = await openRouterService.applyForJob(formHtml, cv, "");
|
||||
|
||||
console.log("Form extraction result:", result);
|
||||
} else {
|
||||
result = getMockFormData();
|
||||
}
|
||||
|
||||
console.log({ result });
|
||||
|
||||
if (result.fields.length === 0) {
|
||||
console.log("No fields extracted, using mock data");
|
||||
} else {
|
||||
for (const field of result.fields) {
|
||||
await randomDelay(1000, 2000);
|
||||
await processField(browser, field);
|
||||
}
|
||||
}
|
||||
|
||||
await randomDelay(160000, 180000);
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a single form field
|
||||
*/
|
||||
async function processField(browser: Browser, field: FormField): Promise<void> {
|
||||
if (field.selector === "") {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("Processing field:", field);
|
||||
|
||||
try {
|
||||
switch (field.field_type) {
|
||||
case "text":
|
||||
case "email":
|
||||
case "tel":
|
||||
case "url":
|
||||
case "search":
|
||||
case "password":
|
||||
await handleTextInput(browser, field);
|
||||
break;
|
||||
case "textarea":
|
||||
await handleTextarea(browser, field);
|
||||
break;
|
||||
case "number":
|
||||
case "range":
|
||||
await handleNumberInput(browser, field);
|
||||
break;
|
||||
case "date":
|
||||
case "datetime-local":
|
||||
case "time":
|
||||
case "month":
|
||||
case "week":
|
||||
await handleDateInput(browser, field);
|
||||
break;
|
||||
case "select":
|
||||
await handleSelect(browser, field);
|
||||
break;
|
||||
case "radio":
|
||||
await handleRadio(browser, field);
|
||||
break;
|
||||
case "checkbox":
|
||||
await handleCheckbox(browser, field);
|
||||
break;
|
||||
case "file":
|
||||
await handleFileUpload(browser, field);
|
||||
break;
|
||||
case "color":
|
||||
await handleColorInput(browser, field);
|
||||
break;
|
||||
case "hidden":
|
||||
await handleHiddenInput(browser, field);
|
||||
break;
|
||||
case "button":
|
||||
case "submit":
|
||||
case "reset":
|
||||
// Skip buttons - they're handled separately
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error processing field ${field.field_name}:`, error);
|
||||
// Continue with other fields
|
||||
}
|
||||
|
||||
await randomDelay(1500, 2000);
|
||||
}
|
||||
91
services/job-applier-ts/src/index.ts
Normal file
91
services/job-applier-ts/src/index.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import * as dotenv from "dotenv";
|
||||
import { RabbitMQClient, RabbitMQConfig } from "@jobs-scraper/rabbitmq-ts";
|
||||
import { work, JobWithDescription } from "./applier/work";
|
||||
import { Browser } from "@jobs-scraper/browser-automation";
|
||||
|
||||
if (dotenv.config({ path: ".local.env" }).error) {
|
||||
console.log("No .local.env file found, trying .env");
|
||||
if (dotenv.config().error) {
|
||||
console.log("No .env file found, using system environment variables");
|
||||
}
|
||||
}
|
||||
|
||||
const JOB_APPLIER_QUEUE = "job.applier";
|
||||
const SCRAPER_EXCHANGE = "scraper_exchange";
|
||||
|
||||
let browser = new Browser({ port: 9223 });
|
||||
let rabbitMQClient: RabbitMQClient | null = null;
|
||||
|
||||
function createRabbitMQConfig(): RabbitMQConfig {
|
||||
return {
|
||||
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
|
||||
queueName: JOB_APPLIER_QUEUE,
|
||||
exchangeName: SCRAPER_EXCHANGE,
|
||||
exchangeType: "topic",
|
||||
durable: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function processJob(job: JobWithDescription): Promise<void> {
|
||||
console.log(
|
||||
`Processing job application for: ${job.job.title} at ${job.job.company}`
|
||||
);
|
||||
|
||||
await work(browser, job, {
|
||||
useMockData: true,
|
||||
openRouterApiKey: process.env.OPENROUTER_API_KEY,
|
||||
openRouterModel: process.env.OPENROUTER_MODEL,
|
||||
});
|
||||
|
||||
console.log("Job application completed successfully!");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log("Starting Job Applier service...");
|
||||
|
||||
try {
|
||||
await browser.init();
|
||||
|
||||
const config = createRabbitMQConfig();
|
||||
rabbitMQClient = new RabbitMQClient(config);
|
||||
await rabbitMQClient.connect();
|
||||
|
||||
await rabbitMQClient.subscribe<JobWithDescription>(processJob);
|
||||
} catch (error) {
|
||||
console.error("Error starting service:", error);
|
||||
await cleanup();
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanup(): Promise<void> {
|
||||
try {
|
||||
if (rabbitMQClient) {
|
||||
await rabbitMQClient.close();
|
||||
rabbitMQClient = null;
|
||||
}
|
||||
await browser.close();
|
||||
console.log("Cleanup completed");
|
||||
} catch (error) {
|
||||
console.error("Error during cleanup:", error);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle graceful shutdown
|
||||
process.on("SIGINT", async () => {
|
||||
console.log("Received SIGINT signal");
|
||||
await cleanup();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on("SIGTERM", async () => {
|
||||
console.log("Received SIGTERM signal");
|
||||
await cleanup();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
main().catch(async (error) => {
|
||||
console.error("Fatal error:", error);
|
||||
await cleanup();
|
||||
process.exit(1);
|
||||
});
|
||||
34
services/job-applier-ts/tsconfig.json
Normal file
34
services/job-applier-ts/tsconfig.json
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2020",
|
||||
"module": "commonjs",
|
||||
"lib": ["ES2020", "DOM"],
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"removeComments": true,
|
||||
"noImplicitAny": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitThis": true,
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"exactOptionalPropertyTypes": false,
|
||||
"noImplicitOverride": true,
|
||||
"noPropertyAccessFromIndexSignature": false,
|
||||
"noUncheckedIndexedAccess": false
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
],
|
||||
"exclude": [
|
||||
"node_modules",
|
||||
"dist"
|
||||
]
|
||||
}
|
||||
|
|
@ -107,21 +107,24 @@ async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise<void> {
|
|||
});
|
||||
|
||||
await randomDelay(1500, 2500);
|
||||
await page.goto("https://www.google.com/search?q=site:boards.greenhouse.io+(Software+OR+Backend+OR+Full-Stack)+Engineer", {
|
||||
waitUntil: ["networkidle2", "domcontentloaded"],
|
||||
timeout: 60000,
|
||||
});
|
||||
// await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
|
||||
// const jobs = await browserService.extractJobListings(searchQuery.location);
|
||||
|
||||
await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
|
||||
const jobs = await browserService.extractJobListings(searchQuery.location);
|
||||
// if (jobs.length > 0) {
|
||||
// await saveJobs(jobs);
|
||||
// await saveJobDescriptions(jobs);
|
||||
|
||||
if (jobs.length > 0) {
|
||||
await saveJobs(jobs);
|
||||
await saveJobDescriptions(jobs);
|
||||
|
||||
console.log(
|
||||
`saved ${jobs.length} jobs from Glassdoor for query:`,
|
||||
searchQuery
|
||||
);
|
||||
} else {
|
||||
console.log("No jobs found for this search query");
|
||||
}
|
||||
// console.log(
|
||||
// `saved ${jobs.length} jobs from Glassdoor for query:`,
|
||||
// searchQuery
|
||||
// );
|
||||
// } else {
|
||||
// console.log("No jobs found for this search query");
|
||||
// }
|
||||
|
||||
await randomDelay(1500, 2500);
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -17,10 +17,7 @@ export const saveJobs = async (jobs: Job[]) => {
|
|||
job_link: job.jobLink,
|
||||
provider: job.provider,
|
||||
status: job.status,
|
||||
job_timestamp:
|
||||
job.jobPostTime instanceof Date && !isNaN(job.jobPostTime.getTime())
|
||||
? job.jobPostTime
|
||||
: new Date(),
|
||||
job_timestamp: job.jobPostTime,
|
||||
}));
|
||||
|
||||
const result = await sql`
|
||||
|
|
|
|||
20
services/scraper-google/.gitlab-ci.yml
Normal file
20
services/scraper-google/.gitlab-ci.yml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
deploy_scraper_google:
|
||||
stage: deploy
|
||||
image: alpine:3.20
|
||||
environment:
|
||||
name: production
|
||||
url: https://jobs-scraper.ai-assistant-bot.xyz
|
||||
rules:
|
||||
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
|
||||
when: manual
|
||||
allow_failure: false
|
||||
before_script:
|
||||
- apk add --no-cache openssh-client rsync
|
||||
- mkdir -p ~/.ssh
|
||||
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
|
||||
- chmod 600 ~/.ssh/id_rsa
|
||||
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
|
||||
script:
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
|
||||
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
|
||||
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"
|
||||
|
|
@ -1,19 +1,15 @@
|
|||
package analyzer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/pkg/openrouter"
|
||||
// "github.com/jobs-scraper/internal/pkg/openai"
|
||||
// "github.com/jobs-scraper/internal/pkg/openrouter"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/internal/openrouter"
|
||||
)
|
||||
|
||||
func GetJobDescription(htmlContent string) (*domain.JobDescription, error) {
|
||||
openRouterApiKey := os.Getenv("OPENROUTER_API_KEY")
|
||||
openRouterModel := os.Getenv("OPENROUTER_MODEL")
|
||||
fmt.Println("key", openRouterApiKey)
|
||||
|
||||
client := openrouter.NewOpenRouterService(openRouterModel, openRouterApiKey)
|
||||
result, err := client.CreateJobDescription(htmlContent)
|
||||
|
|
|
|||
|
|
@ -4,26 +4,18 @@ go 1.24.0
|
|||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
|
||||
github.com/chromedp/chromedp v0.14.2
|
||||
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
|
||||
github.com/jobs-scraper/internal/infrastructure v0.0.0
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/levmv/sked v0.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
golang.org/x/net v0.44.0 // indirect
|
||||
golang.org/x/sys v0.36.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
|
||||
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
|
||||
|
|
|
|||
|
|
@ -2,12 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
|
|||
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
|
||||
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
|
||||
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
|
||||
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
|
||||
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
|
||||
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
|
|
@ -17,23 +11,13 @@ github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjY
|
|||
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
|
||||
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/levmv/sked v0.2.1 h1:XMdLkbuajPM5LSzoBvNzalj6oYHkEfeA7/VPvkIHqs0=
|
||||
github.com/levmv/sked v0.2.1/go.mod h1:w9pFUIpZLPu0Jr2+2JqJtaEIHmwXcvjPHWm6pvbqHVQ=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
|
|
@ -42,8 +26,6 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
|
|||
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
|
|
@ -53,7 +35,6 @@ go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
|||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
|
||||
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
|
|
|||
|
|
@ -2,18 +2,14 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/domain"
|
||||
"github.com/jobs-scraper/internal/browser"
|
||||
"github.com/jobs-scraper/internal/domain"
|
||||
"github.com/jobs-scraper/libs/repo"
|
||||
"github.com/jobs-scraper/services/scraper-google/analyzer"
|
||||
si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
|
||||
|
|
@ -40,31 +36,33 @@ func main() {
|
|||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
// Launch Chrome once and share across all goroutines
|
||||
wsURL, chromeCmd, err := launchChrome()
|
||||
browser, err := browser.NewBrowser(ctx)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to launch Chrome: %v", err)
|
||||
}
|
||||
defer chromeCmd.Process.Kill()
|
||||
log.Printf("Chrome launched with WebSocket URL: %s", wsURL)
|
||||
|
||||
defer browser.Close()
|
||||
|
||||
log.Printf("Chrome launched with WebSocket URL: %s", browser.Instance.WSURL)
|
||||
|
||||
sched := sked.New(ctx)
|
||||
googleLinkStream := make(chan domain.GoogleLink, 100)
|
||||
// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
|
||||
pages := []int{100, 110, 120, 130}
|
||||
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
|
||||
|
||||
log.Println("Starting scraper with scheduler...")
|
||||
|
||||
for _, page := range pages {
|
||||
go func() {
|
||||
log.Printf("Running initial scrape for page offset: %d, inital run", page)
|
||||
Work(ctx, wsURL)(page, googleLinkStream)
|
||||
Work(ctx, browser)(page, googleLinkStream)
|
||||
}()
|
||||
}
|
||||
|
||||
for _, page := range pages {
|
||||
sched.Schedule(func(ctx context.Context) {
|
||||
log.Printf("Running scheduled scrape for page offset: %d", page)
|
||||
Work(ctx, wsURL)(page, googleLinkStream)
|
||||
Work(ctx, browser)(page, googleLinkStream)
|
||||
}).Every(time.Hour * 24)
|
||||
}
|
||||
|
||||
|
|
@ -84,7 +82,7 @@ func main() {
|
|||
continue
|
||||
}
|
||||
|
||||
htmlRaw, url, err := utils.GetHTMLRaw(ctx, wsURL, googleLink)
|
||||
htmlRaw, url, err := utils.GetHTMLRaw(ctx, browser, googleLink)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
|
||||
|
|
@ -143,75 +141,3 @@ func main() {
|
|||
cancel()
|
||||
log.Println("Shutting down scraper...")
|
||||
}
|
||||
|
||||
// launchChrome starts Chrome with remote debugging and returns the WebSocket URL
|
||||
func launchChrome() (string, *exec.Cmd, error) {
|
||||
var chromePath string
|
||||
switch runtime.GOOS {
|
||||
case "darwin":
|
||||
chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
||||
case "linux":
|
||||
chromePath = "google-chrome"
|
||||
case "windows":
|
||||
chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
|
||||
default:
|
||||
return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
|
||||
}
|
||||
|
||||
// Kill any existing Chrome debug instances first
|
||||
exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
cmd := exec.Command(chromePath,
|
||||
"--remote-debugging-port=9222",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--disable-background-networking",
|
||||
"--disable-extensions",
|
||||
"--disable-sync",
|
||||
"--disable-translate",
|
||||
"--metrics-recording-only",
|
||||
"--safebrowsing-disable-auto-update",
|
||||
"--user-data-dir=/tmp/chrome-debug-profile",
|
||||
// "--incognito",
|
||||
// "--headless",
|
||||
)
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
|
||||
}
|
||||
|
||||
// Poll the debug endpoint to get the WebSocket URL
|
||||
var wsURL string
|
||||
for i := 0; i < 50; i++ {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
resp, err := http.Get("http://127.0.0.1:9222/json/version")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var result struct {
|
||||
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
resp.Body.Close()
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if result.WebSocketDebuggerURL != "" {
|
||||
wsURL = result.WebSocketDebuggerURL
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if wsURL == "" {
|
||||
cmd.Process.Kill()
|
||||
return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
|
||||
}
|
||||
|
||||
// Wait for Chrome to be fully ready to accept connections
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
return wsURL, cmd, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import (
|
|||
"database/sql"
|
||||
"log"
|
||||
|
||||
"github.com/jobs-scraper/internal/pkg/infrastructure"
|
||||
"github.com/jobs-scraper/internal/infrastructure"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,28 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func BuildUrl(query string, page int) string {
|
||||
var url strings.Builder
|
||||
url.WriteString("https://www.google.com/search?q=")
|
||||
url.WriteString(query)
|
||||
url.WriteString("&start=")
|
||||
url.WriteString(strconv.Itoa(page))
|
||||
return url.String()
|
||||
params := url.Values{}
|
||||
params.Add("q", query)
|
||||
params.Add("start", strconv.Itoa(page))
|
||||
|
||||
now := time.Now()
|
||||
twoWeeksAgo := now.AddDate(0, 0, -14)
|
||||
|
||||
startDate := twoWeeksAgo.Format("01/02/2006")
|
||||
endDate := now.Format("01/02/2006")
|
||||
|
||||
tbs := "cdr:1,cd_min:" + startDate + ",cd_max:" + endDate
|
||||
params.Add("tbs", tbs)
|
||||
|
||||
// Optional language bias
|
||||
// params.Add("hl", "en")
|
||||
// params.Add("lr", "lang_en")
|
||||
|
||||
return "https://www.google.com/search?" + params.Encode()
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue