Compare commits

...

10 commits

Author SHA1 Message Date
Elshimy Ziad Magdy Taha
af52085ea3 fix errors and restructure project 2026-01-26 17:33:48 +05:00
Elshimy Ziad Magdy Taha
a281363b61 hiring cafe scraper 2025-12-29 12:19:33 +05:00
Elshimy Ziad Magdy Taha
8a4ad07e46 fix swagger 2025-12-18 14:02:13 +05:00
Elshimy Ziad Magdy Taha
13d10e9d11 get jobs method with filters 2025-12-18 12:25:07 +05:00
Elshimy Ziad Magdy Taha
cbcf9bdcc7 refactor 2025-12-17 17:42:42 +05:00
Заид Омар Медхат | Zaid Omar Medhat
18842091a4 fix: add migrations to docker 2025-12-16 18:31:54 +05:00
Заид Омар Медхат | Zaid Omar Medhat
9ef1833073 f 2025-12-16 18:21:40 +05:00
Elshimy Ziad Magdy Taha
bb55154a19 fix ip host rabbitmq 2025-12-16 18:10:05 +05:00
Elshimy Ziad Magdy Taha
2a137d3b61 remove version 2025-12-16 18:07:50 +05:00
Elshimy Ziad Magdy Taha
3187757776 docker for api 2025-12-16 18:05:47 +05:00
124 changed files with 7391 additions and 965 deletions

View file

@ -1,23 +1,6 @@
stages:
- deploy
deploy_production:
stage: deploy
image: alpine:3.20
environment:
name: production
url: https://jobs-scraper.ai-assistant-bot.xyz
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
when: manual
allow_failure: false
before_script:
- apk add --no-cache openssh-client rsync
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
script:
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"
include:
- local: 'services/scraper-google/.gitlab-ci.yml'
- local: 'services/api/.gitlab-ci.yml'

View file

@ -6,6 +6,7 @@ API_DIR=./services/api
SCRAPER_LINKEDIN_DIR=./services/scraper-linkedin
SCRAPER_GLASSDOOR_DIR=./services/scraper-glassdoor
SCRAPER_PLAYWRIGHT_DIR=./services/scraper-playwright
JOB_APPLIER_BROWSER_USE_DIR=./services/job-applier-browser-use
CRON_ANALYZER_DIR=./apps/cron-analyzer
CV_ANALYZER_DIR=./apps/cv-analyzer
DOCS_DIR=./services/api/pkg/swagger
@ -71,6 +72,18 @@ docker-scraper-playwright: ## Build and run Playwright scraper in Docker
@echo "Running Playwright scraper in Docker..."
@docker run --rm --name playwright-scraper playwright-scraper
.PHONY: setup-job-applier-browser-use
setup-job-applier-browser-use: ## Setup the browser-use job applier service
@echo "Setting up browser-use job applier service..."
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv venv && uv pip install -e .
@echo "Installing Chromium browser..."
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uvx browser-use install
.PHONY: run-job-applier-browser-use
run-job-applier-browser-use: ## Run the browser-use job applier service
@echo "Running browser-use job applier service..."
@cd $(JOB_APPLIER_BROWSER_USE_DIR) && uv run python -m src.main
# Apps
.PHONY: run-cron-analyzer
run-cron-analyzer: ## Run the job analysis cron service

View file

@ -4,7 +4,7 @@ import (
"log"
"github.com/jobs-scraper/apps/cron-analyzer/internal"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/infrastructure"
"github.com/joho/godotenv"
)

View file

@ -3,9 +3,9 @@ module github.com/jobs-scraper/apps/cron-analyzer
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/pkg/openrouter v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/domain v0.0.0
github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/openrouter v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/libs/repo v0.0.0
github.com/joho/godotenv v1.5.1
)
@ -20,10 +20,10 @@ require (
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
)
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
replace github.com/jobs-scraper/internal/pkg/openrouter => ../../internal/pkg/openrouter
replace github.com/jobs-scraper/internal/openrouter => ../../internal/openrouter
replace github.com/jobs-scraper/libs/repo => ../../libs/repo

View file

@ -8,10 +8,10 @@ import (
// "path/filepath"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/pkg/openai"
"github.com/jobs-scraper/internal/pkg/openrouter"
"github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/openai"
"github.com/jobs-scraper/internal/openrouter"
"github.com/jobs-scraper/libs/repo"
)

14
go.mod
View file

@ -3,3 +3,17 @@ module github.com/jobs-scraper
go 1.24.0
toolchain go1.24.7
require (
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.2
)
require (
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
golang.org/x/sys v0.36.0 // indirect
)

10
go.sum
View file

@ -0,0 +1,10 @@
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=

12
go.work
View file

@ -3,12 +3,12 @@ go 1.24.0
use (
.
./apps/cron-analyzer
./internal/pkg/domain
./internal/pkg/gemini
./internal/pkg/infrastructure
./internal/pkg/openai
./internal/pkg/openrouter
./internal/pkg/utils
./internal/domain
./internal/gemini
./internal/infrastructure
./internal/openai
./internal/openrouter
./internal/utils
./libs/ports
./libs/repo
./libs/server

View file

@ -43,6 +43,9 @@ github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8=
github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM=
@ -57,9 +60,13 @@ github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9
github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw=
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
@ -102,8 +109,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
@ -121,6 +128,7 @@ github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86w
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
@ -156,7 +164,7 @@ golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@ -169,7 +177,6 @@ golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg=
google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw=
@ -183,7 +190,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.
google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=

403
internal/browser/browser.go Normal file
View file

@ -0,0 +1,403 @@
package browser
import (
"context"
"log"
"math/rand"
"os/exec"
// "strings"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/dom"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
)
type ChromeInstance struct {
WSURL string
Cmd *exec.Cmd
}
type Browser struct {
Instance *ChromeInstance
allocCtx context.Context
allocCancel context.CancelFunc
ctx context.Context
cancel context.CancelFunc
}
// NewBrowser launches Chrome and creates a shared allocator.
// Use NewTab() to create separate tabs for concurrent use.
func NewBrowser(ctx context.Context) (*Browser, error) {
wsURL, cmd, err := launchChrome()
if err != nil {
return nil, err
}
allocCtx, allocCancel := chromedp.NewRemoteAllocator(ctx, wsURL)
// Create initial browser context (first tab)
browserCtx, cancel := chromedp.NewContext(allocCtx)
return &Browser{
Instance: &ChromeInstance{
WSURL: wsURL,
Cmd: cmd,
},
allocCtx: allocCtx,
allocCancel: allocCancel,
ctx: browserCtx,
cancel: cancel,
}, nil
}
// NewTab creates a new browser tab that can be used concurrently.
// Returns a new Browser instance sharing the same Chrome process.
func (b *Browser) NewTab() (*Browser, error) {
tabCtx, cancel := chromedp.NewContext(b.allocCtx)
return &Browser{
Instance: b.Instance,
allocCtx: b.allocCtx,
allocCancel: nil, // Don't close allocator when closing a tab
ctx: tabCtx,
cancel: cancel,
}, nil
}
func (b *Browser) Close() error {
if b.cancel != nil {
b.cancel()
}
if b.allocCancel != nil {
b.allocCancel()
}
// Only kill Chrome process if this is the main browser (has allocCancel),
// not a tab created via NewTab()
if b.allocCancel != nil && b.Instance != nil && b.Instance.Cmd != nil && b.Instance.Cmd.Process != nil {
return b.Instance.Cmd.Process.Kill()
}
return nil
}
func (b *Browser) Navigate(url string) error {
chromedp.UserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36")
err := chromedp.Run(b.ctx,
chromedp.Navigate(url),
)
if err != nil {
return err
}
return nil
}
func (b *Browser) WaitVisible(selector string) error {
ctx, cancel := context.WithTimeout(b.ctx, 30*time.Second)
defer cancel()
err := chromedp.Run(ctx,
chromedp.WaitVisible(selector, chromedp.ByQuery),
)
if err != nil {
return err
}
return nil
}
func (b *Browser) GetNodes(selector string) ([]*cdp.Node, error) {
var nodes []*cdp.Node
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector, chromedp.ByQueryAll),
chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll),
)
if err != nil {
return nodes, err
}
return nodes, nil
}
// GetOuterHTML gets the outer HTML of the first element matching the selector
func (b *Browser) GetOuterHTML(selector string) (string, error) {
var html string
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector, chromedp.ByQuery),
chromedp.OuterHTML(selector, &html, chromedp.ByQuery),
)
if err != nil {
return "", err
}
return html, nil
}
func (b *Browser) GetTextFromNode(node *cdp.Node, selector string) (string, error) {
var text string
err := chromedp.Run(b.ctx,
chromedp.Text(selector, &text, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil {
return "", err
}
return text, nil
}
func (b *Browser) GetAttributeFromNode(node *cdp.Node, selector string, attr string) (string, error) {
var value string
err := chromedp.Run(b.ctx,
chromedp.AttributeValue(selector, attr, &value, nil, chromedp.ByQuery, chromedp.FromNode(node)),
)
if err != nil {
return "", err
}
// value = strings.ReplaceAll(value, "/apply", "")
return value, nil
}
func Evaluate[T any](b *Browser, expression string) (T, error) {
var result T
err := chromedp.Run(b.ctx,
chromedp.Evaluate(expression, &result),
)
if err != nil {
return result, err
}
return result, nil
}
func (b *Browser) GetCurrentLocation() (string, error) {
var finalURL string
err := chromedp.Run(b.ctx,
chromedp.Location(&finalURL),
)
if err != nil {
log.Printf("Failed to get current location: %v", err)
return "", err
}
return finalURL, nil
}
func (b *Browser) GetHTML() (string, error) {
var htmlContent string
err := chromedp.Run(b.ctx,
chromedp.OuterHTML("html", &htmlContent, chromedp.ByQuery),
)
if err != nil {
log.Printf("Failed to get HTML content: %v", err)
return "", err
}
return htmlContent, nil
}
func (b *Browser) WaitForNetworkIdle(timeout time.Duration) error {
err := chromedp.Run(b.ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
return page.SetLifecycleEventsEnabled(true).Do(ctx)
}),
)
if err != nil {
return err
}
ch := make(chan struct{}, 1)
chromedp.ListenTarget(b.ctx, func(ev interface{}) {
if e, ok := ev.(*page.EventLifecycleEvent); ok {
if e.Name == "networkIdle" {
select {
case ch <- struct{}{}:
default:
}
}
}
})
select {
case <-ch:
return nil
case <-time.After(timeout):
// Timeout reached, but don't error - page may still be usable
return nil
case <-b.ctx.Done():
return b.ctx.Err()
}
}
func (b *Browser) NodeToString(node *cdp.Node) (string, error) {
var outerHTML string
err := chromedp.Run(b.ctx,
chromedp.ActionFunc(func(ctx context.Context) error {
h, err := dom.GetOuterHTML().WithNodeID(node.NodeID).Do(ctx)
if err != nil {
return err
}
outerHTML = h
return nil
}),
)
if err != nil {
return "", err
}
return outerHTML, err
}
func (b *Browser) Type(selector string, value string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.SendKeys(selector, value),
)
if err != nil {
return err
}
return nil
}
func (b *Browser) Click(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Click(selector),
)
if err != nil {
return err
}
return nil
}
// Select chooses an option from a <select> element by value
func (b *Browser) Select(selector string, value string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.SetValue(selector, value),
)
if err != nil {
return err
}
return nil
}
// SetCheckbox sets a checkbox to checked or unchecked state
func (b *Browser) SetCheckbox(selector string, checked bool) error {
var isChecked bool
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Evaluate(`document.querySelector('`+selector+`').checked`, &isChecked),
)
if err != nil {
return err
}
if isChecked != checked {
err = chromedp.Run(b.ctx,
chromedp.Click(selector),
)
if err != nil {
return err
}
}
return nil
}
// SetRadio clicks a radio button to select it
func (b *Browser) SetRadio(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Click(selector),
)
if err != nil {
return err
}
return nil
}
// UploadFile sets a file input to the specified file path
func (b *Browser) UploadFile(selector string, filePath string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.SetUploadFiles(selector, []string{filePath}),
)
if err != nil {
return err
}
return nil
}
// SetValue sets the value of an input field directly (useful for hidden, date, color, etc.)
func (b *Browser) SetValue(selector string, value string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitReady(selector),
chromedp.SetValue(selector, value),
)
if err != nil {
return err
}
return nil
}
// Clear clears the value of an input field
func (b *Browser) Clear(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Clear(selector),
)
if err != nil {
return err
}
return nil
}
// Focus focuses on an element
func (b *Browser) Focus(selector string) error {
err := chromedp.Run(b.ctx,
chromedp.WaitVisible(selector),
chromedp.Focus(selector),
)
if err != nil {
return err
}
return nil
}
// RandomDelay waits for a random duration between min and max milliseconds
func RandomDelay(minMs, maxMs int) {
delay := time.Duration(minMs+rand.Intn(maxMs-minMs)) * time.Millisecond
time.Sleep(delay)
}

View file

@ -0,0 +1,103 @@
package browser
import (
"encoding/json"
"fmt"
"net/http"
"os/exec"
"runtime"
"time"
)
func launchChrome() (string, *exec.Cmd, error) {
var chromePath string
switch runtime.GOOS {
case "darwin":
chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
case "linux":
chromePath = "google-chrome"
case "windows":
chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
default:
return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
}
// Kill any existing Chrome debug instances first
exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
time.Sleep(500 * time.Millisecond)
cmd := exec.Command(chromePath,
"--remote-debugging-port=9222",
"--no-first-run",
"--no-default-browser-check",
"--user-data-dir=/tmp/chrome-debug-profile",
// Anti-detection flags
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--disable-dev-shm-usage",
"--disable-browser-side-navigation",
"--disable-gpu",
"--no-sandbox",
"--disable-setuid-sandbox",
// Hide automation indicators
"--disable-extensions",
"--disable-plugins-discovery",
"--disable-bundled-ppapi-flash",
// Reduce fingerprinting
"--disable-background-networking",
"--disable-sync",
"--disable-translate",
"--metrics-recording-only",
"--safebrowsing-disable-auto-update",
"--disable-client-side-phishing-detection",
"--disable-default-apps",
"--disable-hang-monitor",
"--disable-popup-blocking",
"--disable-prompt-on-repost",
// Window size to look like a real browser
"--window-size=1920,1080",
"--start-maximized",
)
if err := cmd.Start(); err != nil {
return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
}
// Poll the debug endpoint to get the WebSocket URL
var wsURL string
for range 50 {
time.Sleep(200 * time.Millisecond)
resp, err := http.Get("http://127.0.0.1:9222/json/version")
if err != nil {
continue
}
var result struct {
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
resp.Body.Close()
continue
}
resp.Body.Close()
if result.WebSocketDebuggerURL != "" {
wsURL = result.WebSocketDebuggerURL
break
}
}
if wsURL == "" {
cmd.Process.Kill()
return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
}
// Wait for Chrome to be fully ready to accept connections
time.Sleep(1 * time.Second)
return wsURL, cmd, nil
}

5
internal/domain/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/internal/domain
go 1.24.0
toolchain go1.24.7

View file

@ -1,6 +1,8 @@
package domain
import "time"
import (
"time"
)
// JobProvider represents different job board providers
type JobProvider int
@ -15,6 +17,7 @@ const (
TokyoDev
JapanDev
Google
HiringCafe
)
const (

53
internal/dto/jobs.go Normal file
View file

@ -0,0 +1,53 @@
package dto
import "github.com/jobs-scraper/internal/domain"
type JobDTO struct {
ID int64 `json:"id"`
Title string `json:"title"`
Company string `json:"company"`
CompanyLink string `json:"companyLink"`
Location string `json:"location"`
JobLink string `json:"jobLink"`
JobPostTime string `json:"jobPostTime,omitempty"`
Provider domain.JobProvider `json:"provider"`
}
func JobFromDomain(j domain.Job) JobDTO {
var postTime string
if j.JobPostTime != nil {
postTime = j.JobPostTime.Format("2006-01-02")
}
return JobDTO{
ID: j.ID,
Title: j.Title,
Company: j.Company,
CompanyLink: j.CompanyLink,
Location: j.Location,
JobLink: j.JobLink,
JobPostTime: postTime,
Provider: j.Provider,
}
}
func JobsFromDomain(jobs []domain.Job) []JobDTO {
dtos := make([]JobDTO, len(jobs))
for i, j := range jobs {
dtos[i] = JobFromDomain(j)
}
return dtos
}
type JobWithDescriptionDTO struct {
JobDTO
Description string `json:"description"`
Criteria map[string]string `json:"criteria,omitempty"`
}
func JobWithDescriptionFromDomain(j domain.JobWithDescription) JobWithDescriptionDTO {
return JobWithDescriptionDTO{
JobDTO: JobFromDomain(j.Job),
Description: j.JobDescription.Description,
Criteria: j.JobDescription.Criteria,
}
}

View file

@ -7,7 +7,7 @@ import (
"log"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/domain"
"google.golang.org/genai"
)

View file

@ -1,4 +1,4 @@
module github.com/jobs-scraper/internal/pkg/gemini
module github.com/jobs-scraper/internal/gemini
go 1.24.0

View file

@ -152,22 +152,32 @@ func GetMigrationVersion(db *sql.DB) (uint, bool, error) {
}
func getMigrationsPath() (string, error) {
// Get the directory of the current source file
_, filename, _, ok := runtime.Caller(0)
if !ok {
return "", fmt.Errorf("failed to get current file path")
if envPath := os.Getenv("MIGRATIONS_PATH"); envPath != "" {
if _, err := os.Stat(envPath); err == nil {
return envPath, nil
}
}
// Get the internal directory (go up from internal/pkg/infrastructure/ to internal/)
// Current path: .../internal/pkg/infrastructure/db.go
// Need to go up: infrastructure -> pkg -> internal
paths := []string{
"./internal/migrations",
"../internal/migrations",
"../../internal/migrations",
}
for _, p := range paths {
if _, err := os.Stat(p); err == nil {
return p, nil
}
}
_, filename, _, ok := runtime.Caller(0)
if ok {
internalDir := filepath.Dir(filepath.Dir(filepath.Dir(filename)))
migrationsPath := filepath.Join(internalDir, "migrations")
// Verify the migrations directory exists
if _, err := os.Stat(migrationsPath); err != nil {
return "", fmt.Errorf("migrations directory not found at %s: %w", migrationsPath, err)
if _, err := os.Stat(migrationsPath); err == nil {
return migrationsPath, nil
}
}
return migrationsPath, nil
return "", fmt.Errorf("migrations directory not found")
}

View file

@ -1,4 +1,4 @@
module github.com/jobs-scraper/internal/pkg/infrastructure
module github.com/jobs-scraper/internal/infrastructure
go 1.24.0
@ -9,16 +9,16 @@ require (
github.com/gorilla/mux v1.8.1
github.com/jobs-scraper/libs/ports v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0
github.com/jobs-scraper/internal/pkg/utils v0.0.0
github.com/jobs-scraper/internal/utils v0.0.0
github.com/lib/pq v1.10.9
github.com/rabbitmq/amqp091-go v1.10.0
)
replace github.com/jobs-scraper/libs/repo => ../../../libs/repo
replace github.com/jobs-scraper/libs/repo => ../../libs/repo
replace github.com/jobs-scraper/internal/pkg/utils => ../utils
replace github.com/jobs-scraper/internal/utils => ../utils
replace github.com/jobs-scraper/libs/ports => ../../../libs/ports
replace github.com/jobs-scraper/libs/ports => ../../libs/ports
require (
github.com/hashicorp/errwrap v1.1.0 // indirect

View file

@ -9,7 +9,7 @@ import (
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/internal/pkg/utils"
"github.com/jobs-scraper/internal/utils"
"github.com/gorilla/mux"
)

View file

@ -21,6 +21,7 @@ const (
DeadLetterExchange = "scraper_dlx"
CvAnalyzeExchange = "cv_exchange"
CvAnalyzeQueue = "cv.analyze"
JobApplierQueue = "job.applier"
)
type RabbitMQClient struct {
@ -31,10 +32,25 @@ type RabbitMQClient struct {
type MessageHandler func([]byte) error
func NewRabbitMQClient() (*RabbitMQClient, error) {
// Get RabbitMQ URL from environment or use default
rabbitmqURL := os.Getenv("RABBITMQ_URL")
if rabbitmqURL == "" {
rabbitmqURL = "amqp://guest:guest@localhost:5672/"
host := os.Getenv("RABBITMQ_HOST")
if host == "" {
host = "localhost"
}
port := os.Getenv("RABBITMQ_PORT")
if port == "" {
port = "5672"
}
user := os.Getenv("RABBITMQ_USER")
if user == "" {
user = "guest"
}
password := os.Getenv("RABBITMQ_PASSWORD")
if password == "" {
password = "guest"
}
rabbitmqURL = fmt.Sprintf("amqp://%s:%s@%s:%s/", user, password, host, port)
}
// Connect to RabbitMQ with retry logic
@ -114,6 +130,7 @@ func (r *RabbitMQClient) setupInfrastructure() error {
JapanDevQueue: "scraper.japandev",
GlassDoorQueue: "scraper.glassdoor",
GoogleQueue: "scraper.google",
JobApplierQueue: "job.applier",
}
// Declare queues with dead letter exchange and TTL

View file

@ -0,0 +1 @@
DROP INDEX IF EXISTS idx_jobs_provider_location;

View file

@ -0,0 +1 @@
CREATE INDEX idx_jobs_provider_location ON jobs(provider, location);

View file

@ -36,8 +36,8 @@ import (
"fmt"
"os"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/openai"
"github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/openai"
)
func main() {

10
internal/openai/go.mod Normal file
View file

@ -0,0 +1,10 @@
module github.com/jobs-scraper/internal/openai
go 1.24.0
require (
github.com/jobs-scraper/internal/domain v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.41.2
)
replace github.com/jobs-scraper/internal/domain => ../domain

View file

@ -1,6 +1,6 @@
package openai
import "github.com/jobs-scraper/internal/pkg/domain"
import "github.com/jobs-scraper/internal/domain"
// JobAnalysisService defines the interface for job analysis services
type JobAnalysisService interface {

View file

@ -8,7 +8,7 @@ import (
"os"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/domain"
"github.com/sashabaranov/go-openai"
)

View file

@ -0,0 +1,12 @@
module github.com/jobs-scraper/internal/openrouter
go 1.24.0
require (
github.com/eduardolat/openroutergo v0.1.0
github.com/jobs-scraper/internal/domain v0.0.0
)
require github.com/orsinium-labs/enum v1.4.0 // indirect
replace github.com/jobs-scraper/internal/domain => ../domain

View file

@ -7,9 +7,56 @@ import (
"strings"
"github.com/eduardolat/openroutergo"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/domain"
)
// FieldType represents HTML input field types
type FieldType string
const (
FieldTypeText FieldType = "text"
FieldTypeEmail FieldType = "email"
FieldTypeTel FieldType = "tel"
FieldTypePassword FieldType = "password"
FieldTypeNumber FieldType = "number"
FieldTypeDate FieldType = "date"
FieldTypeDatetimeLocal FieldType = "datetime-local"
FieldTypeTime FieldType = "time"
FieldTypeMonth FieldType = "month"
FieldTypeWeek FieldType = "week"
FieldTypeURL FieldType = "url"
FieldTypeSearch FieldType = "search"
FieldTypeColor FieldType = "color"
FieldTypeRange FieldType = "range"
FieldTypeFile FieldType = "file"
FieldTypeHidden FieldType = "hidden"
FieldTypeCheckbox FieldType = "checkbox"
FieldTypeRadio FieldType = "radio"
FieldTypeSelect FieldType = "select"
FieldTypeTextarea FieldType = "textarea"
FieldTypeButton FieldType = "button"
FieldTypeSubmit FieldType = "submit"
FieldTypeReset FieldType = "reset"
)
// FormField represents a single form field extracted from HTML
type FormField struct {
Label string `json:"label"`
FieldName string `json:"field_name"`
FieldType FieldType `json:"field_type"`
Value string `json:"value"`
Placeholder string `json:"placeholder,omitempty"`
Required bool `json:"required"`
Selector string `json:"selector"`
}
// FormExtractionResult represents the structured response from form extraction
type FormExtractionResult struct {
Fields []FormField `json:"fields"`
ApplyButton string `json:"apply_button"`
AdjustedCV string `json:"adjusted_cv"`
}
// JobAnalysisResult represents the structured response from job analysis
type JobAnalysisResult struct {
Recommendation string `json:"recommendation"`
@ -199,3 +246,117 @@ Extract from this HTML Content:
return jobDescription, nil
}
// ExtractFormFields extracts form fields from HTML and populates them based on CV data.
// It also adjusts the CV based on job type (frontend/backend/fullstack).
func (s *OpenRouterService) CreateCV(htmlForm string, cvMarkdown string, jobDescription string) (*FormExtractionResult, error) {
client, err := openroutergo.
NewClient().
WithAPIKey(s.apiKey).
Create()
if err != nil {
return nil, fmt.Errorf("failed to create client: %v", err)
}
prompt := fmt.Sprintf(`You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
1. Extract ALL form fields and populate them with appropriate values from the CV
2. Adjust the CV based on the job type (frontend/backend/fullstack)
HTML FORM:
%s
JOB DESCRIPTION:
%s
ORIGINAL CV (Markdown):
%s
CV ADJUSTMENT RULES:
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
- If FULLSTACK: Keep both frontend and backend content.
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
- Maintain the same markdown structure and formatting as the original CV.
- Do NOT invent new skills or experience - only filter existing content.
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Extract ALL input fields, textareas, selects, and buttons from the form
3. For each field, determine the appropriate value from the ADJUSTED CV
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
5. Identify the apply/submit button text
6. Use the exact JSON schema below
JSON Schema:
{
"fields": [
{
"label": "Field label or name attribute",
"field_name": "name or id attribute of the input",
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
"value": "Value to populate based on CV data",
"placeholder": "Placeholder text if any",
"required": true|false,
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
}
],
"apply_button": "Unique CSS selector for the form's submit button",
"adjusted_cv": "The full adjusted CV in markdown format, tailored to the job type"
}
FIELD MAPPING GUIDELINES:
- Name fields: Extract full name, first name, last name from CV
- Email: Use email from CV contact info
- Phone: Use phone number from CV
- LinkedIn/Portfolio/Website: Use URLs from CV
- Experience/Years: Calculate from CV work history
- Current company/title: Use most recent from CV
- Skills: List relevant skills from ADJUSTED CV
- Education: Use education details from CV
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
- Salary expectations: Leave empty unless specified in CV
- Location/Address: Use from CV contact info
- Resume/CV upload: Leave value empty (file upload)
Start your response with { and end with }`, htmlForm, jobDescription, cvMarkdown)
_, resp, err := client.
NewChatCompletion().
WithModel(s.model).
WithSystemMessage("You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.").
WithUserMessage(prompt).
Execute()
if err != nil {
return nil, fmt.Errorf("failed to execute completion: %v", err)
}
if len(resp.Choices) == 0 {
return nil, fmt.Errorf("no response choices received from API")
}
jsonContent := resp.Choices[0].Message.Content
log.Printf("Raw API response for form extraction: %s", jsonContent)
if len(jsonContent) > 0 && jsonContent[0] == '<' {
return nil, fmt.Errorf("API returned HTML/XML instead of JSON. Response: %s", jsonContent[:min(200, len(jsonContent))])
}
jsonContent = cleanMarkdownCodeBlocks(jsonContent)
var result FormExtractionResult
if err := json.Unmarshal([]byte(jsonContent), &result); err != nil {
if syntaxErr, ok := err.(*json.SyntaxError); ok {
start := max(int(syntaxErr.Offset)-50, 0)
end := int(syntaxErr.Offset) + 50
if end > len(jsonContent) {
end = len(jsonContent)
}
log.Printf("JSON parse error at position %d, context: ...%s...", syntaxErr.Offset, jsonContent[start:end])
}
return nil, fmt.Errorf("failed to parse JSON response: %v. Raw response: %s", err, jsonContent[:min(500, len(jsonContent))])
}
return &result, nil
}

View file

@ -1,5 +0,0 @@
module github.com/jobs-scraper/internal/pkg/domain
go 1.24.0
toolchain go1.24.7

View file

@ -1,10 +0,0 @@
module github.com/jobs-scraper/internal/pkg/openai
go 1.24.0
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.41.2
)
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

View file

@ -1,12 +0,0 @@
module github.com/jobs-scraper/internal/pkg/openrouter
go 1.24.0
require (
github.com/eduardolat/openroutergo v0.1.0
github.com/jobs-scraper/internal/pkg/domain v0.0.0
)
require github.com/orsinium-labs/enum v1.4.0 // indirect
replace github.com/jobs-scraper/internal/pkg/domain => ../domain

View file

@ -1,5 +0,0 @@
module github.com/jobs-scraper/internal/pkg/utils
go 1.24.0
toolchain go1.24.7

View file

@ -1,425 +0,0 @@
package utils
import (
"context"
// "crypto/tls"
"fmt"
"io"
"net/http"
// "net/url"
"time"
)
type RetryConfig struct {
MaxRetries int
BaseDelay time.Duration
MaxDelay time.Duration
}
type RetryableHTTPRequestImpl struct {
client *http.Client
config RetryConfig
}
func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl {
return NewRetryableHTTPRequestWithProxy(config, "")
}
func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl {
// var transport *http.Transport
// if proxyURL != "" {
// proxyUrl, err := url.Parse(proxyURL)
// if err != nil {
// panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err))
// }
// transport = &http.Transport{
// Proxy: http.ProxyURL(proxyUrl),
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// }
// }
client := &http.Client{
Timeout: 30 * time.Second,
// Transport: transport,
}
return &RetryableHTTPRequestImpl{
client: client,
config: config,
}
}
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
var lastErr error
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
for _, header := range headers {
req.Header.Set(header.Get("Key"), header.Get("Value"))
}
resp, err := s.client.Do(req)
if err != nil {
lastErr = err
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
// Success
return resp, nil
} else if resp.StatusCode == http.StatusTooManyRequests {
// 429 Too Many Requests - retry with backoff
resp.Body.Close()
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
} else {
// All other errors (4xx, 5xx) - don't retry
resp.Body.Close()
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
}
// max attempts for now = 3
if attempt < s.config.MaxRetries {
// Increment delay by 2 seconds for each attempt
delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ...
// Cap the delay to MaxDelay if set
if s.config.MaxDelay > 0 && delay > s.config.MaxDelay {
delay = s.config.MaxDelay
}
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries)
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
// Continue to next attempt
}
}
}
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
}
// 5.10.245.81:80
// 172.67.70.206:80
// 190.93.245.33:80
// 185.221.160.176:80
// 141.101.123.156:80
// 103.21.244.129:80
// 103.169.142.59:80
// 172.67.81.185:80
// 37.18.73.60:5566
// 209.38.83.56:1088
// 89.169.36.109:1080
// 103.21.244.248:80
// 172.67.70.92:80
// 172.67.88.34:80
// 141.101.120.38:80
// 172.67.75.255:80
// 172.67.75.171:80
// 45.131.4.157:80
// 141.101.121.208:80
// 141.101.120.70:80
// 103.21.244.70:80
// 103.21.244.140:80
// 103.21.244.138:80
// 154.16.146.43:80
// 128.199.202.122:3128
// 160.153.0.28:80
// 185.170.166.31:80
// 172.67.70.95:80
// 141.101.123.91:80
// 141.101.121.81:80
// 141.101.121.71:80
// 141.101.120.224:80
// 192.111.137.35:4145
// 45.12.31.20:80
// 103.21.244.150:80
// 103.21.244.214:80
// 103.21.244.29:80
// 195.85.23.88:80
// 23.227.39.134:80
// 66.42.224.229:41679
// 172.67.73.123:80
// 79.110.200.27:8000
// 5.182.34.33:80
// 103.21.244.174:80
// 172.67.75.203:80
// 141.101.122.150:80
// 141.101.120.35:80
// 175.47.237.95:6128
// 103.21.244.185:80
// 141.101.122.86:80
// 141.101.121.89:80
// 103.21.244.105:80
// 103.21.244.234:80
// 172.67.70.20:80
// 163.223.172.27:1080
// 172.67.84.128:80
// 103.21.244.57:80
// 31.43.179.191:80
// 173.245.59.61:80
// 141.101.120.48:80
// 185.162.230.19:80
// 172.67.172.150:80
// 159.112.235.63:80
// 172.64.40.184:80
// 103.21.244.55:80
// 103.21.244.49:80
// 172.67.70.228:80
// 103.21.244.8:80
// 66.235.200.77:80
// 45.85.119.147:80
// 172.67.177.231:80
// 173.245.49.28:80
// 110.232.92.49:8080
// 212.183.88.212:80
// 108.162.193.73:80
// 185.238.228.29:80
// 172.67.127.8:80
// 164.38.155.86:80
// 160.153.0.18:80
// 141.101.122.119:80
// 103.21.244.186:80
// 172.67.172.162:80
// 8.218.39.40:10800
// 47.237.132.101:60031
// 222.59.173.105:44193
// 143.110.190.60:1080
// 152.53.194.55:32059
// 115.187.50.40:5678
// 171.254.219.180:1080
// 103.165.157.155:8080
// 45.233.169.57:999
// 222.59.173.105:45035
// 202.40.179.18:4145
// 170.79.181.188:60606
// 37.186.66.36:3629
// 104.200.152.30:4145
// 106.13.58.110:8888
// 202.40.181.220:31247
// 43.135.36.240:80
// 103.82.27.107:10001
// 192.252.209.158:4145
// 142.54.236.97:4145
// 142.54.239.1:4145
// 184.170.245.148:4145
// 68.71.247.130:4145
// 72.195.34.58:4145
// 172.65.90.2:80
// 172.65.90.0:80
// 36.138.53.26:10019
// 183.215.23.242:9091
// 45.131.7.34:80
// 141.101.120.230:80
// 172.64.78.193:80
// 141.101.122.9:80
// 103.21.244.80:80
// 103.21.244.147:80
// 103.21.244.106:80
// 103.21.244.16:80
// 199.34.229.210:80
// 141.101.90.98:80
// 172.67.70.127:80
// 103.21.244.221:80
// 172.67.171.203:80
// 172.67.181.188:80
// 72.195.101.99:4145
// 141.101.120.106:80
// 103.21.244.180:80
// 103.21.244.102:80
// 172.67.70.233:80
// 108.162.193.107:80
// 45.131.208.112:80
// 154.194.12.195:80
// 69.84.182.23:80
// 141.101.120.128:80
// 141.101.120.149:80
// 141.101.120.232:80
// 141.101.120.6:80
// 172.67.98.18:80
// 141.101.113.20:80
// 208.65.90.21:4145
// 198.177.254.131:4145
// 142.54.237.38:4145
// 192.252.220.89:4145
// 199.187.210.54:4145
// 199.102.105.242:4145
// 199.102.107.145:4145
// 198.8.84.3:4145
// 98.182.171.161:4145
// 192.111.129.150:4145
// 184.170.248.5:4145
// 98.188.47.150:4145
// 125.228.94.232:4145
// 125.228.94.153:4145
// 185.162.229.219:80
// 103.21.244.22:80
// 172.64.89.97:80
// 103.21.244.54:80
// 103.21.244.37:80
// 141.101.123.225:80
// 103.21.244.168:80
// 103.21.244.149:80
// 103.21.244.100:80
// 103.21.244.24:80
// 23.227.39.209:80
// 172.67.83.15:80
// 45.131.4.241:80
// 141.101.120.201:80
// 172.67.191.237:80
// 172.67.127.188:80
// 103.21.244.92:80
// 103.160.204.22:80
// 220.197.44.36:3128
// 39.185.41.193:5911
// 23.227.39.121:80
// 139.162.78.109:80
// 188.114.99.144:80
// 45.131.5.37:80
// 45.131.4.250:80
// 23.227.39.65:80
// 141.101.121.191:80
// 172.67.188.16:80
// 172.67.254.148:80
// 221.1.104.177:7302
// 222.59.173.105:44008
// 103.21.244.83:80
// 172.64.149.1:80
// 45.12.30.22:80
// 222.59.173.105:44027
// 172.64.149.26:80
// 103.21.244.46:80
// 45.12.31.242:80
// 23.227.38.195:80
// 172.67.177.162:80
// 103.160.204.200:80
// 172.67.70.129:80
// 141.101.123.245:80
// 185.162.230.117:80
// 185.162.230.183:80
// 69.61.200.104:36181
// 208.65.90.3:4145
// 72.223.188.92:4145
// 192.252.214.17:4145
// 68.71.242.118:4145
// 192.252.210.233:4145
// 107.181.161.81:4145
// 107.181.168.145:4145
// 206.220.175.2:4145
// 72.37.217.3:4145
// 192.252.208.70:14282
// 70.166.167.55:57745
// 192.111.137.37:18762
// 98.188.47.132:4145
// 62.99.138.162:80
// 172.64.90.186:80
// 172.67.74.57:80
// 45.131.6.67:80
// 141.101.122.37:80
// 103.21.244.134:80
// 172.64.155.71:80
// 5.182.34.139:80
// 45.131.6.31:80
// 141.101.121.21:80
// 141.101.120.190:80
// 103.21.244.161:80
// 103.21.244.156:80
// 103.21.244.151:80
// 172.64.89.0:80
// 58.216.109.17:800
// 58.241.88.18:800
// 36.147.78.166:80
// 170.244.26.36:8888
// 170.244.25.52:8888
// 170.244.26.206:8888
// 36.138.53.26:10017
// 201.148.32.162:80
// 203.19.38.114:1080
// 31.43.179.204:80
// 45.12.31.50:80
// 170.244.27.142:8888
// 170.244.26.195:8888
// 170.244.27.58:8888
// 170.244.27.61:8888
// 170.244.27.150:8888
// 47.243.94.125:1080
// 40.177.65.8:80
// 47.57.13.107:80
// 194.158.203.14:80
// 194.219.134.234:80
// 103.21.244.51:80
// 103.21.244.160:80
// 103.21.244.144:80
// 213.33.126.130:80
// 103.21.244.69:80
// 103.21.244.192:80
// 103.21.244.133:80
// 141.193.213.189:80
// 68.71.254.6:4145
// 185.221.160.21:80
// 188.114.99.97:80
// 63.141.128.94:80
// 141.193.213.213:80
// 185.162.228.234:80
// 213.143.113.82:80
// 154.194.12.207:80
// 189.203.181.34:1080
// 202.144.134.150:5678
// 190.242.157.215:8080
// 72.49.49.11:31034
// 142.54.237.34:4145
// 68.71.249.153:48606
// 192.252.209.155:14455
// 192.252.216.86:4145
// 142.54.229.249:4145
// 209.97.150.167:8080
// 192.252.220.92:17328
// 98.178.72.21:10919
// 144.124.228.87:1080
// 183.240.46.42:80
// 32.223.6.94:80
// 192.252.214.20:15864
// 198.177.252.24:4145
// 192.252.208.67:14287
// 198.8.94.170:4145
// 68.71.241.33:4145
// 98.181.137.83:4145
// 45.12.30.139:80
// 103.21.244.97:80
// 65.1.148.157:80
// 199.58.184.97:4145
// 141.101.120.148:80
// 185.238.228.77:80
// 172.67.202.134:80
// 172.67.182.49:80
// 45.131.210.112:80
// 141.101.120.100:80
// 103.21.244.59:80
// 103.21.244.189:80
// 103.21.244.154:80
// 103.21.244.23:80
// 173.245.49.40:80
// 82.200.235.134:38191
// 43.224.118.89:2626
// 119.148.47.226:16464
// 211.230.49.122:3128
// 62.171.159.232:8888
// 115.127.112.34:1080
// 185.191.236.162:3128
// 41.223.119.156:3128
// 103.138.123.242:8082
// 121.169.46.116:1090
// 185.145.185.218:8080
// 45.166.93.113:999
// 163.53.204.178:9813
// 181.224.226.154:8080

5
internal/utils/go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/jobs-scraper/internal/utils
go 1.24.0
toolchain go1.24.7

View file

@ -0,0 +1,26 @@
package utils
import "math"
type PageViewModel struct {
Total int `json:"total"`
PageNumber int `json:"pageNumber"`
PageSize int `json:"pageSize"`
TotalPages int `json:"totalPages"`
HasPrevious bool `json:"hasPrevious"`
HasNext bool `json:"hasNext"`
}
// NewPageViewModel creates a new PageViewModel with calculated fields
func NewPageViewModel(count, pageNumber, pageSize int) *PageViewModel {
totalPages := int(math.Ceil(float64(count) / float64(pageSize)))
return &PageViewModel{
Total: count,
PageNumber: pageNumber,
PageSize: pageSize,
TotalPages: totalPages,
HasPrevious: pageNumber > 1,
HasNext: pageNumber < totalPages,
}
}

View file

@ -0,0 +1,110 @@
package utils
import (
"context"
// "crypto/tls"
"fmt"
"io"
"net/http"
// "net/url"
"time"
)
type RetryConfig struct {
MaxRetries int
BaseDelay time.Duration
MaxDelay time.Duration
}
type RetryableHTTPRequestImpl struct {
client *http.Client
config RetryConfig
}
func NewRetryableHTTPRequest(config RetryConfig) *RetryableHTTPRequestImpl {
return NewRetryableHTTPRequestWithProxy(config, "")
}
func NewRetryableHTTPRequestWithProxy(config RetryConfig, proxyURL string) *RetryableHTTPRequestImpl {
// var transport *http.Transport
// if proxyURL != "" {
// proxyUrl, err := url.Parse(proxyURL)
// if err != nil {
// panic(fmt.Sprintf("Failed to parse proxy URL '%s': %v", proxyURL, err))
// }
// transport = &http.Transport{
// Proxy: http.ProxyURL(proxyUrl),
// TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// }
// }
client := &http.Client{
Timeout: 30 * time.Second,
// Transport: transport,
}
return &RetryableHTTPRequestImpl{
client: client,
config: config,
}
}
func (s *RetryableHTTPRequestImpl) RetryableHTTPRequest(ctx context.Context, url, method string, body io.Reader, headers []http.Header) (*http.Response, error) {
var lastErr error
for attempt := 0; attempt <= s.config.MaxRetries; attempt++ {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
for _, header := range headers {
req.Header.Set(header.Get("Key"), header.Get("Value"))
}
resp, err := s.client.Do(req)
if err != nil {
lastErr = err
fmt.Printf("Request attempt %d failed: %v\n", attempt+1, err)
} else if resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices {
// Success
return resp, nil
} else if resp.StatusCode == http.StatusTooManyRequests {
// 429 Too Many Requests - retry with backoff
resp.Body.Close()
lastErr = fmt.Errorf("rate limited: %d %s", resp.StatusCode, resp.Status)
fmt.Printf("Request attempt %d failed with status %d (rate limited)\n", attempt+1, resp.StatusCode)
} else {
// All other errors (4xx, 5xx) - don't retry
resp.Body.Close()
return nil, fmt.Errorf("request failed %d: %s", resp.StatusCode, resp.Status)
}
// max attempts for now = 3
if attempt < s.config.MaxRetries {
// Increment delay by 2 seconds for each attempt
delay := time.Second * time.Duration(2*(attempt+1)) // 2s, 4s, 6s, ...
// Cap the delay to MaxDelay if set
if s.config.MaxDelay > 0 && delay > s.config.MaxDelay {
delay = s.config.MaxDelay
}
fmt.Printf("Retrying in %v... (attempt %d/%d)\n", delay, attempt+1, s.config.MaxRetries)
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-time.After(delay):
// Continue to next attempt
}
}
}
return nil, fmt.Errorf("all retry attempts failed, last error: %w", lastErr)
}

1146
libs/browser-automation/package-lock.json generated Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,25 @@
{
"name": "@jobs-scraper/browser-automation",
"version": "1.0.0",
"description": "Browser automation library using Puppeteer - ported from Go chromedp implementation",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "ts-node src/index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": ["puppeteer", "browser", "automation", "scraping"],
"author": "",
"license": "ISC",
"dependencies": {
"dotenv": "^17.2.3",
"puppeteer-core": "^24.25.0"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,389 @@
import puppeteer, {
Browser as PuppeteerBrowser,
Page,
ElementHandle,
} from "puppeteer-core";
import { ChildProcess } from "child_process";
import {
launchChromeWithDebugging,
waitForChromeReady,
getChromeWebSocketUrl,
} from "./utils/chrome-launcher";
import { randomDelay } from "./utils/helpers";
export interface BrowserOptions {
headless?: boolean;
port?: number;
userAgent?: string;
}
export class Browser {
private browser: PuppeteerBrowser | null = null;
private page: Page | null = null;
private chromeProcess: ChildProcess | null = null;
private port: number;
private userAgent: string;
constructor(options: BrowserOptions = {}) {
this.port = options.port || 9222;
this.userAgent =
options.userAgent ||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
}
/**
* Initialize the browser by launching Chrome and connecting via Puppeteer
*/
async init(headless: boolean = false): Promise<void> {
this.chromeProcess = launchChromeWithDebugging(this.port, headless);
await waitForChromeReady(this.port);
const wsUrl = await getChromeWebSocketUrl(this.port);
this.browser = await puppeteer.connect({
browserWSEndpoint: wsUrl,
defaultViewport: null,
});
console.log("Connected to browser successfully");
const pages = await this.browser.pages();
if (pages.length === 0) {
this.page = await this.browser.newPage();
} else {
this.page = pages[0];
}
await this.page.setUserAgent(this.userAgent);
}
/**
* Close the browser and kill Chrome process
*/
async close(): Promise<void> {
if (this.browser) {
await this.browser.close();
this.browser = null;
}
if (this.chromeProcess) {
this.chromeProcess.kill("SIGINT");
this.chromeProcess = null;
}
console.log("Browser closed");
}
/**
* Create a new tab
*/
async newTab(): Promise<Page> {
if (!this.browser) {
throw new Error("Browser not initialized");
}
return await this.browser.newPage();
}
/**
* Get the current page
*/
getPage(): Page {
if (!this.page) {
throw new Error("Page not initialized");
}
return this.page;
}
/**
* Navigate to a URL
*/
async navigate(url: string): Promise<void> {
const page = this.getPage();
await page.goto(url, {
waitUntil: ["networkidle2", "domcontentloaded"],
timeout: 60000,
});
}
/**
* Wait for an element to be visible
*/
async waitVisible(
selector: string,
timeout: number = 30000
): Promise<ElementHandle<Element> | null> {
const page = this.getPage();
return await page.waitForSelector(selector, {
visible: true,
timeout,
});
}
/**
* Wait for an element to be ready in the DOM (not necessarily visible)
*/
async waitReady(
selector: string,
timeout: number = 30000
): Promise<ElementHandle<Element> | null> {
const page = this.getPage();
return await page.waitForSelector(selector, {
timeout,
});
}
/**
* Get all elements matching a selector
*/
async getNodes(selector: string): Promise<ElementHandle<Element>[]> {
const page = this.getPage();
await this.waitVisible(selector);
return await page.$$(selector);
}
/**
* Get the outer HTML of the first element matching the selector
*/
async getOuterHTML(selector: string): Promise<string> {
const page = this.getPage();
await this.waitVisible(selector);
const html = await page.$eval(selector, (el) => el.outerHTML);
return html;
}
/**
* Get text content from an element
*/
async getText(selector: string): Promise<string> {
const page = this.getPage();
await this.waitVisible(selector);
const text = await page.$eval(selector, (el) => el.textContent || "");
return text.trim();
}
/**
* Get an attribute value from an element
*/
async getAttribute(selector: string, attribute: string): Promise<string> {
const page = this.getPage();
await this.waitVisible(selector);
const value = await page.$eval(
selector,
(el, attr) => el.getAttribute(attr) || "",
attribute
);
return value;
}
/**
* Evaluate JavaScript in the browser context
*/
async evaluate<T>(expression: string): Promise<T> {
const page = this.getPage();
return (await page.evaluate(expression)) as T;
}
/**
* Get the current page URL
*/
async getCurrentLocation(): Promise<string> {
const page = this.getPage();
return page.url();
}
/**
* Get the full HTML of the page
*/
async getHTML(): Promise<string> {
const page = this.getPage();
return await page.content();
}
/**
* Wait for network to be idle
*/
async waitForNetworkIdle(
idleTime: number = 500,
timeout: number = 30000
): Promise<void> {
const page = this.getPage();
await page.waitForNetworkIdle({ idleTime, timeout });
}
/**
* Type text into an input field (simulates keystrokes)
*/
async type(selector: string, value: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.type(selector, value);
}
/**
* Click on an element
*/
async click(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.click(selector);
}
/**
* Select an option from a <select> element by value
*/
async select(selector: string, value: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.select(selector, value);
}
/**
* Set a checkbox to checked or unchecked state
*/
async setCheckbox(selector: string, checked: boolean): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
const isChecked = await page.$eval(
selector,
(el) => (el as HTMLInputElement).checked
);
if (isChecked !== checked) {
await page.click(selector);
}
}
/**
* Click a radio button to select it
*/
async setRadio(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.click(selector);
}
/**
* Upload a file to a file input
*/
async uploadFile(selector: string, filePath: string): Promise<void> {
const page = this.getPage();
await this.waitReady(selector);
const input = (await page.$(selector)) as ElementHandle<HTMLInputElement> | null;
if (!input) {
throw new Error(`File input not found: ${selector}`);
}
await input.uploadFile(filePath);
}
/**
* Set the value of an input field directly (useful for hidden, date, color, etc.)
*/
async setValue(selector: string, value: string): Promise<void> {
const page = this.getPage();
await this.waitReady(selector);
await page.$eval(
selector,
(el, val) => {
(el as HTMLInputElement).value = val;
el.dispatchEvent(new Event("input", { bubbles: true }));
el.dispatchEvent(new Event("change", { bubbles: true }));
},
value
);
}
/**
* Clear the value of an input field
*/
async clear(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
// Triple-click to select all text, then delete
await page.click(selector, { clickCount: 3 });
await page.keyboard.press("Backspace");
}
/**
* Focus on an element
*/
async focus(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.focus(selector);
}
/**
* Take a screenshot
*/
async screenshot(path?: string): Promise<Buffer> {
const page = this.getPage();
const buffer = await page.screenshot({ path });
return buffer as Buffer;
}
/**
* Random delay between min and max milliseconds
*/
async randomDelay(minMs: number, maxMs: number): Promise<void> {
await randomDelay(minMs, maxMs);
}
/**
* Scroll to an element
*/
async scrollToElement(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.$eval(selector, (el) => {
el.scrollIntoView({ behavior: "smooth", block: "center" });
});
}
/**
* Check if an element exists
*/
async exists(selector: string): Promise<boolean> {
const page = this.getPage();
const element = await page.$(selector);
return element !== null;
}
/**
* Get the count of elements matching a selector
*/
async count(selector: string): Promise<number> {
const page = this.getPage();
const elements = await page.$$(selector);
return elements.length;
}
/**
* Press a keyboard key
*/
async pressKey(key: string): Promise<void> {
const page = this.getPage();
await page.keyboard.press(key as any);
}
/**
* Triple-click to select all text in an input
*/
async selectAllText(selector: string): Promise<void> {
const page = this.getPage();
await this.waitVisible(selector);
await page.click(selector, { clickCount: 3 });
}
/**
* Clear and type new value (common pattern)
*/
async clearAndType(selector: string, value: string): Promise<void> {
await this.clear(selector);
await this.type(selector, value);
}
}

View file

@ -0,0 +1,18 @@
export { Browser, BrowserOptions } from "./browser";
export {
launchChromeWithDebugging,
waitForChromeReady,
getChromeWebSocketUrl,
ChromeVersionResponse,
} from "./utils/chrome-launcher";
export { randomDelay, sleep } from "./utils/helpers";
export {
OpenRouterService,
OpenRouterServiceOptions,
FieldType,
FormField,
FormExtractionResult,
JobAnalysisResult,
JobDescription,
getMockFormData,
} from "./openrouter";

View file

@ -0,0 +1,12 @@
export { OpenRouterService, OpenRouterServiceOptions } from "./openrouter";
export {
FieldType,
FormField,
FormExtractionResult,
JobAnalysisResult,
JobDescription,
OpenRouterMessage,
OpenRouterRequest,
OpenRouterResponse,
} from "./types";
export { getMockFormData } from "./mock-data";

View file

@ -0,0 +1,102 @@
import { FormExtractionResult } from "./types";
/**
* Returns mock form extraction result for testing
*/
export function getMockFormData(): FormExtractionResult {
return {
fields: [
{
label: "Resume",
field_name: "resume",
field_type: "file",
value: "",
placeholder: "",
required: true,
selector: "input[data-ui='resume']",
},
{
label: "Cover letter",
field_name: "cover_letter",
field_type: "textarea",
value:
"Dear KOMOJU Hiring Team,\n\nI am excited to apply for the Fullstack Developer position at KOMOJU. With over 3 years of experience building web applications and leading development projects, I am confident I can contribute to your Fraud Prevention Team's mission of protecting merchants and customers across borders.\n\nMy experience includes leading the development of online shopping platforms, mentoring junior developers, and coordinating complex projects. I have hands-on experience with modern web technologies including React, Node.js, and microservices architecture. I am a motivated self-starter who can work independently while collaborating effectively with cross-functional teams.\n\nI am eager to bring my technical skills and passion for product-focused development to KOMOJU's innovative payment gateway solutions.\n\nThank you for your consideration.\n\nBest regards,\nZiad Elshimy",
placeholder: "",
required: false,
selector: "#cover_letter",
},
{
label:
"Do you have at least 2 years of personal or professional experience with Ruby?",
field_name: "QA_10609776",
field_type: "radio",
value: "false",
placeholder: "",
required: true,
selector: "input[name='QA_10609776']",
},
{
label: "Could you tell us more about your Ruby experience?",
field_name: "QA_10609777",
field_type: "textarea",
value:
"I do not have professional Ruby experience, but I have extensive experience with JavaScript (React, Node.js), TypeScript, and Go. I am a fast learner with strong fundamentals in software development best practices, data structures, and algorithms. I am excited about the opportunity to learn Ruby and contribute to KOMOJU's fraud prevention systems.",
placeholder: "",
required: true,
selector: "#QA_10609777",
},
{
label:
"Have you ever investigated and resolved a production issue that wasn't reproducible in a development or staging environment?",
field_name: "QA_10609885",
field_type: "radio",
value: "true",
placeholder: "",
required: true,
selector: "input[name='QA_10609885']",
},
{
label:
"If the answer for above question is yes, can you tell us how did you approach it?",
field_name: "QA_10609886",
field_type: "textarea",
value:
"Yes, I have experience debugging production issues. My approach includes: 1) Analyzing production logs and error reports to identify patterns, 2) Using monitoring tools to track system behavior and performance metrics, 3) Creating targeted tests to reproduce the issue in isolated environments, 4) Collaborating with team members to validate hypotheses, and 5) Implementing and deploying fixes with proper testing and monitoring. I understand the importance of being systematic and thorough when diagnosing complex production issues.",
placeholder: "",
required: true,
selector: "#QA_10609886",
},
{
label: "Do you currently reside in Japan?",
field_name: "QA_10609774",
field_type: "radio",
value: "false",
placeholder: "",
required: true,
selector: "input[name='QA_10609774']",
},
{
label:
"This role requires at least 5 hours of overlap with Japan Standard Time (JST) business hours. Are you able to accommodate this?",
field_name: "QA_10609887",
field_type: "radio",
value: "true",
placeholder: "",
required: true,
selector: "input[name='QA_10609887']",
},
{
label:
"Are you willing to relocate to Japan? If so, what are your motivations for doing so?",
field_name: "QA_10609888",
field_type: "textarea",
value:
"Yes, I am willing to relocate to Japan. My motivations include: 1) Joining KOMOJU's innovative team working on cutting-edge payment technology, 2) Experiencing Japan's world-class technology culture and work ethic, 3) Contributing to a product that powers payments for major platforms like Steam and TikTok, 4) Growing my career in an international environment with diverse perspectives, and 5) Embracing the opportunity to learn Japanese culture and language while working on globally impactful projects.",
placeholder: "",
required: true,
selector: "#QA_10609888",
},
],
apply_button: `button[data-ui="apply-button"]`,
};
}

View file

@ -0,0 +1,256 @@
import {
FormExtractionResult,
JobAnalysisResult,
JobDescription,
OpenRouterRequest,
OpenRouterResponse,
} from "./types";
export interface OpenRouterServiceOptions {
model: string;
apiKey: string;
}
export class OpenRouterService {
private model: string;
private apiKey: string;
private baseUrl = "https://openrouter.ai/api/v1/chat/completions";
constructor(options: OpenRouterServiceOptions) {
this.model = options.model;
this.apiKey = options.apiKey;
}
/**
* Clean markdown code blocks from response
*/
private cleanMarkdownCodeBlocks(content: string): string {
content = content.trim();
if (content.startsWith("```json")) {
content = content.slice(7);
} else if (content.startsWith("```")) {
content = content.slice(3);
}
if (content.endsWith("```")) {
content = content.slice(0, -3);
}
return content.trim();
}
/**
* Make a request to OpenRouter API
*/
private async makeRequest(
systemMessage: string,
userMessage: string
): Promise<string> {
const request: OpenRouterRequest = {
model: this.model,
messages: [
{ role: "system", content: systemMessage },
{ role: "user", content: userMessage },
],
};
const response = await fetch(this.baseUrl, {
method: "POST",
headers: {
Authorization: `Bearer ${this.apiKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify(request),
});
if (!response.ok) {
const errorText = await response.text();
throw new Error(
`OpenRouter API error: ${response.status} - ${errorText}`
);
}
const data = (await response.json()) as OpenRouterResponse;
if (!data.choices || data.choices.length === 0) {
throw new Error("No response choices received from API");
}
const content = data.choices[0].message.content;
if (content.startsWith("<")) {
throw new Error(
`API returned HTML/XML instead of JSON: ${content.slice(0, 200)}`
);
}
return this.cleanMarkdownCodeBlocks(content);
}
/**
* Analyze a job description against a CV
*/
async analyzeJobDescription(
cv: string,
jobDesc: JobDescription
): Promise<JobAnalysisResult> {
const userMessage = `Analyze the following CV against the job description and criteria, then provide a recommendation following the schema below.
1) If there are missing skills, try to guess if they still match based on similar skills or experience in the cv.
for example: Javascript is mentioned in the cv, but the job requires Vanilla js, since they are the same thing, it should be included in the matching skills.
2) The job shouldn't require any language skills, preferbly only english.
3) The job should be remote, or provide relocation to the country.
CV:
${cv}
Job Description:
${jobDesc.description}
Job Criteria (key-value):
${JSON.stringify(jobDesc.criteria, null, 2)}
CRITICAL OUTPUT REQUIREMENTS:
- Return ONLY raw JSON - NO markdown formatting whatsoever
- NO backticks, NO code blocks, NO json prefix
- NO additional text before or after the JSON
- Start your response directly with { and end with }
- Use this exact schema and key names:
{
"recommendation": "apply" | "do_not_apply",
"confidence_score": number, // integer 0-100
"matching_skills": [string],
"missing_skills": [string],
"experience_match": "excellent" | "good" | "fair" | "poor",
"summary": string,
"improvement_suggestions": [string]
}`;
const systemMessage =
"You are an expert HR assistant specializing in job application analysis. You help candidates determine if they should apply for specific positions based on their CV and the job requirements. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
const jsonContent = await this.makeRequest(systemMessage, userMessage);
console.log("Raw API response:", jsonContent);
const result = JSON.parse(jsonContent) as JobAnalysisResult;
return result;
}
/**
* Create a job description from HTML content
*/
async createJobDescription(htmlContent: string): Promise<JobDescription> {
const userMessage = `You are a job posting data extractor. Analyze the HTML content and extract data according to this JSON structure:
{"description":"job description text","criteria":{"title":"Job Title","company":"Company","location":"Location","salary":"Salary","skills":"Skills","experience":"Experience","job_type":"Job Type","remote":"Remote"}}
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Use SINGLE-LINE JSON (no pretty printing, no newlines inside the JSON)
3. All criteria values MUST be strings (never use booleans, numbers, or arrays)
4. Properly escape all quotes inside strings using \\"
5. Use only standard ASCII quotes ("), never smart quotes (" " ' ')
7. If no job description is found, use empty string for "description": ""
8. All criteria fields must be present with empty string "" if not found
Example output:
{"description":"Develop software applications...","criteria":{"title":"Backend Engineer","company":"Tech Corp","location":"Remote","salary":"$120k","skills":"Go, Docker","experience":"3+ years","job_type":"Full-time","remote":"Yes"}}
Extract from this HTML Content:
${htmlContent}`;
const systemMessage =
"You are an expert job description extractor. You convert unstructured job description text into a structured JSON format. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
const jsonContent = await this.makeRequest(systemMessage, userMessage);
console.log("Raw API response for job description:", jsonContent);
const result = JSON.parse(jsonContent) as JobDescription;
return result;
}
/**
* Extract form fields from HTML and populate them based on CV data.
* Also adjusts the CV based on job type (frontend/backend/fullstack).
*/
async applyForJob(
htmlForm: string,
cvMarkdown: string,
jobDescription: string
): Promise<FormExtractionResult> {
const userMessage = `You are a form field extractor, auto-filler, and CV adapter. Analyze the HTML form and job description, then:
1. Extract ALL form fields and populate them with appropriate values from the CV
HTML FORM:
${htmlForm}
JOB DESCRIPTION:
${jobDescription}
ORIGINAL CV (Markdown):
${cvMarkdown}
CV ADJUSTMENT RULES:
- Detect if the job is: FRONTEND, BACKEND, or FULLSTACK based on the job description
- If FRONTEND: Keep only frontend-related skills, projects, and experience (React, Vue, Angular, CSS, HTML, UI/UX, etc.). Remove backend-specific content.
- If BACKEND: Keep only backend-related skills, projects, and experience (APIs, databases, servers, Go, Node.js, Python, etc.). Remove frontend-specific content.
- If FULLSTACK: Keep both frontend and backend content.
- For any OTHER job type (not frontend/backend/fullstack): Treat as FRONTEND by default.
- Maintain the same markdown structure and formatting as the original CV.
- Do NOT invent new skills or experience - only filter existing content.
CRITICAL RULES:
1. Return ONLY valid JSON - no markdown, no code blocks, no backticks, no explanations
2. Extract ALL input fields, textareas, selects, and buttons from the form
3. For each field, determine the appropriate value from the ADJUSTED CV
4. If a field cannot be populated from the CV (e.g., password, captcha), leave value as empty string
5. Identify the apply/submit button text
6. Use the exact JSON schema below
7. If a field is already populated, ignore it (don't send it in the results)
JSON Schema:
{
"fields": [
{
"label": "Field label or name attribute",
"field_name": "name or id attribute of the input",
"field_type": "text|email|tel|password|number|date|datetime-local|time|month|week|url|search|color|range|file|hidden|checkbox|radio|select|textarea|button|submit|reset",
"value": "Value to populate based on CV data",
"placeholder": "Placeholder text if any",
"required": true|false,
"selector": "Unique CSS selector for the input (e.g., #email, input[name='email'], .form-field-email). Use id selector if available, otherwise name attribute, otherwise class. Empty string if no unique selector can be determined."
}
],
"apply_button": "Unique CSS selector for the form's submit button",
}
FIELD MAPPING GUIDELINES:
- Name fields: Extract full name, first name, last name from CV
- Email: Use email from CV contact info
- Phone: Use phone number from CV
- LinkedIn/Portfolio/Website: Use URLs from CV
- Experience/Years: Calculate from CV work history
- Current company/title: Use most recent from CV
- Skills: List relevant skills from ADJUSTED CV
- Education: Use education details from CV
- Cover letter/Message: Generate a brief professional message based on ADJUSTED CV
- Salary expectations: Leave empty unless specified in CV
- Location/Address: Use from CV contact info
- Resume/CV upload: Leave value empty (file upload)
Start your response with { and end with }`;
const systemMessage =
"You are an expert form analyzer and auto-filler. You extract form fields from HTML and intelligently populate them with data from a CV/resume. CRITICAL: You must respond with ONLY raw JSON - no markdown, no code blocks, no backticks, no additional text. Start directly with { and end with }.";
const jsonContent = await this.makeRequest(systemMessage, userMessage);
console.log("Raw API response for form extraction:", jsonContent);
const result = JSON.parse(jsonContent) as FormExtractionResult;
return result;
}
}

View file

@ -0,0 +1,104 @@
/**
* FieldType represents HTML input field types
*/
export type FieldType =
| "text"
| "email"
| "tel"
| "password"
| "number"
| "date"
| "datetime-local"
| "time"
| "month"
| "week"
| "url"
| "search"
| "color"
| "range"
| "file"
| "hidden"
| "checkbox"
| "radio"
| "select"
| "textarea"
| "button"
| "submit"
| "reset";
/**
* FormField represents a single form field extracted from HTML
*/
export interface FormField {
label: string;
field_name: string;
field_type: FieldType;
value: string;
placeholder?: string;
required: boolean;
selector: string;
}
/**
* FormExtractionResult represents the structured response from form extraction
*/
export interface FormExtractionResult {
fields: FormField[];
apply_button: string;
}
/**
* JobAnalysisResult represents the structured response from job analysis
*/
export interface JobAnalysisResult {
recommendation: "apply" | "do_not_apply";
confidence_score: number;
matching_skills: string[];
missing_skills: string[];
experience_match: "excellent" | "good" | "fair" | "poor";
summary: string;
improvement_suggestions: string[];
}
/**
* JobDescription represents a parsed job description
*/
export interface JobDescription {
description: string;
criteria: Record<string, string>;
}
/**
* OpenRouter API message format
*/
export interface OpenRouterMessage {
role: "system" | "user" | "assistant";
content: string;
}
/**
* OpenRouter API request body
*/
export interface OpenRouterRequest {
model: string;
messages: OpenRouterMessage[];
}
/**
* OpenRouter API response
*/
export interface OpenRouterResponse {
id: string;
choices: {
message: {
role: string;
content: string;
};
finish_reason: string;
}[];
usage?: {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
};
}

View file

@ -0,0 +1,144 @@
import fs from "fs";
import path from "path";
import os from "os";
import { spawn, ChildProcess } from "child_process";
import net from "net";
export interface ChromeVersionResponse {
webSocketDebuggerUrl: string;
}
const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile");
function getChromePath(): string {
const platform = os.platform();
const paths: Record<NodeJS.Platform, string[]> = {
darwin: [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
"/Applications/Chromium.app/Contents/MacOS/Chromium",
],
win32: [
process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe",
process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe",
process.env["PROGRAMFILES(X86)"] +
"\\Google\\Chrome\\Application\\chrome.exe",
],
linux: [
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
"/usr/bin/chromium",
"/usr/bin/chromium-browser",
],
aix: [],
freebsd: [],
openbsd: [],
sunos: [],
android: [],
haiku: [],
cygwin: [],
netbsd: [],
};
const platformPaths = paths[platform] || [];
for (const chromePath of platformPaths) {
try {
if (fs.existsSync(chromePath)) {
return chromePath;
}
} catch {
continue;
}
}
throw new Error("Could not find Chrome/Chromium installation");
}
export function launchChromeWithDebugging(
port: number = 9222,
headless: boolean = false
): ChildProcess {
try {
const chromePath = getChromePath();
const args = [
`--remote-debugging-port=${port}`,
`--user-data-dir=${userDataDir}`,
"--remote-allow-origins=*",
"--incognito",
];
if (headless) {
args.push("--headless=new");
}
console.log(`Launching Chrome at: ${chromePath}`);
const chromeProcess = spawn(chromePath, args, {
detached: true,
stdio: "ignore",
});
chromeProcess.unref();
chromeProcess.on("error", (err) => {
console.error("Failed to start Chrome:", err);
});
chromeProcess.on("exit", (code) => {
if (code !== 0) {
console.error(`Chrome process exited with code ${code}`);
}
});
console.log(`Chrome launched successfully with debugging port ${port}`);
return chromeProcess;
} catch (error) {
console.error("Failed to launch Chrome:", error);
throw error;
}
}
export async function waitForChromeReady(
port: number = 9222,
timeout: number = 30000
): Promise<void> {
const startTime = Date.now();
return new Promise((resolve, reject) => {
function check() {
const client = net.createConnection({ port }, () => {
client.end();
resolve();
});
client.on("error", () => {
if (Date.now() - startTime > timeout) {
reject(new Error("Timeout waiting for Chrome to start"));
} else {
setTimeout(check, 500);
}
});
}
check();
});
}
export async function getChromeWebSocketUrl(
port: number = 9222
): Promise<string> {
const response = await fetch(`http://localhost:${port}/json/version`, {
headers: {
Origin: "",
},
});
if (!response.ok) {
throw new Error("Failed to connect to Chrome debugging port");
}
const chromeInstance = (await response.json()) as ChromeVersionResponse;
return chromeInstance.webSocketDebuggerUrl;
}

View file

@ -0,0 +1,14 @@
/**
* Random delay between min and max milliseconds
*/
export function randomDelay(minMs: number, maxMs: number): Promise<void> {
const delay = Math.floor(Math.random() * (maxMs - minMs) + minMs);
return new Promise((resolve) => setTimeout(resolve, delay));
}
/**
* Sleep for a specified number of milliseconds
*/
export function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

View file

@ -0,0 +1,34 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": [
"src/**/*.ts"
],
"exclude": [
"node_modules",
"dist"
]
}

View file

@ -4,6 +4,6 @@ go 1.24.0
toolchain go1.24.7
require github.com/jobs-scraper/internal/pkg/domain v0.0.0
require github.com/jobs-scraper/internal/domain v0.0.0
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/internal/domain => ../../internal/domain

View file

@ -1,9 +1,10 @@
package ports
import "github.com/jobs-scraper/internal/pkg/domain"
import "github.com/jobs-scraper/internal/domain"
type JobCommands struct {
CreateJob JobCommandHandler
ApplyForJob ApplyForJobCommandHandler
}
// CreateJobCommand represents the command to create a job
@ -19,3 +20,7 @@ type CreateJobCommand struct {
type JobCommandHandler interface {
Handle(cmd CreateJobCommand) error
}
type ApplyForJobCommandHandler interface {
Handle(jobId int) error
}

25
libs/ports/job-queries.go Normal file
View file

@ -0,0 +1,25 @@
package ports
import (
"github.com/jobs-scraper/internal/dto"
"github.com/jobs-scraper/internal/domain"
)
type JobQueries struct {
GetJobs JobQueryHandler
}
// GetJobQuery represents the command to create a job
type GetJobQuery struct {
Location string `schema:"location"`
Keywords string `schema:"keywords"`
FWT string `schema:"fwt"`
Provider domain.JobProvider `schema:"provider"`
PageNumber int `schema:"pageNumber"`
PageSize int `schema:"pageSize"`
}
// JobQueryHandler defines the interface for handling job queries
type JobQueryHandler interface {
Handle(query GetJobQuery) ([]dto.JobDTO, int, error)
}

102
libs/rabbitmq-ts/package-lock.json generated Normal file
View file

@ -0,0 +1,102 @@
{
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"amqplib": "^0.10.9"
},
"devDependencies": {
"@types/amqplib": "^0.10.8",
"typescript": "^5.9.3"
}
},
"node_modules/@types/amqplib": {
"version": "0.10.8",
"resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz",
"integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/node": {
"version": "25.0.3",
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.0.3.tgz",
"integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"node_modules/amqplib": {
"version": "0.10.9",
"resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz",
"integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==",
"license": "MIT",
"dependencies": {
"buffer-more-ints": "~1.0.0",
"url-parse": "~1.5.10"
},
"engines": {
"node": ">=10"
}
},
"node_modules/buffer-more-ints": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz",
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==",
"license": "MIT"
},
"node_modules/querystringify": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==",
"license": "MIT"
},
"node_modules/requires-port": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==",
"license": "MIT"
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"dev": true,
"license": "MIT"
},
"node_modules/url-parse": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
"license": "MIT",
"dependencies": {
"querystringify": "^2.1.1",
"requires-port": "^1.0.0"
}
}
}
}

View file

@ -0,0 +1,21 @@
{
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"build": "tsc",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"amqplib": "^0.10.9"
},
"devDependencies": {
"@types/amqplib": "^0.10.8",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,173 @@
import * as amqp from "amqplib";
export interface RabbitMQConfig {
url: string;
queueName: string;
exchangeName: string;
exchangeType: "direct" | "topic" | "fanout" | "headers";
durable: boolean;
}
export interface SearchQuery {
keywords: string;
location: string;
fwt?: string;
numPages: number;
}
export class RabbitMQClient {
private connection: amqp.ChannelModel | null = null;
private channel: amqp.Channel | null = null;
private config: RabbitMQConfig;
constructor(config: RabbitMQConfig) {
this.config = config;
}
async connect(): Promise<void> {
try {
console.log(`Connecting to RabbitMQ at: ${this.config.url}`);
this.connection = await amqp.connect(this.config.url);
console.log("Connected to RabbitMQ successfully");
if (!this.connection) {
throw new Error("Failed to establish RabbitMQ connection");
}
this.channel = await this.connection.createChannel();
if (!this.channel) {
throw new Error("Failed to create RabbitMQ channel");
}
// Ensure exchange and queue exist
await this.channel.assertExchange(
this.config.exchangeName,
this.config.exchangeType,
{ durable: this.config.durable }
);
await this.channel.assertQueue(this.config.queueName, {
durable: this.config.durable,
arguments: {
"x-dead-letter-exchange": "scraper_dlx",
"x-dead-letter-routing-key": this.config.queueName,
"x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds
"x-max-retries": 3,
},
});
await this.channel.bindQueue(
this.config.queueName,
this.config.exchangeName,
this.config.queueName
);
// Set prefetch to process one message at a time
await this.channel.prefetch(1);
// Handle connection events
this.connection.on("close", () => {
console.log("RabbitMQ connection closed");
this.connection = null;
this.channel = null;
});
this.connection.on("error", (error: Error) => {
console.error("RabbitMQ connection error:", error);
this.connection = null;
this.channel = null;
});
} catch (error) {
console.error("Failed to connect to RabbitMQ:", error);
throw error;
}
}
async subscribe<T = SearchQuery>(
messageHandler: (message: T) => Promise<void>
): Promise<void> {
if (!this.channel) {
throw new Error(
"RabbitMQ channel not initialized. Call connect() first."
);
}
console.log(
`Waiting for messages from ${this.config.queueName}. To exit press CTRL+C`
);
await this.channel.consume(
this.config.queueName,
async (msg: amqp.ConsumeMessage | null) => {
if (msg && this.channel) {
try {
const messageContent = msg.content.toString();
console.log(`Received message: ${messageContent}`);
const message: T = JSON.parse(messageContent);
console.log(`Processing message:`, message);
// Process the message using the provided handler
await messageHandler(message);
// Acknowledge the message on success
this.channel.ack(msg);
console.log("Message processed successfully");
} catch (error) {
console.error("Error processing message:", error);
// Reject the message and don't requeue it
if (this.channel) {
this.channel.nack(msg, false, false);
}
}
}
}
);
}
async close(): Promise<void> {
try {
if (this.channel) {
try {
await this.channel.close();
} catch {
// Channel may already be closing
}
this.channel = null;
}
if (this.connection) {
try {
await this.connection.close();
} catch {
// Connection may already be closing
}
this.connection = null;
}
console.log("RabbitMQ connection closed gracefully");
} catch (error) {
console.error("Error closing RabbitMQ connection:", error);
}
}
isConnected(): boolean {
return this.connection !== null && this.channel !== null;
}
}
export function createRabbitMQConfig(): RabbitMQConfig {
return {
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor",
exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange",
exchangeType: "topic",
durable: true,
};
}
export async function createRabbitMQClient(): Promise<RabbitMQClient> {
const config = createRabbitMQConfig();
const client = new RabbitMQClient(config);
await client.connect();
return client;
}

View file

@ -0,0 +1,29 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist"]
}

View file

@ -5,8 +5,8 @@ go 1.24.0
toolchain go1.24.7
require (
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/internal/domain v0.0.0
github.com/lib/pq v1.10.9
)
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/internal/domain => ../../internal/domain

View file

@ -4,7 +4,7 @@ import (
"database/sql"
"fmt"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/domain"
"github.com/lib/pq"
)

View file

@ -6,7 +6,7 @@ import (
"fmt"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/domain"
)
type JobDescriptionRepository struct {

View file

@ -3,10 +3,12 @@ package repo
import (
"bytes"
"database/sql"
"encoding/json"
"fmt"
"strconv"
"strings"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/domain"
)
type JobRepository struct {
@ -94,33 +96,92 @@ func (r *JobRepository) SaveJobs(jobs []domain.Job) error {
return nil
}
func (r *JobRepository) GetAllJobs() ([]domain.Job, error) {
rows, err := r.db.Query("SELECT * FROM jobs")
type JobFilter struct {
Location string
Keywords string
Provider domain.JobProvider
PageSize int
PageNumber int
}
func (r *JobRepository) GetAllJobs(filter JobFilter) ([]domain.Job, int, error) {
query := "SELECT id, title, company, company_link, location, job_link, job_timestamp, provider FROM jobs"
countQuery := "SELECT COUNT(*) FROM jobs"
var conditions []string
var args []interface{}
paramIdx := 1
if filter.Location != "" {
conditions = append(conditions, fmt.Sprintf("location ILIKE $%d", paramIdx))
args = append(args, "%"+filter.Location+"%")
paramIdx++
}
if filter.Keywords != "" {
conditions = append(conditions, fmt.Sprintf("(title ILIKE $%d OR company ILIKE $%d)", paramIdx, paramIdx))
args = append(args, "%"+filter.Keywords+"%")
paramIdx++
}
if filter.Provider > 0 {
conditions = append(conditions, fmt.Sprintf("provider = $%d", paramIdx))
args = append(args, filter.Provider)
paramIdx++
}
whereClause := ""
if len(conditions) > 0 {
whereClause = " WHERE " + strings.Join(conditions, " AND ")
query += whereClause
countQuery += whereClause
}
var totalCount int
countArgs := make([]interface{}, len(args))
copy(countArgs, args)
if err := r.db.QueryRow(countQuery, countArgs...).Scan(&totalCount); err != nil {
return nil, 0, fmt.Errorf("error counting jobs: %v", err)
}
if filter.PageSize > 0 {
query += fmt.Sprintf(" LIMIT $%d", paramIdx)
args = append(args, filter.PageSize)
paramIdx++
if filter.PageNumber > 0 {
offset := (filter.PageNumber - 1) * filter.PageSize
query += fmt.Sprintf(" OFFSET $%d", paramIdx)
args = append(args, offset)
paramIdx++
}
}
rows, err := r.db.Query(query, args...)
if err != nil {
return nil, fmt.Errorf("error querying jobs: %v", err)
return nil, 0, fmt.Errorf("error querying jobs: %v", err)
}
defer rows.Close()
var jobs []domain.Job
for rows.Next() {
var job domain.Job
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink); err != nil {
return nil, fmt.Errorf("error scanning job row: %v", err)
if err := rows.Scan(&job.ID, &job.Title, &job.Company, &job.CompanyLink, &job.Location, &job.JobLink, &job.JobPostTime, &job.Provider); err != nil {
return nil, 0, fmt.Errorf("error scanning job row: %v", err)
}
jobs = append(jobs, job)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("error iterating over job rows: %v", err)
return nil, 0, fmt.Errorf("error iterating over job rows: %v", err)
}
return jobs, nil
return jobs, totalCount, nil
}
func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
var job domain.Job
sqlStatement := `
SELECT id, title, company, company_link, location, job_link, job_timestamp
SELECT id, title, company, company_link, location, job_link, job_timestamp, provider
FROM jobs
WHERE id = $1
`
@ -133,6 +194,7 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
&job.Location,
&job.JobLink,
&job.JobPostTime,
&job.Provider,
)
if err == sql.ErrNoRows {
@ -146,6 +208,49 @@ func (r *JobRepository) GetJobByID(id int) (*domain.Job, error) {
return &job, nil
}
func (r *JobRepository) GetJobWithDescription(id int) (*domain.JobWithDescription, error) {
var job domain.JobWithDescription
var criteriaJSON []byte
sqlStatement := `
SELECT j.id, j.title, j.company, j.company_link, j.location, j.job_link, j.job_timestamp, j.provider,
COALESCE(jd.description, ''), COALESCE(jd.job_criteria, '{}'::jsonb)
FROM jobs j
LEFT JOIN job_descriptions jd ON j.id = jd.job_id
WHERE j.id = $1
`
err := r.db.QueryRow(sqlStatement, id).Scan(
&job.Job.ID,
&job.Job.Title,
&job.Job.Company,
&job.Job.CompanyLink,
&job.Job.Location,
&job.Job.JobLink,
&job.Job.JobPostTime,
&job.Job.Provider,
&job.JobDescription.Description,
&criteriaJSON,
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("job with ID %d not found", id)
}
if err != nil {
return nil, fmt.Errorf("error querying job: %v", err)
}
// Parse criteria JSON
if err := json.Unmarshal(criteriaJSON, &job.JobDescription.Criteria); err != nil {
job.JobDescription.Criteria = make(map[string]string)
}
job.JobDescription.JobID = job.Job.ID
return &job, nil
}
func prepareQueryCreateBulk(s string, models []*domain.Job) (string, []interface{}) {
bf := bytes.Buffer{}
values := make([]interface{}, 0, len(models)*7)

View file

@ -0,0 +1,20 @@
deploy_api:
stage: deploy
image: alpine:3.20
environment:
name: production
url: https://jobs-scraper.ai-assistant-bot.xyz
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
when: manual
allow_failure: false
before_script:
- apk add --no-cache openssh-client rsync
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
script:
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/api && docker compose up -d --build"

36
services/api/Dockerfile Normal file
View file

@ -0,0 +1,36 @@
# Build stage
FROM golang:1.24-alpine AS builder
# Install build dependencies
RUN apk add --no-cache git
WORKDIR /build
# Copy go mod files
COPY go.mod go.sum ./
RUN go mod download
# Copy the entire monorepo
COPY . .
# Build the API service
WORKDIR /build/services/api
RUN CGO_ENABLED=0 GOOS=linux go build -o api ./cmd/server
# Runtime stage
FROM alpine:latest
RUN apk add --no-cache ca-certificates
WORKDIR /app
# Copy the binary from builder
COPY --from=builder /build/services/api/api .
# Copy migrations
COPY --from=builder /build/internal/migrations ./internal/migrations
EXPOSE 8080
# Run the API
ENTRYPOINT ["./api"]

View file

@ -10,8 +10,8 @@ import (
"time"
"github.com/gorilla/mux"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/infrastructure"
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/api/internal/app"
httpHandler "github.com/jobs-scraper/services/api/pkg/http"
@ -60,20 +60,22 @@ func main() {
log.Fatalf("Failed to run migrations: %v", err)
}
router := mux.NewRouter()
log.Println("Successfully ran migrations")
// Swagger endpoint
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
router := mux.NewRouter()
router.Use(httpHandler.CORSMiddleware)
router.Use(httpHandler.LogsMiddleware)
// Swagger endpoint
router.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
app := app.NewApplication(router, db, rmq)
// Create job analysis result repository
jobAnalysisResultRepo := repo.NewJobAnalysisResultRepository(db)
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo)
jobHandler := httpHandler.NewJobHandler(app.JobCommands, jobAnalysisResultRepo, app.JobQueries)
jobHandler.RegisterRoutes(router)

View file

@ -0,0 +1,41 @@
services:
api:
build:
context: ../../
dockerfile: services/api/Dockerfile
container_name: jobs-api
ports:
- "8080:8080"
environment:
DB_HOST: shared_postgres
DB_PORT: 5432
DB_NAME: jobs_scraper
DB_USER: scraper_user
DB_PASSWORD: Cocowawa_12345
# RabbitMQ Configuration
RABBITMQ_HOST: 172.18.0.9
RABBITMQ_PORT: 5672
RABBITMQ_USER: admin
RABBITMQ_PASSWORD: 123Kari123!
networks:
- proxy
restart: unless-stopped
labels:
- traefik.enable=true
- traefik.docker.network=proxy
- traefik.http.routers.api-jobs.rule=Host(`api-jobs.ai-assistant-bot.xyz`)
- traefik.http.routers.api-jobs.entrypoints=web,websecure
- traefik.http.routers.api-jobs.tls.certresolver=le
- traefik.http.services.api-jobs.loadbalancer.server.port=8080
# deploy:
# resources:
# limits:
# memory: 512M
# cpus: "1"
# reservations:
# memory: 128M
# cpus: "0.25"
networks:
proxy:
external: true

View file

@ -4,9 +4,10 @@ go 1.24.0
require (
github.com/gorilla/mux v1.8.1
github.com/jobs-scraper/internal/pkg/domain v0.0.0
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/pkg/utils v0.0.0
github.com/gorilla/schema v1.4.1
github.com/jobs-scraper/internal/domain v0.0.0
github.com/jobs-scraper/internal/infrastructure v0.0.0-00010101000000-000000000000
github.com/jobs-scraper/internal/utils v0.0.0
github.com/jobs-scraper/libs/ports v0.0.0
github.com/jobs-scraper/libs/repo v0.0.0
github.com/jobs-scraper/libs/server v0.0.0-00010101000000-000000000000
@ -38,11 +39,11 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
)
replace github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain
replace github.com/jobs-scraper/internal/domain => ../../internal/domain
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure
replace github.com/jobs-scraper/internal/pkg/utils => ../../internal/pkg/utils
replace github.com/jobs-scraper/internal/utils => ../../internal/utils
replace github.com/jobs-scraper/libs/ports => ../../libs/ports

View file

@ -44,6 +44,8 @@ github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9Knoi
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E=
github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@ -56,6 +58,7 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -113,7 +116,9 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
@ -122,10 +127,12 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

View file

@ -4,11 +4,12 @@ import (
"database/sql"
"github.com/gorilla/mux"
"github.com/jobs-scraper/services/api/internal/commands/job"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/libs/server"
jobCommands "github.com/jobs-scraper/services/api/internal/commands/job"
jobQueries "github.com/jobs-scraper/services/api/internal/queries/job"
)
type Application struct {
@ -16,6 +17,7 @@ type Application struct {
DB *sql.DB
Server *server.AppServer
JobCommands *ports.JobCommands
JobQueries *ports.JobQueries
}
func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient) *Application {
@ -28,7 +30,11 @@ func NewApplication(router *mux.Router, db *sql.DB, rmq *rabbitmq.RabbitMQClient
Router: router,
DB: db,
JobCommands: &ports.JobCommands{
CreateJob: job.NewCreateJobHandler(jobRepo, rmq),
CreateJob: jobCommands.NewCreateJobHandler(jobRepo, rmq),
ApplyForJob: jobCommands.NewApplyForJobHandler(jobRepo, rmq),
},
JobQueries: &ports.JobQueries{
GetJobs: jobQueries.NewGetJobsHandler(jobRepo),
},
}
}

View file

@ -0,0 +1,44 @@
package job
import (
"encoding/json"
"github.com/jobs-scraper/internal/dto"
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/repo"
)
type ApplyForJob struct {
jobRepo *repo.JobRepository
rmq *rabbitmq.RabbitMQClient
}
func NewApplyForJobHandler(jobRepo *repo.JobRepository, rmq *rabbitmq.RabbitMQClient) *ApplyForJob {
return &ApplyForJob{
jobRepo: jobRepo,
rmq: rmq,
}
}
func (aj *ApplyForJob) Handle(jobId int) error {
job, err := aj.jobRepo.GetJobWithDescription(jobId)
if err != nil {
return err
}
jobDTO := dto.JobWithDescriptionFromDomain(*job)
jsonData, err := json.Marshal(jobDTO)
if err != nil {
return err
}
err = aj.rmq.Publish(rabbitmq.JobApplierQueue, rabbitmq.ScraperExchange, jsonData)
if err != nil {
return err
}
return nil
}

View file

@ -5,8 +5,8 @@ import (
"errors"
"log"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/infrastructure/rabbitmq"
"github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/infrastructure/rabbitmq"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
)
@ -33,25 +33,21 @@ func (cj *CreateJob) Handle(cmd ports.CreateJobCommand) error {
jsonData, err := json.Marshal(jobRequest)
if err != nil {
log.Printf("Error marshaling JSON: %v", err)
return err
}
switch cmd.Provider {
case domain.LinkedIn:
if err := cj.rmq.Publish(rabbitmq.LinkedInQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
log.Printf("Error publishing LinkedIn message: %v", err)
return err
}
log.Printf("Published LinkedIn job request: %s", string(jsonData))
case domain.Glassdoor:
if err := cj.rmq.Publish(rabbitmq.GlassDoorQueue, rabbitmq.ScraperExchange, jsonData); err != nil {
log.Printf("Error publishing Glassdoor message: %v", err)
return err
}
log.Printf("Published Glassdoor job request: %s", string(jsonData))
default:
log.Printf("Unsupported job provider: %v", cmd.Provider)
return errors.New("unsupported job provider")
}

View file

@ -0,0 +1,34 @@
package job
import (
"github.com/jobs-scraper/internal/dto"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
)
type GetJobs struct {
jobRepo *repo.JobRepository
}
func NewGetJobsHandler(jobRepo *repo.JobRepository) *GetJobs {
return &GetJobs{
jobRepo: jobRepo,
}
}
func (cj *GetJobs) Handle(query ports.GetJobQuery) ([]dto.JobDTO, int, error) {
filter := repo.JobFilter{
Location: query.Location,
Keywords: query.Keywords,
Provider: query.Provider,
PageSize: query.PageSize,
PageNumber: query.PageNumber,
}
results, totalCount, err := cj.jobRepo.GetAllJobs(filter)
if err != nil {
return nil, 0, err
}
return dto.JobsFromDomain(results), totalCount, nil
}

View file

@ -7,27 +7,31 @@ import (
"net/http"
"strconv"
"github.com/gorilla/mux"
"github.com/gorilla/schema"
"github.com/jobs-scraper/internal/utils"
"github.com/jobs-scraper/libs/ports"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/internal/pkg/utils"
"github.com/gorilla/mux"
)
// JobHandler handles HTTP requests related to jobs
type JobHandler struct {
jobCommands *ports.JobCommands
jobAnalysisResultRepo *repo.JobAnalysisResultRepository
jobQueries *ports.JobQueries
}
// NewJobHandler creates a new instance of JobHandler
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository) *JobHandler {
func NewJobHandler(jobCommands *ports.JobCommands, jobAnalysisResultRepo *repo.JobAnalysisResultRepository, jobQueries *ports.JobQueries) *JobHandler {
return &JobHandler{
jobCommands: jobCommands,
jobAnalysisResultRepo: jobAnalysisResultRepo,
jobQueries: jobQueries,
}
}
var decoder = schema.NewDecoder()
// RegisterRoutes registers all routes to the router
func (h *JobHandler) RegisterRoutes(router *mux.Router) {
jobs := router.PathPrefix("/jobs").Subrouter()
@ -38,6 +42,8 @@ func (h *JobHandler) RegisterRoutes(router *mux.Router) {
jobs.HandleFunc("/analysis", utils.Make(h.GetAllAnalysisResults)).Methods("GET")
jobs.HandleFunc("/{id}/analysis", utils.Make(h.GetJobAnalysisResult)).Methods("GET")
jobs.HandleFunc("/analysis/top-matches", utils.Make(h.GetTopMatches)).Methods("GET")
jobs.HandleFunc("", utils.Make(h.GetJobs)).Methods("GET")
jobs.HandleFunc("/apply/{id}", utils.Make(h.ApplyForJob)).Methods("POST")
}
// CreateJob handles the creation of a new job
@ -68,6 +74,100 @@ func (h *JobHandler) CreateJob(w http.ResponseWriter, r *http.Request) error {
return nil
}
// Apply applies for a job by id
// @Summary applies for a job
// @Description Applies for a job with the provided id
// @Tags jobs
// @Param id query string false "Job ID"
// @Accept json
// @Produce json
// @Success 201 {object} map[string]string "Successfully applied for job"
// @Failure 400 {object} map[string]string "Invalid request data"
// @Failure 500 {object} map[string]string "Internal server error"
// @Router /jobs/apply/{id} [post]
func (h *JobHandler) ApplyForJob(w http.ResponseWriter, r *http.Request) error {
jobId := r.URL.Query().Get("id")
if jobId == "" {
slog.Error("Job ID is required")
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("job ID is required"))
}
jobIdInt, err := strconv.Atoi(jobId)
if err != nil {
slog.Error("Invalid Job ID", "error", err)
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid job ID"))
}
err = h.jobCommands.ApplyForJob.Handle(jobIdInt)
if err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusInternalServerError, fmt.Errorf("Internal server error"))
}
utils.WriteJSON(w, http.StatusCreated, map[string]string{"message": "Success"})
return nil
}
// GetJobs handles the retrieval of all jobs
// @Summary Gets all jobs
// @Description Gets all jobs with the provided specifications
// @Tags jobs
// @Param location query string false "Location filter"
// @Param keywords query string false "Keywords filter (searches title and company)"
// @Param fwt query string false "Full/Part time filter"
// @Param provider query int false "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)" Enums(0,1,2,3,4,5,6,7)
// @Param pageNumber query int false "Page number"
// @Param pageSize query int false "Number of elements"
// @Accept json
// @Produce json
// @Success 200 {object} map[string]any "Successfully retrieved jobs"
// @Failure 400 {object} map[string]any "Invalid request data"
// @Failure 500 {object} map[string]any "Internal server error"
// @Router /jobs [get]
func (h *JobHandler) GetJobs(w http.ResponseWriter, r *http.Request) error {
query := ports.GetJobQuery{
PageNumber: 1,
PageSize: 10,
}
if pageNumberStr := r.URL.Query().Get("pageNumber"); pageNumberStr != "" {
if page, err := strconv.Atoi(pageNumberStr); err == nil && page > 0 {
query.PageNumber = page
}
}
if pageSizeStr := r.URL.Query().Get("pageSize"); pageSizeStr != "" {
if pageSize, err := strconv.Atoi(pageSizeStr); err == nil && pageSize > 0 {
query.PageSize = pageSize
}
}
if err := decoder.Decode(&query, r.URL.Query()); err != nil {
slog.Error("Failed to decode query params", "error", err)
return utils.NewAPIError(http.StatusBadRequest, fmt.Errorf("invalid query parameters: %w", err))
}
result, totalCount, err := h.jobQueries.GetJobs.Handle(query)
if err != nil {
slog.Error(err.Error())
return utils.NewAPIError(http.StatusInternalServerError, err)
}
pagination := utils.NewPageViewModel(totalCount, query.PageNumber, query.PageSize)
utils.WriteJSON(w, http.StatusOK, map[string]any{
"jobs": result,
"pagination": pagination,
})
return nil
}
// GetAllAnalysisResults handles retrieving all job analysis results
// @Summary Get all job analysis results
// @Description Retrieves all job analysis results from the database

View file

@ -13,11 +13,17 @@ import (
func CORSMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Set CORS headers
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With")
origin := r.Header.Get("Origin")
if origin == "" {
origin = "*"
}
w.Header().Set("Access-Control-Allow-Origin", origin)
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Requested-With, X-API-Key")
w.Header().Set("Access-Control-Allow-Credentials", "true")
w.Header().Set("Access-Control-Max-Age", "86400")
w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Range")
// Handle preflight OPTIONS requests
if r.Method == "OPTIONS" {

View file

@ -16,6 +16,90 @@ const docTemplate = `{
"basePath": "{{.BasePath}}",
"paths": {
"/jobs": {
"get": {
"description": "Gets all jobs with the provided specifications",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Gets all jobs",
"parameters": [
{
"type": "string",
"description": "Location filter",
"name": "location",
"in": "query"
},
{
"type": "string",
"description": "Keywords filter (searches title and company)",
"name": "keywords",
"in": "query"
},
{
"type": "string",
"description": "Full/Part time filter",
"name": "fwt",
"in": "query"
},
{
"enum": [
0,
1,
2,
3,
4,
5,
6,
7
],
"type": "integer",
"description": "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)",
"name": "provider",
"in": "query"
},
{
"type": "integer",
"description": "Page number",
"name": "pageNumber",
"in": "query"
},
{
"type": "integer",
"description": "Number of elements",
"name": "pageSize",
"in": "query"
}
],
"responses": {
"200": {
"description": "Successfully retrieved jobs",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
},
"post": {
"description": "Creates a new job with the provided specifications",
"consumes": [
@ -151,6 +235,58 @@ const docTemplate = `{
}
}
},
"/jobs/apply/{id}": {
"post": {
"description": "Applies for a job with the provided id",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "applies for a job",
"parameters": [
{
"type": "string",
"description": "Job ID",
"name": "id",
"in": "query"
}
],
"responses": {
"201": {
"description": "Successfully applied for job",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/{id}/analysis": {
"get": {
"description": "Retrieves the analysis result for a specific job",
@ -254,7 +390,9 @@ const docTemplate = `{
2,
3,
4,
5
5,
6,
7
],
"x-enum-varnames": [
"LinkedIn",
@ -262,7 +400,9 @@ const docTemplate = `{
"Glassdoor",
"Bayt",
"TokyoDev",
"JapanDev"
"JapanDev",
"Google",
"HiringCafe"
]
},
"ports.CreateJobCommand": {

View file

@ -9,6 +9,90 @@
"basePath": "/",
"paths": {
"/jobs": {
"get": {
"description": "Gets all jobs with the provided specifications",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "Gets all jobs",
"parameters": [
{
"type": "string",
"description": "Location filter",
"name": "location",
"in": "query"
},
{
"type": "string",
"description": "Keywords filter (searches title and company)",
"name": "keywords",
"in": "query"
},
{
"type": "string",
"description": "Full/Part time filter",
"name": "fwt",
"in": "query"
},
{
"enum": [
0,
1,
2,
3,
4,
5,
6,
7
],
"type": "integer",
"description": "Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4, JapanDev=5, Google=6, HiringCafe=7)",
"name": "provider",
"in": "query"
},
{
"type": "integer",
"description": "Page number",
"name": "pageNumber",
"in": "query"
},
{
"type": "integer",
"description": "Number of elements",
"name": "pageSize",
"in": "query"
}
],
"responses": {
"200": {
"description": "Successfully retrieved jobs",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": true
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": true
}
}
}
},
"post": {
"description": "Creates a new job with the provided specifications",
"consumes": [
@ -144,6 +228,58 @@
}
}
},
"/jobs/apply/{id}": {
"post": {
"description": "Applies for a job with the provided id",
"consumes": [
"application/json"
],
"produces": [
"application/json"
],
"tags": [
"jobs"
],
"summary": "applies for a job",
"parameters": [
{
"type": "string",
"description": "Job ID",
"name": "id",
"in": "query"
}
],
"responses": {
"201": {
"description": "Successfully applied for job",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"400": {
"description": "Invalid request data",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
},
"500": {
"description": "Internal server error",
"schema": {
"type": "object",
"additionalProperties": {
"type": "string"
}
}
}
}
}
},
"/jobs/{id}/analysis": {
"get": {
"description": "Retrieves the analysis result for a specific job",
@ -247,7 +383,9 @@
2,
3,
4,
5
5,
6,
7
],
"x-enum-varnames": [
"LinkedIn",
@ -255,7 +393,9 @@
"Glassdoor",
"Bayt",
"TokyoDev",
"JapanDev"
"JapanDev",
"Google",
"HiringCafe"
]
},
"ports.CreateJobCommand": {

View file

@ -33,6 +33,8 @@ definitions:
- 3
- 4
- 5
- 6
- 7
type: integer
x-enum-varnames:
- LinkedIn
@ -41,6 +43,8 @@ definitions:
- Bayt
- TokyoDev
- JapanDev
- Google
- HiringCafe
ports.CreateJobCommand:
properties:
fwt:
@ -61,6 +65,66 @@ info:
version: "1.0"
paths:
/jobs:
get:
consumes:
- application/json
description: Gets all jobs with the provided specifications
parameters:
- description: Location filter
in: query
name: location
type: string
- description: Keywords filter (searches title and company)
in: query
name: keywords
type: string
- description: Full/Part time filter
in: query
name: fwt
type: string
- description: Job provider (LinkedIn=0, Indeed=1, Glassdoor=2, Bayt=3, TokyoDev=4,
JapanDev=5, Google=6, HiringCafe=7)
enum:
- 0
- 1
- 2
- 3
- 4
- 5
- 6
- 7
in: query
name: provider
type: integer
- description: Page number
in: query
name: pageNumber
type: integer
- description: Number of elements
in: query
name: pageSize
type: integer
produces:
- application/json
responses:
"200":
description: Successfully retrieved jobs
schema:
additionalProperties: true
type: object
"400":
description: Invalid request data
schema:
additionalProperties: true
type: object
"500":
description: Internal server error
schema:
additionalProperties: true
type: object
summary: Gets all jobs
tags:
- jobs
post:
consumes:
- application/json
@ -186,4 +250,38 @@ paths:
summary: Get top matching jobs
tags:
- jobs
/jobs/apply/{id}:
post:
consumes:
- application/json
description: Applies for a job with the provided id
parameters:
- description: Job ID
in: query
name: id
type: string
produces:
- application/json
responses:
"201":
description: Successfully applied for job
schema:
additionalProperties:
type: string
type: object
"400":
description: Invalid request data
schema:
additionalProperties:
type: string
type: object
"500":
description: Internal server error
schema:
additionalProperties:
type: string
type: object
summary: applies for a job
tags:
- jobs
swagger: "2.0"

View file

@ -0,0 +1,17 @@
DB_HOST=localhost
DB_PORT=5432
DB_USER=your_db_user
DB_PASSWORD=your_db_password
DB_NAME=linkedin_jobs
DB_SSLMODE=disable
SERVER_PORT=8080
SERVER_HOST=localhost
OPENAI_API_KEY=sk-GtdLX9YBOCsBEgBL4rBONA
OPENAI_BASE_URL=https://hubai.loe.gg/v1
OPENAI_MODEL=gpt-4o-mini
CV_AI_MODEL=alibaba/tongyi-deepresearch-30b-a3b:free
OPENROUTER_API_KEY=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
GEMINI_API_KEY=AIzaSyBkGcsPdqhIrjg-wbLOuhAYmjOCt3A_zxM

Binary file not shown.

View file

@ -0,0 +1,65 @@
# Ziad Elshimy
## Frontend Developer
Results-driven Fullstack Developer with extensive experience in building responsive and visually appealing web applications. Seeking to advance to a Senior Fullstack Developer position, where I can utilize my technical expertise and leadership skills to guide a team of developers in the successful execution of complex projects.
`ziadshimy7@gmail.com`
`01223381370`
`Alexandria`
[LinkedIn](https://www.linkedin.com/in/ziad-elshimy-1b31601b6)
[GitHub](https://github.com/ziadshimy7)
---
## Work Experience
### kari - Fullstack Developer
**Jun 2022 - current**
- Led a project to advance the company's online shopping platform, attracting more daily visitors and boosting conversion rates.
- Led the development of multiple internal projects.
- Mentored 2 junior developers, culminating in both earning promotions within 8 months due to enhanced skills.
- Led task planning and coordinated project timelines, reducing overall project duration by 20%.
### Callibri - Frontend Developer
**Jan 2022 - Mar 2022**
- Collaborated with a designer to develop a user-friendly website interface, increasing user engagement.
- Collaborated closely with senior developers to manage a complex design project, increasing efficiency.
### EJADA - Frontend Developer
**Mar 2021 - Dec 2021**
- Troubleshooted the website's problems and stay up to date on technology.
---
## Education
### Ural Federal University - Bachelor's degree, Computer and Information Sciences, General
**Jan 2017 - Dec 2021**
---
## Skills
- HTML5
- Cascading Style Sheets (CSS)
- SCSS
- Tailwind css
- JavaScript
- TypeScript
- React.js
- Redux.js
- Next.js
- Node.js
- SSR
- Webpack
- Vite
- Go (Golang)
- Microservices
- docker
- docker-compose
---
## Certifications
- CCNA
- The Complete JavaScript Course 2023- Udemy
- React - The Complete Guide - Udemy

287
services/job-applier-ts/package-lock.json generated Normal file
View file

@ -0,0 +1,287 @@
{
"name": "job-applier",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "job-applier",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
"dotenv": "^17.2.3"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
},
"../../libs/browser-automation": {
"name": "@jobs-scraper/browser-automation",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"dotenv": "^17.2.3",
"puppeteer-core": "^24.25.0"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
},
"../../libs/rabbitmq-ts": {
"name": "@jobs-scraper/rabbitmq-ts",
"version": "1.0.0",
"license": "ISC",
"dependencies": {
"amqplib": "^0.10.9"
},
"devDependencies": {
"@types/amqplib": "^0.10.8",
"typescript": "^5.9.3"
}
},
"node_modules/@cspotcode/source-map-support": {
"version": "0.8.1",
"resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz",
"integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/trace-mapping": "0.3.9"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@jobs-scraper/browser-automation": {
"resolved": "../../libs/browser-automation",
"link": true
},
"node_modules/@jobs-scraper/rabbitmq-ts": {
"resolved": "../../libs/rabbitmq-ts",
"link": true
},
"node_modules/@jridgewell/resolve-uri": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz",
"integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6.0.0"
}
},
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.5.5",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz",
"integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==",
"dev": true,
"license": "MIT"
},
"node_modules/@jridgewell/trace-mapping": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz",
"integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@jridgewell/resolve-uri": "^3.0.3",
"@jridgewell/sourcemap-codec": "^1.4.10"
}
},
"node_modules/@tsconfig/node10": {
"version": "1.0.12",
"resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz",
"integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node12": {
"version": "1.0.11",
"resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz",
"integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node14": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz",
"integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==",
"dev": true,
"license": "MIT"
},
"node_modules/@tsconfig/node16": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz",
"integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "24.10.4",
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.4.tgz",
"integrity": "sha512-vnDVpYPMzs4wunl27jHrfmwojOGKya0xyM3sH+UE5iv5uPS6vX7UIoh6m+vQc5LGBq52HBKPIn/zcSZVzeDEZg==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~7.16.0"
}
},
"node_modules/acorn": {
"version": "8.15.0",
"resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz",
"integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==",
"dev": true,
"license": "MIT",
"bin": {
"acorn": "bin/acorn"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/acorn-walk": {
"version": "8.3.4",
"resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz",
"integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==",
"dev": true,
"license": "MIT",
"dependencies": {
"acorn": "^8.11.0"
},
"engines": {
"node": ">=0.4.0"
}
},
"node_modules/arg": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
"integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
"dev": true,
"license": "MIT"
},
"node_modules/create-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz",
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"dev": true,
"license": "MIT"
},
"node_modules/diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
"dev": true,
"license": "BSD-3-Clause",
"engines": {
"node": ">=0.3.1"
}
},
"node_modules/dotenv": {
"version": "17.2.3",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
"integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/make-error": {
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
"integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
"dev": true,
"license": "ISC"
},
"node_modules/ts-node": {
"version": "10.9.2",
"resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz",
"integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@cspotcode/source-map-support": "^0.8.0",
"@tsconfig/node10": "^1.0.7",
"@tsconfig/node12": "^1.0.7",
"@tsconfig/node14": "^1.0.0",
"@tsconfig/node16": "^1.0.2",
"acorn": "^8.4.1",
"acorn-walk": "^8.1.1",
"arg": "^4.1.0",
"create-require": "^1.1.0",
"diff": "^4.0.1",
"make-error": "^1.1.1",
"v8-compile-cache-lib": "^3.0.1",
"yn": "3.1.1"
},
"bin": {
"ts-node": "dist/bin.js",
"ts-node-cwd": "dist/bin-cwd.js",
"ts-node-esm": "dist/bin-esm.js",
"ts-node-script": "dist/bin-script.js",
"ts-node-transpile-only": "dist/bin-transpile.js",
"ts-script": "dist/bin-script-deprecated.js"
},
"peerDependencies": {
"@swc/core": ">=1.2.50",
"@swc/wasm": ">=1.2.50",
"@types/node": "*",
"typescript": ">=2.7"
},
"peerDependenciesMeta": {
"@swc/core": {
"optional": true
},
"@swc/wasm": {
"optional": true
}
}
},
"node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
"dev": true,
"license": "Apache-2.0",
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
},
"engines": {
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "7.16.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
"dev": true,
"license": "MIT"
},
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
"integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==",
"dev": true,
"license": "MIT"
},
"node_modules/yn": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
"integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
}
}
}

View file

@ -0,0 +1,25 @@
{
"name": "job-applier",
"version": "1.0.0",
"description": "Job application automation service using Puppeteer",
"main": "dist/index.js",
"scripts": {
"build": "tsc",
"start": "node dist/index.js",
"dev": "ts-node src/index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": ["job", "automation", "puppeteer", "application"],
"author": "",
"license": "ISC",
"dependencies": {
"@jobs-scraper/browser-automation": "file:../../libs/browser-automation",
"@jobs-scraper/rabbitmq-ts": "file:../../libs/rabbitmq-ts",
"dotenv": "^17.2.3"
},
"devDependencies": {
"@types/node": "^24.8.1",
"ts-node": "^10.9.2",
"typescript": "^5.9.3"
}
}

View file

@ -0,0 +1,347 @@
import fs from "fs";
import path from "path";
import {
Browser,
randomDelay,
OpenRouterService,
FormField,
getMockFormData,
FormExtractionResult,
} from "@jobs-scraper/browser-automation";
/**
* Job represents a job posting
*/
export interface Job {
id: number;
title: string;
company: string;
companyLink: string;
location: string;
jobLink: string;
provider: number;
jobPostTime: string;
}
export interface JobWithDescription {
job: Job;
jobDescription: JobDescription;
}
export interface JobDescription {
description: string;
criteria: Record<string, string>;
}
/**
* Read CV from file
*/
export function readCV(cvPath: string = "cv.txt"): string {
const absolutePath = path.isAbsolute(cvPath)
? cvPath
: path.join(process.cwd(), cvPath);
return fs.readFileSync(absolutePath, "utf-8");
}
/**
* Read CV from file
*/
export function getCVPath(cvPath: string = "CV.pdf"): string {
return path.isAbsolute(cvPath) ? cvPath : path.join(process.cwd(), cvPath);
}
/**
* Handle text, email, tel, url, search, password inputs
*/
async function handleTextInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.clear(field.selector);
await browser.type(field.selector, "");
await browser.type(field.selector, field.value);
}
/**
* Handle textarea inputs
*/
async function handleTextarea(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.clear(field.selector);
await browser.type(field.selector, field.value);
}
/**
* Handle number and range inputs
*/
async function handleNumberInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
/**
* Handle date, datetime-local, time, month, week inputs
*/
async function handleDateInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
/**
* Handle select dropdown inputs
*/
async function handleSelect(browser: Browser, field: FormField): Promise<void> {
if (field.value === "") {
return;
}
await browser.select(field.selector, field.value);
}
/**
* Handle radio button inputs
*/
async function handleRadio(browser: Browser, field: FormField): Promise<void> {
if (field.value === "") {
return;
}
await browser.setRadio(field.selector);
}
/**
* Handle checkbox inputs
*/
async function handleCheckbox(
browser: Browser,
field: FormField
): Promise<void> {
const checked =
field.value === "true" || field.value === "1" || field.value === "yes";
await browser.setCheckbox(field.selector, checked);
}
/**
* Handle file input fields
*/
async function handleFileUpload(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.uploadFile(field.selector, field.value);
}
/**
* Handle color picker inputs
*/
async function handleColorInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
/**
* Handle hidden input fields
*/
async function handleHiddenInput(
browser: Browser,
field: FormField
): Promise<void> {
if (field.value === "") {
return;
}
await browser.setValue(field.selector, field.value);
}
export interface WorkOptions {
useMockData?: boolean;
openRouterApiKey?: string;
openRouterModel?: string;
}
/**
* Main work function that fills out a job application form
*/
export async function work(
browser: Browser,
job: JobWithDescription,
options: WorkOptions = {}
): Promise<void> {
const { openRouterApiKey = "", openRouterModel = "", useMockData } = options;
try {
await browser.init();
await browser.navigate(`${job.job.jobLink}apply`);
await browser.evaluate<boolean>(
`
(() => {
const acceptBtn = document.querySelector('[data-ui="cookie-consent-accept"]');
if (acceptBtn) {
acceptBtn.click();
return true;
}
return false;
})()
`
);
let result: FormExtractionResult;
const formHtml = await browser.getOuterHTML("form");
// Read the CV
const cv = readCV();
const cvPDFPath = getCVPath();
console.log({ cvPDFPath });
// let resumeButtonClicked: boolean = false;
// while (!resumeButtonClicked) {
// resumeButtonClicked = await browser.evaluate<boolean>(
// `
// (() => {
// const buttons = document.querySelectorAll('button');
// const resumeBtn = Array.from(buttons).find(btn =>
// btn.innerText.toLowerCase().includes('resume') ||
// btn.innerText.toLowerCase().includes('import cv')
// );
// if (resumeBtn) {
// resumeBtn.click();
// return true;
// }
// return false;
// })()
// `
// );
// }
// console.log({ resumeButtonClicked });
// await browser.uploadFile("input#file-upload", cvPDFPath);
await randomDelay(5000, 7000);
// await browser.uploadFile("input#resume-upload-input", cvPDFPath);
if (!useMockData) {
const openRouterService = new OpenRouterService({
apiKey: openRouterApiKey,
model: openRouterModel,
});
console.log("Calling OpenRouter service...");
result = await openRouterService.applyForJob(formHtml, cv, "");
console.log("Form extraction result:", result);
} else {
result = getMockFormData();
}
console.log({ result });
if (result.fields.length === 0) {
console.log("No fields extracted, using mock data");
} else {
for (const field of result.fields) {
await randomDelay(1000, 2000);
await processField(browser, field);
}
}
await randomDelay(160000, 180000);
} finally {
await browser.close();
}
}
/**
* Process a single form field
*/
async function processField(browser: Browser, field: FormField): Promise<void> {
if (field.selector === "") {
return;
}
console.log("Processing field:", field);
try {
switch (field.field_type) {
case "text":
case "email":
case "tel":
case "url":
case "search":
case "password":
await handleTextInput(browser, field);
break;
case "textarea":
await handleTextarea(browser, field);
break;
case "number":
case "range":
await handleNumberInput(browser, field);
break;
case "date":
case "datetime-local":
case "time":
case "month":
case "week":
await handleDateInput(browser, field);
break;
case "select":
await handleSelect(browser, field);
break;
case "radio":
await handleRadio(browser, field);
break;
case "checkbox":
await handleCheckbox(browser, field);
break;
case "file":
await handleFileUpload(browser, field);
break;
case "color":
await handleColorInput(browser, field);
break;
case "hidden":
await handleHiddenInput(browser, field);
break;
case "button":
case "submit":
case "reset":
// Skip buttons - they're handled separately
return;
default:
return;
}
} catch (error) {
console.error(`Error processing field ${field.field_name}:`, error);
// Continue with other fields
}
await randomDelay(1500, 2000);
}

View file

@ -0,0 +1,91 @@
import * as dotenv from "dotenv";
import { RabbitMQClient, RabbitMQConfig } from "@jobs-scraper/rabbitmq-ts";
import { work, JobWithDescription } from "./applier/work";
import { Browser } from "@jobs-scraper/browser-automation";
if (dotenv.config({ path: ".local.env" }).error) {
console.log("No .local.env file found, trying .env");
if (dotenv.config().error) {
console.log("No .env file found, using system environment variables");
}
}
const JOB_APPLIER_QUEUE = "job.applier";
const SCRAPER_EXCHANGE = "scraper_exchange";
let browser = new Browser({ port: 9223 });
let rabbitMQClient: RabbitMQClient | null = null;
function createRabbitMQConfig(): RabbitMQConfig {
return {
url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/",
queueName: JOB_APPLIER_QUEUE,
exchangeName: SCRAPER_EXCHANGE,
exchangeType: "topic",
durable: true,
};
}
async function processJob(job: JobWithDescription): Promise<void> {
console.log(
`Processing job application for: ${job.job.title} at ${job.job.company}`
);
await work(browser, job, {
useMockData: true,
openRouterApiKey: process.env.OPENROUTER_API_KEY,
openRouterModel: process.env.OPENROUTER_MODEL,
});
console.log("Job application completed successfully!");
}
async function main() {
console.log("Starting Job Applier service...");
try {
await browser.init();
const config = createRabbitMQConfig();
rabbitMQClient = new RabbitMQClient(config);
await rabbitMQClient.connect();
await rabbitMQClient.subscribe<JobWithDescription>(processJob);
} catch (error) {
console.error("Error starting service:", error);
await cleanup();
process.exit(1);
}
}
async function cleanup(): Promise<void> {
try {
if (rabbitMQClient) {
await rabbitMQClient.close();
rabbitMQClient = null;
}
await browser.close();
console.log("Cleanup completed");
} catch (error) {
console.error("Error during cleanup:", error);
}
}
// Handle graceful shutdown
process.on("SIGINT", async () => {
console.log("Received SIGINT signal");
await cleanup();
process.exit(0);
});
process.on("SIGTERM", async () => {
console.log("Received SIGTERM signal");
await cleanup();
process.exit(0);
});
main().catch(async (error) => {
console.error("Fatal error:", error);
await cleanup();
process.exit(1);
});

View file

@ -0,0 +1,34 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "commonjs",
"lib": ["ES2020", "DOM"],
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"removeComments": true,
"noImplicitAny": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"noUnusedLocals": false,
"noUnusedParameters": false,
"exactOptionalPropertyTypes": false,
"noImplicitOverride": true,
"noPropertyAccessFromIndexSignature": false,
"noUncheckedIndexedAccess": false
},
"include": [
"src/**/*.ts"
],
"exclude": [
"node_modules",
"dist"
]
}

View file

@ -107,21 +107,24 @@ async function scrapeGlassdoorJobs(searchQuery: SearchQuery): Promise<void> {
});
await randomDelay(1500, 2500);
await page.goto("https://www.google.com/search?q=site:boards.greenhouse.io+(Software+OR+Backend+OR+Full-Stack)+Engineer", {
waitUntil: ["networkidle2", "domcontentloaded"],
timeout: 60000,
});
// await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
// const jobs = await browserService.extractJobListings(searchQuery.location);
await browserService.searchJobs(searchQuery.keywords, searchQuery.location);
const jobs = await browserService.extractJobListings(searchQuery.location);
// if (jobs.length > 0) {
// await saveJobs(jobs);
// await saveJobDescriptions(jobs);
if (jobs.length > 0) {
await saveJobs(jobs);
await saveJobDescriptions(jobs);
console.log(
`saved ${jobs.length} jobs from Glassdoor for query:`,
searchQuery
);
} else {
console.log("No jobs found for this search query");
}
// console.log(
// `saved ${jobs.length} jobs from Glassdoor for query:`,
// searchQuery
// );
// } else {
// console.log("No jobs found for this search query");
// }
await randomDelay(1500, 2500);
} catch (error) {

View file

@ -17,10 +17,7 @@ export const saveJobs = async (jobs: Job[]) => {
job_link: job.jobLink,
provider: job.provider,
status: job.status,
job_timestamp:
job.jobPostTime instanceof Date && !isNaN(job.jobPostTime.getTime())
? job.jobPostTime
: new Date(),
job_timestamp: job.jobPostTime,
}));
const result = await sql`

View file

@ -0,0 +1,20 @@
deploy_scraper_google:
stage: deploy
image: alpine:3.20
environment:
name: production
url: https://jobs-scraper.ai-assistant-bot.xyz
rules:
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
when: manual
allow_failure: false
before_script:
- apk add --no-cache openssh-client rsync
- mkdir -p ~/.ssh
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' > ~/.ssh/id_rsa
- chmod 600 ~/.ssh/id_rsa
- ssh-keyscan -H "$DEPLOY_HOST" >> ~/.ssh/known_hosts
script:
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "mkdir -p /opt/apps/jobs-scraper"
- rsync -az --delete --exclude='.git' --exclude='.env' --exclude='.env.*' --exclude='.local.env' ./ "$DEPLOY_USER@$DEPLOY_HOST:/opt/apps/jobs-scraper/"
- ssh "$DEPLOY_USER@$DEPLOY_HOST" "cd /opt/apps/jobs-scraper/services/scraper-google && docker compose up -d --build"

View file

@ -1,19 +1,15 @@
package analyzer
import (
"fmt"
"os"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/pkg/openrouter"
// "github.com/jobs-scraper/internal/pkg/openai"
// "github.com/jobs-scraper/internal/pkg/openrouter"
"github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/internal/openrouter"
)
func GetJobDescription(htmlContent string) (*domain.JobDescription, error) {
openRouterApiKey := os.Getenv("OPENROUTER_API_KEY")
openRouterModel := os.Getenv("OPENROUTER_MODEL")
fmt.Println("key", openRouterApiKey)
client := openrouter.NewOpenRouterService(openRouterModel, openRouterApiKey)
result, err := client.CreateJobDescription(htmlContent)

View file

@ -4,26 +4,18 @@ go 1.24.0
require (
github.com/PuerkitoBio/goquery v1.10.3
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d
github.com/chromedp/chromedp v0.14.2
github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0
github.com/jobs-scraper/internal/infrastructure v0.0.0
github.com/joho/godotenv v1.5.1
github.com/levmv/sked v0.2.1
)
require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/chromedp/sysutil v1.1.0 // indirect
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
github.com/gobwas/pool v0.2.1 // indirect
github.com/gobwas/ws v1.4.0 // indirect
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/lib/pq v1.10.9 // indirect
golang.org/x/net v0.44.0 // indirect
golang.org/x/sys v0.36.0 // indirect
)
replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure
replace github.com/jobs-scraper/internal/infrastructure => ../../internal/infrastructure

View file

@ -2,12 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU=
github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k=
github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM=
github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo=
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -17,23 +11,13 @@ github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjY
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs=
github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
github.com/levmv/sked v0.2.1 h1:XMdLkbuajPM5LSzoBvNzalj6oYHkEfeA7/VPvkIHqs0=
github.com/levmv/sked v0.2.1/go.mod h1:w9pFUIpZLPu0Jr2+2JqJtaEIHmwXcvjPHWm6pvbqHVQ=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
@ -42,8 +26,6 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
@ -53,7 +35,6 @@ go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View file

@ -2,18 +2,14 @@ package main
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"os/exec"
"os/signal"
"runtime"
"syscall"
"time"
"github.com/jobs-scraper/internal/pkg/domain"
"github.com/jobs-scraper/internal/browser"
"github.com/jobs-scraper/internal/domain"
"github.com/jobs-scraper/libs/repo"
"github.com/jobs-scraper/services/scraper-google/analyzer"
si "github.com/jobs-scraper/services/scraper-google/setup-infrastructure"
@ -40,31 +36,33 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
// Launch Chrome once and share across all goroutines
wsURL, chromeCmd, err := launchChrome()
browser, err := browser.NewBrowser(ctx)
if err != nil {
log.Fatalf("Failed to launch Chrome: %v", err)
}
defer chromeCmd.Process.Kill()
log.Printf("Chrome launched with WebSocket URL: %s", wsURL)
defer browser.Close()
log.Printf("Chrome launched with WebSocket URL: %s", browser.Instance.WSURL)
sched := sked.New(ctx)
googleLinkStream := make(chan domain.GoogleLink, 100)
// pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
pages := []int{100, 110, 120, 130}
pages := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}
log.Println("Starting scraper with scheduler...")
for _, page := range pages {
go func() {
log.Printf("Running initial scrape for page offset: %d, inital run", page)
Work(ctx, wsURL)(page, googleLinkStream)
Work(ctx, browser)(page, googleLinkStream)
}()
}
for _, page := range pages {
sched.Schedule(func(ctx context.Context) {
log.Printf("Running scheduled scrape for page offset: %d", page)
Work(ctx, wsURL)(page, googleLinkStream)
Work(ctx, browser)(page, googleLinkStream)
}).Every(time.Hour * 24)
}
@ -84,7 +82,7 @@ func main() {
continue
}
htmlRaw, url, err := utils.GetHTMLRaw(ctx, wsURL, googleLink)
htmlRaw, url, err := utils.GetHTMLRaw(ctx, browser, googleLink)
if err != nil {
log.Printf("Error fetching HTML raw for link %s: %v", googleLink.Link, err)
@ -143,75 +141,3 @@ func main() {
cancel()
log.Println("Shutting down scraper...")
}
// launchChrome starts Chrome with remote debugging and returns the WebSocket URL
func launchChrome() (string, *exec.Cmd, error) {
var chromePath string
switch runtime.GOOS {
case "darwin":
chromePath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
case "linux":
chromePath = "google-chrome"
case "windows":
chromePath = "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"
default:
return "", nil, fmt.Errorf("unsupported OS: %s", runtime.GOOS)
}
// Kill any existing Chrome debug instances first
exec.Command("pkill", "-f", "remote-debugging-port=9222").Run()
time.Sleep(500 * time.Millisecond)
cmd := exec.Command(chromePath,
"--remote-debugging-port=9222",
"--no-first-run",
"--no-default-browser-check",
"--disable-background-networking",
"--disable-extensions",
"--disable-sync",
"--disable-translate",
"--metrics-recording-only",
"--safebrowsing-disable-auto-update",
"--user-data-dir=/tmp/chrome-debug-profile",
// "--incognito",
// "--headless",
)
if err := cmd.Start(); err != nil {
return "", nil, fmt.Errorf("failed to start Chrome: %w", err)
}
// Poll the debug endpoint to get the WebSocket URL
var wsURL string
for i := 0; i < 50; i++ {
time.Sleep(200 * time.Millisecond)
resp, err := http.Get("http://127.0.0.1:9222/json/version")
if err != nil {
continue
}
var result struct {
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
}
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
resp.Body.Close()
continue
}
resp.Body.Close()
if result.WebSocketDebuggerURL != "" {
wsURL = result.WebSocketDebuggerURL
break
}
}
if wsURL == "" {
cmd.Process.Kill()
return "", nil, fmt.Errorf("timeout waiting for Chrome WebSocket URL")
}
// Wait for Chrome to be fully ready to accept connections
time.Sleep(1 * time.Second)
return wsURL, cmd, nil
}

View file

@ -4,7 +4,7 @@ import (
"database/sql"
"log"
"github.com/jobs-scraper/internal/pkg/infrastructure"
"github.com/jobs-scraper/internal/infrastructure"
"github.com/joho/godotenv"
)

View file

@ -1,15 +1,28 @@
package utils
import (
"net/url"
"strconv"
"strings"
"time"
)
func BuildUrl(query string, page int) string {
var url strings.Builder
url.WriteString("https://www.google.com/search?q=")
url.WriteString(query)
url.WriteString("&start=")
url.WriteString(strconv.Itoa(page))
return url.String()
params := url.Values{}
params.Add("q", query)
params.Add("start", strconv.Itoa(page))
now := time.Now()
twoWeeksAgo := now.AddDate(0, 0, -14)
startDate := twoWeeksAgo.Format("01/02/2006")
endDate := now.Format("01/02/2006")
tbs := "cdr:1,cd_min:" + startDate + ",cd_max:" + endDate
params.Add("tbs", tbs)
// Optional language bias
// params.Add("hl", "en")
// params.Add("lr", "lang_en")
return "https://www.google.com/search?" + params.Encode()
}

Some files were not shown because too many files have changed in this diff Show more