diff --git a/go.mod b/go.mod index bde6a6a..8ba1e95 100644 --- a/go.mod +++ b/go.mod @@ -3,3 +3,17 @@ module github.com/jobs-scraper go 1.24.0 toolchain go1.24.7 + +require ( + github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d + github.com/chromedp/chromedp v0.14.2 +) + +require ( + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect + golang.org/x/sys v0.36.0 // indirect +) diff --git a/go.sum b/go.sum index e69de29..2e91dff 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU= +github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= diff --git a/go.work b/go.work index 9435af1..998a0ea 100644 --- a/go.work +++ b/go.work @@ -14,6 +14,5 @@ use ( ./libs/server ./services/api ./services/scraper-google - ./services/scraper-hiringcafe ./services/scraper-linkedin ) diff --git a/go.work.sum b/go.work.sum index af7ee3a..bf72b7c 100644 --- a/go.work.sum +++ b/go.work.sum @@ -43,6 +43,9 @@ github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= +github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= github.com/cncf/xds/go v0.0.0-20240723142845-024c85f92f20/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM= @@ -57,9 +60,13 @@ github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9 github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= +github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= @@ -102,8 +109,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -121,6 +128,7 @@ github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86w github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= @@ -156,7 +164,7 @@ golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -169,7 +177,6 @@ golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw= @@ -183,7 +190,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240827150818-7e3bb234dfed/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg= diff --git a/services/api/go.mod b/services/api/go.mod index 872ff05..b586643 100644 --- a/services/api/go.mod +++ b/services/api/go.mod @@ -4,6 +4,7 @@ go 1.24.0 require ( github.com/gorilla/mux v1.8.1 + github.com/gorilla/schema v1.4.1 github.com/jobs-scraper/internal/pkg/domain v0.0.0 github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0-00010101000000-000000000000 github.com/jobs-scraper/internal/pkg/utils v0.0.0 @@ -22,7 +23,6 @@ require ( github.com/go-openapi/spec v0.20.6 // indirect github.com/go-openapi/swag v0.19.15 // indirect github.com/golang-migrate/migrate/v4 v4.19.0 // indirect - github.com/gorilla/schema v1.4.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/services/scraper-glassdoor/queries/save-jobs.ts b/services/scraper-glassdoor/queries/save-jobs.ts index dc88deb..c43eee1 100644 --- a/services/scraper-glassdoor/queries/save-jobs.ts +++ b/services/scraper-glassdoor/queries/save-jobs.ts @@ -17,10 +17,7 @@ export const saveJobs = async (jobs: Job[]) => { job_link: job.jobLink, provider: job.provider, status: job.status, - job_timestamp: - job.jobPostTime instanceof Date && !isNaN(job.jobPostTime.getTime()) - ? job.jobPostTime - : new Date(), + job_timestamp: job.jobPostTime, })); const result = await sql` diff --git a/services/scraper-google/go.mod b/services/scraper-google/go.mod index e143882..644817b 100644 --- a/services/scraper-google/go.mod +++ b/services/scraper-google/go.mod @@ -4,8 +4,6 @@ go 1.24.0 require ( github.com/PuerkitoBio/goquery v1.10.3 - github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d - github.com/chromedp/chromedp v0.14.2 github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 github.com/joho/godotenv v1.5.1 github.com/levmv/sked v0.2.1 @@ -13,17 +11,11 @@ require ( require ( github.com/andybalholm/cascadia v1.3.3 // indirect - github.com/chromedp/sysutil v1.1.0 // indirect - github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect - github.com/gobwas/httphead v0.1.0 // indirect - github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.4.0 // indirect github.com/golang-migrate/migrate/v4 v4.19.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/lib/pq v1.10.9 // indirect golang.org/x/net v0.44.0 // indirect - golang.org/x/sys v0.36.0 // indirect ) replace github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure diff --git a/services/scraper-google/go.sum b/services/scraper-google/go.sum index c5d03ed..6117c4d 100644 --- a/services/scraper-google/go.sum +++ b/services/scraper-google/go.sum @@ -2,12 +2,6 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= -github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU= -github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= -github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= -github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= -github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= -github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -17,23 +11,13 @@ github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjY github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= -github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= -github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/levmv/sked v0.2.1 h1:XMdLkbuajPM5LSzoBvNzalj6oYHkEfeA7/VPvkIHqs0= github.com/levmv/sked v0.2.1/go.mod h1:w9pFUIpZLPu0Jr2+2JqJtaEIHmwXcvjPHWm6pvbqHVQ= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -42,8 +26,6 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -53,7 +35,6 @@ go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/services/scraper-google/work.go b/services/scraper-google/work.go index d768655..bcdb3f1 100644 --- a/services/scraper-google/work.go +++ b/services/scraper-google/work.go @@ -19,11 +19,11 @@ func Work(mainCtx context.Context, b *browser.Browser) func(page int, stream cha } defer tab.Close() - // Random delay before starting to look more human browser.RandomDelay(1000, 3000) // query := "site:boards.greenhouse.io (Software OR Backend OR Full-Stack) Engineer inurl:gh_jid" - query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` + // query := `site:lever.co (Software OR Backend OR Full-Stack) Engineer lang:en` + query := `site:workable.com (Software OR Backend OR Full-Stack) Engineer lang:en` // url := utils.BuildUrl(query, page) err = tab.Navigate("https://linkedin.com") diff --git a/services/scraper-hiringcafe/.env b/services/scraper-hiringcafe/.env new file mode 100644 index 0000000..2739b5b --- /dev/null +++ b/services/scraper-hiringcafe/.env @@ -0,0 +1,12 @@ +DB_HOST=localhost +DB_PORT=5432 +DB_USER=postgres +DB_PASSWORD=password +DB_NAME=linkedin_jobs +DB_SSLMODE=disable + +SERVER_PORT=8080 +SERVER_HOST=localhost + +CV_AI_MODEL=your_ai_model +OPENROUTER_API_KEY=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 \ No newline at end of file diff --git a/services/scraper-hiringcafe/api/client.go b/services/scraper-hiringcafe/api/client.go deleted file mode 100644 index 3bf62da..0000000 --- a/services/scraper-hiringcafe/api/client.go +++ /dev/null @@ -1,245 +0,0 @@ -package api - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "time" -) - -const ( - BaseURL = "https://hiring.cafe" - SearchURL = BaseURL + "/api/search-jobs" - UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" - PageSize = 20 -) - -// Client handles API requests to hiring.cafe -type Client struct { - httpClient *http.Client -} - -// NewClient creates a new API client -func NewClient() *Client { - return &Client{ - httpClient: &http.Client{ - Timeout: 30 * time.Second, - }, - } -} - -// SearchRequest represents the search request payload (based on RSSHub implementation) -type SearchRequest struct { - Size int `json:"size"` - Page int `json:"page"` - SearchState SearchState `json:"searchState"` -} - -// SearchState contains search parameters -type SearchState struct { - SearchQuery string `json:"searchQuery"` - SortBy string `json:"sortBy"` // date, default, compensation_desc, experience_asc -} - -// SearchResponse represents the API response -type SearchResponse struct { - Results []JobResult `json:"results"` - Total int `json:"total"` -} - -// JobResult represents a job result from the API -type JobResult struct { - ID string `json:"id"` - ApplyURL string `json:"apply_url"` - JobInformation JobInformation `json:"job_information"` - ProcessedJobData ProcessedJobData `json:"v5_processed_job_data"` -} - -// JobInformation contains basic job info -type JobInformation struct { - Title string `json:"title"` - Description string `json:"description"` -} - -// ProcessedJobData contains processed job details -type ProcessedJobData struct { - CompanyName string `json:"company_name"` - IsCompensationTransparent bool `json:"is_compensation_transparent"` - YearlyMinCompensation *float64 `json:"yearly_min_compensation"` - YearlyMaxCompensation *float64 `json:"yearly_max_compensation"` - WorkplaceType string `json:"workplace_type"` - RequirementsSummary string `json:"requirements_summary"` - JobCategory string `json:"job_category"` - RoleActivities []string `json:"role_activities"` - FormattedWorkplaceLocation string `json:"formatted_workplace_location"` - EstimatedPublishDateMillis string `json:"estimated_publish_date_millis"` -} - -// Job is a simplified job structure for external use -type Job struct { - ID string - ApplyURL string - Title string - Description string - Company string - CompanyURL string - Location string - Remote bool - Salary string - PostedAt time.Time - DescriptionClean string -} - -// SearchFilters contains user-configurable search filters -type SearchFilters struct { - Query string - Country string - Remote bool - Hybrid bool - Onsite bool - FullTime bool - PartTime bool - Contract bool - Internship bool - EntryLevel bool - MidLevel bool - SeniorLevel bool - DateRangeDays int - MaxPages int -} - -// DefaultFilters returns default search filters -func DefaultFilters() SearchFilters { - return SearchFilters{ - Query: "", - Country: "United States", - Remote: true, - Hybrid: true, - Onsite: true, - FullTime: true, - PartTime: false, - Contract: false, - Internship: false, - EntryLevel: true, - MidLevel: true, - SeniorLevel: false, - DateRangeDays: 30, - MaxPages: 10, - } -} - -// Search performs a job search with the given filters -func (c *Client) Search(filters SearchFilters) ([]Job, error) { - var allJobs []Job - page := 1 - - for { - jobs, totalCount, err := c.searchPage(filters, page) - if err != nil { - return nil, fmt.Errorf("failed to search page %d: %w", page, err) - } - - allJobs = append(allJobs, jobs...) - - // Check if we've fetched all jobs or reached max pages - if len(allJobs) >= totalCount || page >= filters.MaxPages || len(jobs) == 0 { - break - } - - page++ - // Rate limiting - be nice to the API - time.Sleep(500 * time.Millisecond) - } - - return allJobs, nil -} - -// searchPage performs a single page search using the correct API format -func (c *Client) searchPage(filters SearchFilters, page int) ([]Job, int, error) { - reqBody := SearchRequest{ - Size: PageSize, - Page: page - 1, // API uses 0-based indexing - SearchState: SearchState{ - SearchQuery: filters.Query, - SortBy: "date", - }, - } - - jsonBody, err := json.Marshal(reqBody) - if err != nil { - return nil, 0, fmt.Errorf("failed to marshal request: %w", err) - } - - log.Printf("Sending request to %s with body: %s", SearchURL, string(jsonBody)) - - req, err := http.NewRequest("POST", SearchURL, bytes.NewBuffer(jsonBody)) - if err != nil { - return nil, 0, fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", UserAgent) - req.Header.Set("Accept", "application/json") - req.Header.Set("Origin", BaseURL) - req.Header.Set("Referer", BaseURL+"/") - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, 0, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, 0, fmt.Errorf("failed to read response: %w", err) - } - - if resp.StatusCode != http.StatusOK { - return nil, 0, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body)) - } - - var searchResp SearchResponse - if err := json.Unmarshal(body, &searchResp); err != nil { - return nil, 0, fmt.Errorf("failed to unmarshal response: %w", err) - } - - // Convert JobResult to Job - var jobs []Job - for _, result := range searchResp.Results { - job := Job{ - ID: result.ID, - ApplyURL: result.ApplyURL, - Title: result.JobInformation.Title, - Description: result.JobInformation.Description, - Company: result.ProcessedJobData.CompanyName, - Location: result.ProcessedJobData.FormattedWorkplaceLocation, - Remote: result.ProcessedJobData.WorkplaceType == "remote", - DescriptionClean: result.JobInformation.Description, - } - - // Parse salary if available - if result.ProcessedJobData.IsCompensationTransparent && - result.ProcessedJobData.YearlyMinCompensation != nil && - result.ProcessedJobData.YearlyMaxCompensation != nil { - job.Salary = fmt.Sprintf("$%.0f - $%.0f", - *result.ProcessedJobData.YearlyMinCompensation, - *result.ProcessedJobData.YearlyMaxCompensation) - } - - // Parse posted date - if result.ProcessedJobData.EstimatedPublishDateMillis != "" { - // Try to parse as milliseconds timestamp - var millis int64 - if _, err := fmt.Sscanf(result.ProcessedJobData.EstimatedPublishDateMillis, "%d", &millis); err == nil { - job.PostedAt = time.UnixMilli(millis) - } - } - - jobs = append(jobs, job) - } - - return jobs, searchResp.Total, nil -} diff --git a/services/scraper-hiringcafe/browser/browser.go b/services/scraper-hiringcafe/browser/browser.go deleted file mode 100644 index 4891581..0000000 --- a/services/scraper-hiringcafe/browser/browser.go +++ /dev/null @@ -1,583 +0,0 @@ -package browser - -import ( - "context" - "encoding/json" - "fmt" - "log" - "math/rand" - "net" - "net/http" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "time" - - "github.com/chromedp/cdproto/cdp" - "github.com/chromedp/cdproto/page" - "github.com/chromedp/chromedp" -) - -// Config holds browser configuration options -type Config struct { - Headless bool - Timeout time.Duration - UserDataDir string - DebugPort int - UseRealChrome bool // Connect to real Chrome instead of launching automated one - ConnectToExisting bool // Connect to already running Chrome (user must start it manually) -} - -// DefaultConfig returns default browser configuration -func DefaultConfig() Config { - return Config{ - Headless: false, - Timeout: 10 * time.Minute, - UserDataDir: filepath.Join(os.TempDir(), "hiringcafe-chrome-profile"), - DebugPort: 9224, // Use different port to avoid conflicts - UseRealChrome: true, - ConnectToExisting: false, - } -} - -// Browser wraps chromedp browser context with anti-detection features -type Browser struct { - allocCtx context.Context - allocCancel context.CancelFunc - ctx context.Context - cancel context.CancelFunc - config Config - chromeCmd *exec.Cmd -} - -// chromeResponse holds the Chrome DevTools Protocol response -type chromeResponse struct { - WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"` -} - -// ErrCaptchaDetected is returned when bot detection/CAPTCHA is detected -var ErrCaptchaDetected = fmt.Errorf("captcha or bot detection detected - terminating") - -// New creates a new browser instance with anti-detection measures -func New(parentCtx context.Context, config Config) (*Browser, error) { - // Always use the automated real Chrome approach now - return newRealChromeAutomated(parentCtx, config) -} - -// newRealChromeAutomated launches Chrome like a normal user would and connects to it -func newRealChromeAutomated(parentCtx context.Context, config Config) (*Browser, error) { - b := &Browser{config: config} - - // Check if Chrome is already running on the debug port - conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", config.DebugPort), time.Second) - if err == nil { - conn.Close() - log.Printf("Chrome already running on port %d, connecting to it...", config.DebugPort) - } else { - // Launch Chrome with remote debugging - if err := b.launchChromeClean(); err != nil { - return nil, fmt.Errorf("failed to launch Chrome: %w", err) - } - - // Wait for Chrome to be ready - if err := b.waitForChrome(); err != nil { - b.Close() - return nil, fmt.Errorf("chrome not ready: %w", err) - } - } - - // Get WebSocket URL - wsURL, err := b.getWebSocketURL() - if err != nil { - b.Close() - return nil, fmt.Errorf("failed to get WebSocket URL: %w", err) - } - - log.Printf("Connected to Chrome at %s", wsURL) - - // Connect to Chrome via WebSocket - allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL) - ctx, cancel := chromedp.NewContext(allocCtx) - - // Set timeout - ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout) - - b.allocCtx = allocCtx - b.allocCancel = allocCancel - b.ctx = ctx - b.cancel = func() { - timeoutCancel() - cancel() - } - - return b, nil -} - -// launchChromeClean launches Chrome as cleanly as possible (like a normal user) -func (b *Browser) launchChromeClean() error { - chromePath := findChromePath() - if chromePath == "" { - return fmt.Errorf("chrome not found") - } - - // Ensure user data directory exists - if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil { - return fmt.Errorf("failed to create user data dir: %w", err) - } - - // Minimal args - just what's needed for remote debugging - // Avoid any automation-related flags that might be detected - args := []string{ - fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort), - fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir), - "--remote-allow-origins=*", - "--no-first-run", - "--no-default-browser-check", - "--start-maximized", - } - - log.Printf("Launching Chrome: %s", chromePath) - b.chromeCmd = exec.Command(chromePath, args...) - b.chromeCmd.Stdout = nil - b.chromeCmd.Stderr = nil - - if err := b.chromeCmd.Start(); err != nil { - return fmt.Errorf("failed to start Chrome: %w", err) - } - - log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort) - return nil -} - -// newRealChrome launches a real Chrome browser and connects via DevTools Protocol -func newRealChrome(parentCtx context.Context, config Config) (*Browser, error) { - b := &Browser{config: config} - - // Launch Chrome with remote debugging - if err := b.launchChrome(); err != nil { - return nil, fmt.Errorf("failed to launch Chrome: %w", err) - } - - // Wait for Chrome to be ready - if err := b.waitForChrome(); err != nil { - b.Close() - return nil, fmt.Errorf("Chrome not ready: %w", err) - } - - // Get WebSocket URL - wsURL, err := b.getWebSocketURL() - if err != nil { - b.Close() - return nil, fmt.Errorf("failed to get WebSocket URL: %w", err) - } - - log.Printf("Connecting to Chrome at %s", wsURL) - - // Connect to Chrome via WebSocket - allocCtx, allocCancel := chromedp.NewRemoteAllocator(parentCtx, wsURL) - ctx, cancel := chromedp.NewContext(allocCtx) - - // Set timeout - ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout) - - b.allocCtx = allocCtx - b.allocCancel = allocCancel - b.ctx = ctx - b.cancel = func() { - timeoutCancel() - cancel() - } - - // Inject anti-detection scripts - if err := b.injectAntiDetection(); err != nil { - log.Printf("Warning: could not inject anti-detection: %v", err) - } - - return b, nil -} - -// launchChrome launches Chrome with remote debugging enabled -func (b *Browser) launchChrome() error { - chromePath := findChromePath() - if chromePath == "" { - return fmt.Errorf("Chrome not found") - } - - // Ensure user data directory exists - if err := os.MkdirAll(b.config.UserDataDir, 0755); err != nil { - return fmt.Errorf("failed to create user data dir: %w", err) - } - - args := []string{ - fmt.Sprintf("--remote-debugging-port=%d", b.config.DebugPort), - fmt.Sprintf("--user-data-dir=%s", b.config.UserDataDir), - "--remote-allow-origins=*", - "--no-first-run", - "--no-default-browser-check", - "--disable-background-networking", - "--disable-client-side-phishing-detection", - "--disable-default-apps", - "--disable-hang-monitor", - "--disable-popup-blocking", - "--disable-prompt-on-repost", - "--disable-sync", - "--disable-translate", - "--metrics-recording-only", - "--safebrowsing-disable-auto-update", - "--password-store=basic", - "--use-mock-keychain", - } - - log.Printf("Launching Chrome: %s", chromePath) - b.chromeCmd = exec.Command(chromePath, args...) - b.chromeCmd.Stdout = nil - b.chromeCmd.Stderr = nil - - if err := b.chromeCmd.Start(); err != nil { - return fmt.Errorf("failed to start Chrome: %w", err) - } - - log.Printf("Chrome launched with PID %d on port %d", b.chromeCmd.Process.Pid, b.config.DebugPort) - return nil -} - -// waitForChrome waits for Chrome to be ready to accept connections -func (b *Browser) waitForChrome() error { - timeout := time.After(30 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - - for { - select { - case <-timeout: - return fmt.Errorf("timeout waiting for Chrome to start") - case <-ticker.C: - conn, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%d", b.config.DebugPort), time.Second) - if err == nil { - conn.Close() - time.Sleep(500 * time.Millisecond) // Give Chrome a moment to fully initialize - return nil - } - } - } -} - -// getWebSocketURL gets the WebSocket debugger URL from Chrome -func (b *Browser) getWebSocketURL() (string, error) { - resp, err := http.Get(fmt.Sprintf("http://localhost:%d/json/version", b.config.DebugPort)) - if err != nil { - return "", err - } - defer resp.Body.Close() - - var result chromeResponse - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return "", err - } - - return result.WebSocketDebuggerURL, nil -} - -// findChromePath finds the Chrome executable path -func findChromePath() string { - var paths []string - - switch runtime.GOOS { - case "darwin": - paths = []string{ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - } - case "linux": - paths = []string{ - "/usr/bin/google-chrome", - "/usr/bin/google-chrome-stable", - "/usr/bin/chromium", - "/usr/bin/chromium-browser", - } - case "windows": - paths = []string{ - os.Getenv("LOCALAPPDATA") + "\\Google\\Chrome\\Application\\chrome.exe", - os.Getenv("PROGRAMFILES") + "\\Google\\Chrome\\Application\\chrome.exe", - os.Getenv("PROGRAMFILES(X86)") + "\\Google\\Chrome\\Application\\chrome.exe", - } - } - - for _, p := range paths { - if _, err := os.Stat(p); err == nil { - return p - } - } - return "" -} - -// newAutomatedChrome creates a browser using chromedp's built-in launcher (fallback) -func newAutomatedChrome(parentCtx context.Context, config Config) (*Browser, error) { - opts := []chromedp.ExecAllocatorOption{ - chromedp.NoFirstRun, - chromedp.NoDefaultBrowserCheck, - chromedp.DisableGPU, - chromedp.NoSandbox, - chromedp.Flag("disable-blink-features", "AutomationControlled"), - chromedp.Flag("disable-infobars", true), - chromedp.Flag("disable-dev-shm-usage", true), - chromedp.Flag("window-size", "1920,1080"), - chromedp.UserAgent(getRandomUserAgent()), - } - - if config.Headless { - opts = append(opts, chromedp.Flag("headless", "new")) - } - - if config.UserDataDir != "" { - opts = append(opts, chromedp.UserDataDir(config.UserDataDir)) - } - - allocCtx, allocCancel := chromedp.NewExecAllocator(parentCtx, opts...) - ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf)) - ctx, timeoutCancel := context.WithTimeout(ctx, config.Timeout) - - b := &Browser{ - allocCtx: allocCtx, - allocCancel: allocCancel, - ctx: ctx, - cancel: func() { - timeoutCancel() - cancel() - }, - config: config, - } - - if err := b.injectAntiDetection(); err != nil { - b.Close() - return nil, fmt.Errorf("failed to inject anti-detection: %w", err) - } - - return b, nil -} - -// Context returns the browser context -func (b *Browser) Context() context.Context { - return b.ctx -} - -// Close closes the browser and kills Chrome process if we launched it -func (b *Browser) Close() { - if b.cancel != nil { - b.cancel() - } - if b.allocCancel != nil { - b.allocCancel() - } - // Kill Chrome process if we launched it - if b.chromeCmd != nil && b.chromeCmd.Process != nil { - log.Println("Killing Chrome process...") - b.chromeCmd.Process.Kill() - } -} - -// injectAntiDetection injects JavaScript to bypass bot detection -func (b *Browser) injectAntiDetection() error { - // JavaScript to override navigator properties and hide automation - antiDetectionScript := ` - // Override webdriver property - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined, - }); - - // Override plugins - Object.defineProperty(navigator, 'plugins', { - get: () => [1, 2, 3, 4, 5], - }); - - // Override languages - Object.defineProperty(navigator, 'languages', { - get: () => ['en-US', 'en'], - }); - - // Override platform - Object.defineProperty(navigator, 'platform', { - get: () => 'MacIntel', - }); - - // Override hardware concurrency - Object.defineProperty(navigator, 'hardwareConcurrency', { - get: () => 8, - }); - - // Override device memory - Object.defineProperty(navigator, 'deviceMemory', { - get: () => 8, - }); - - // Override permissions query - const originalQuery = window.navigator.permissions.query; - window.navigator.permissions.query = (parameters) => ( - parameters.name === 'notifications' ? - Promise.resolve({ state: Notification.permission }) : - originalQuery(parameters) - ); - - // Override chrome runtime - window.chrome = { - runtime: {}, - }; - - // Override console.debug to prevent detection - window.console.debug = () => {}; - - // Add mouse movement simulation capability - window.simulateMouseMovement = function() { - const event = new MouseEvent('mousemove', { - bubbles: true, - cancelable: true, - clientX: Math.random() * window.innerWidth, - clientY: Math.random() * window.innerHeight - }); - document.dispatchEvent(event); - }; - ` - - return chromedp.Run(b.ctx, - chromedp.ActionFunc(func(ctx context.Context) error { - _, err := page.AddScriptToEvaluateOnNewDocument(antiDetectionScript).Do(ctx) - return err - }), - ) -} - -// Navigate navigates to a URL and checks for CAPTCHA/bot detection -func (b *Browser) Navigate(url string) error { - if err := chromedp.Run(b.ctx, - chromedp.Navigate(url), - chromedp.WaitVisible("body", chromedp.ByQuery), - chromedp.Sleep(2*time.Second), - ); err != nil { - return err - } - - // Check for CAPTCHA/bot detection - if detected, err := b.CheckForCaptcha(); err != nil { - return err - } else if detected { - return ErrCaptchaDetected - } - - return nil -} - -// CheckForCaptcha checks if the page shows CAPTCHA or bot detection -func (b *Browser) CheckForCaptcha() (bool, error) { - var pageContent string - var pageTitle string - - if err := chromedp.Run(b.ctx, - chromedp.Title(&pageTitle), - chromedp.Evaluate(`document.body.innerText`, &pageContent), - ); err != nil { - return false, err - } - - // Check for common CAPTCHA/bot detection indicators - captchaIndicators := []string{ - "Failed to verify your browser", - "Vercel Security Checkpoint", - "Please verify you are a human", - "Access denied", - "Checking your browser", - "Please complete the security check", - "captcha", - "CAPTCHA", - "robot", - "bot detection", - "security challenge", - "Code 10", - "Code 21", - "Code 22", - } - - contentLower := strings.ToLower(pageContent) - titleLower := strings.ToLower(pageTitle) - - for _, indicator := range captchaIndicators { - indicatorLower := strings.ToLower(indicator) - if strings.Contains(contentLower, indicatorLower) || strings.Contains(titleLower, indicatorLower) { - log.Printf("CAPTCHA/Bot detection detected: found '%s'", indicator) - return true, nil - } - } - - return false, nil -} - -// WaitAndClick waits for an element and clicks it with human-like delay -func (b *Browser) WaitAndClick(selector string) error { - RandomDelay(500, 1500) - return chromedp.Run(b.ctx, - chromedp.WaitVisible(selector, chromedp.ByQuery), - chromedp.Click(selector, chromedp.ByQuery), - ) -} - -// Type types text with human-like delays between keystrokes -func (b *Browser) Type(selector, text string) error { - RandomDelay(300, 800) - return chromedp.Run(b.ctx, - chromedp.WaitVisible(selector, chromedp.ByQuery), - chromedp.Click(selector, chromedp.ByQuery), - chromedp.Sleep(200*time.Millisecond), - chromedp.SendKeys(selector, text, chromedp.ByQuery), - ) -} - -// Evaluate evaluates JavaScript and returns the result -func (b *Browser) Evaluate(script string, result interface{}) error { - return chromedp.Run(b.ctx, - chromedp.Evaluate(script, result), - ) -} - -// Sleep pauses execution for the specified duration -func (b *Browser) Sleep(d time.Duration) error { - return chromedp.Run(b.ctx, chromedp.Sleep(d)) -} - -// Nodes returns nodes matching the selector -func (b *Browser) Nodes(selector string) ([]*cdp.Node, error) { - var nodes []*cdp.Node - err := chromedp.Run(b.ctx, - chromedp.Nodes(selector, &nodes, chromedp.ByQueryAll), - ) - return nodes, err -} - -// SimulateHumanBehavior performs random mouse movements and scrolls -func (b *Browser) SimulateHumanBehavior() error { - return chromedp.Run(b.ctx, - chromedp.Evaluate(`window.simulateMouseMovement()`, nil), - chromedp.Sleep(time.Duration(rand.Intn(500)+200)*time.Millisecond), - chromedp.Evaluate(`window.scrollBy(0, Math.random() * 100)`, nil), - ) -} - -// RandomDelay waits for a random duration between min and max milliseconds -func RandomDelay(minMs, maxMs int) { - delay := time.Duration(rand.Intn(maxMs-minMs)+minMs) * time.Millisecond - time.Sleep(delay) -} - -// getRandomUserAgent returns a random modern user agent -func getRandomUserAgent() string { - userAgents := []string{ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36", - "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15", - } - return userAgents[rand.Intn(len(userAgents))] -} diff --git a/services/scraper-hiringcafe/db/db.ts b/services/scraper-hiringcafe/db/db.ts new file mode 100644 index 0000000..c281440 --- /dev/null +++ b/services/scraper-hiringcafe/db/db.ts @@ -0,0 +1,19 @@ +import postgres from "postgres"; +import dotenv from "dotenv"; + +if (dotenv.config({ path: ".local.env" }).error) { + console.log("No .local.env file found, trying .env"); + if (dotenv.config().error) { + console.log("No .env file found, using system environment variables"); + } +} + +const sql = postgres({ + host: process.env.DB_HOST, + port: Number(process.env.DB_PORT) || 5432, + database: process.env.DB_NAME, + username: process.env.DB_USER, + password: process.env.DB_PASSWORD, +}); + +export default sql; diff --git a/services/scraper-hiringcafe/go.mod b/services/scraper-hiringcafe/go.mod deleted file mode 100644 index 71f7bf6..0000000 --- a/services/scraper-hiringcafe/go.mod +++ /dev/null @@ -1,31 +0,0 @@ -module github.com/jobs-scraper/services/scraper-hiringcafe - -go 1.24.0 - -require ( - github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d - github.com/chromedp/chromedp v0.14.2 - github.com/jobs-scraper/internal/pkg/domain v0.0.0 - github.com/jobs-scraper/internal/pkg/infrastructure v0.0.0 - github.com/jobs-scraper/libs/repo v0.0.0 - github.com/joho/godotenv v1.5.1 -) - -require ( - github.com/chromedp/sysutil v1.1.0 // indirect - github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect - github.com/gobwas/httphead v0.1.0 // indirect - github.com/gobwas/pool v0.2.1 // indirect - github.com/gobwas/ws v1.4.0 // indirect - github.com/golang-migrate/migrate/v4 v4.19.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/lib/pq v1.10.9 // indirect - golang.org/x/sys v0.36.0 // indirect -) - -replace ( - github.com/jobs-scraper/internal/pkg/domain => ../../internal/pkg/domain - github.com/jobs-scraper/internal/pkg/infrastructure => ../../internal/pkg/infrastructure - github.com/jobs-scraper/libs/repo => ../../libs/repo -) diff --git a/services/scraper-hiringcafe/go.sum b/services/scraper-hiringcafe/go.sum deleted file mode 100644 index ef02ac7..0000000 --- a/services/scraper-hiringcafe/go.sum +++ /dev/null @@ -1,88 +0,0 @@ -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d h1:ZtA1sedVbEW7EW80Iz2GR3Ye6PwbJAJXjv7D74xG6HU= -github.com/chromedp/cdproto v0.0.0-20250803210736-d308e07a266d/go.mod h1:NItd7aLkcfOA/dcMXvl8p1u+lQqioRMq/SqDp71Pb/k= -github.com/chromedp/chromedp v0.14.2 h1:r3b/WtwM50RsBZHMUm9fsNhhzRStTHrKdr2zmwbZSzM= -github.com/chromedp/chromedp v0.14.2/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= -github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= -github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= -github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= -github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= -github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= -github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= -github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= -github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= -github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= -github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 h1:iizUGZ9pEquQS5jTGkh4AqeeHCMbfbjeb0zMt0aEFzs= -github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2/go.mod h1:TiCD2a1pcmjd7YnhGH0f/zKNcCD06B029pHhzV23c2M= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= -github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE= -github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= -golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/services/scraper-hiringcafe/index.ts b/services/scraper-hiringcafe/index.ts new file mode 100644 index 0000000..70a31ab --- /dev/null +++ b/services/scraper-hiringcafe/index.ts @@ -0,0 +1,189 @@ +import puppeteer, { Browser, Page } from "puppeteer-core"; +import { ChildProcess } from "child_process"; +import * as dotenv from "dotenv"; +import { + ChromeVersionResponse, + launchChromeWithDebugging, + randomDelay, + waitForChromeReady, +} from "./utils/browser"; +import { BrowserService } from "./services/browserService"; +import { saveJob } from "./queries/save-jobs"; +import { saveJobDescription } from "./queries/save-job-description"; +import { Job } from "./types/job"; + +if (dotenv.config({ path: ".local.env" }).error) { + console.log("No .local.env file found, trying .env"); + if (dotenv.config().error) { + console.log("No .env file found, using system environment variables"); + } +} + +let globalBrowser: Browser | null = null; +let chrome: ChildProcess; + +async function cleanup() { + console.log("Starting cleanup process..."); + + if (globalBrowser) { + console.log("Closing browser on app shutdown"); + await globalBrowser.close(); + globalBrowser = null; + } + + if (chrome) { + chrome.kill("SIGINT"); + } + + console.log("Cleanup completed"); +} + +async function initializeBrowser(): Promise<{ browser: Browser; page: Page }> { + chrome = launchChromeWithDebugging(); + await waitForChromeReady(); + + const response = await fetch("http://localhost:9222/json/version", { + headers: { + Origin: "", + }, + }); + + if (!response.ok) { + throw new Error("Failed to connect to Chrome debugging port"); + } + + const chromeInstance = (await response.json()) as ChromeVersionResponse; + const { webSocketDebuggerUrl } = chromeInstance; + + const browser = await puppeteer.connect({ + browserWSEndpoint: webSocketDebuggerUrl, + defaultViewport: null, + }); + + globalBrowser = browser; + console.log("Connected to browser successfully"); + + let pages = await browser.pages(); + console.log(`Found ${pages.length} pages`); + + let existingcookies = await browser.cookies(); + await browser.deleteCookie(...existingcookies); + + let page: Page; + if (pages.length === 0) { + console.log("No pages found, creating a new page"); + page = await browser.newPage(); + } else { + page = pages[0]; + } + + await page.setUserAgent( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ); + + return { browser, page }; +} + +async function scrapeHiringCafeJobs(): Promise { + try { + const { page } = await initializeBrowser(); + const browserService = new BrowserService(page); + + const searchUrl = `https://hiring.cafe/?searchState=%7B%22defaultToUserLocation%22%3Afalse%2C%22searchQuery%22%3A%22software%22%2C%22dateFetchedPastNDays%22%3A2%7D`; + + await page.goto(searchUrl, { + waitUntil: ["networkidle2", "domcontentloaded"], + timeout: 60000, + }); + + const searchElement = await page.waitForSelector("#query-search-v4", { + timeout: 60000, + }); + + if (searchElement != null) { + console.log("found search element, continue"); + } else { + console.log("could not find search element, aborting"); + return; + } + + await page.waitForNetworkIdle({ idleTime: 2000, timeout: 0 }); + + let jobLinks: string[] = []; + + while (jobLinks.length < 1) { + await randomDelay(1000, 2000); + jobLinks = await browserService.getJobLinks(); + console.log(`Found ${jobLinks.length} job links`); + } + + for (const link of jobLinks) { + try { + const job = await browserService.getJobData(link); + + const jobId = await saveJob(job); + + if (jobId) { + await saveJobDescription({ + criteria: job.criteria, + description: job.description || "", + jobId: jobId, + }); + } + await randomDelay(1000, 2000); + } catch (error) { + console.error(`Error processing job link ${link}:`, error); + await randomDelay(1000, 2000); + + continue; + } + } + + console.log("Saved job listings to the database"); + + console.log("Saved job descriptions to the database"); + + await randomDelay(1500, 2500); + } catch (error) { + console.error("Error scraping hiring cafe jobs:", error); + throw error; + } + + if (globalBrowser) { + console.log("Closing browser on app shutdown"); + await globalBrowser.close(); + globalBrowser = null; + } + + if (chrome) { + chrome.kill("SIGINT"); + } +} + +async function main() { + console.log("Starting Hiring Cafe scraper service..."); + await scrapeHiringCafeJobs(); +} + +process.on("SIGINT", async () => { + console.log("Received SIGINT signal"); + await cleanup(); + if (chrome) { + chrome.kill("SIGINT"); + } + process.exit(0); +}); + +process.on("SIGTERM", async () => { + console.log("Received SIGTERM signal"); + await cleanup(); + if (chrome) { + chrome.kill("SIGTERM"); + } + process.exit(0); +}); + +main().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); diff --git a/services/scraper-hiringcafe/main.go b/services/scraper-hiringcafe/main.go deleted file mode 100644 index 850cff6..0000000 --- a/services/scraper-hiringcafe/main.go +++ /dev/null @@ -1,70 +0,0 @@ -package main - -import ( - "context" - "log" - "os" - "os/signal" - "syscall" - - "github.com/jobs-scraper/internal/pkg/infrastructure" - "github.com/jobs-scraper/libs/repo" - "github.com/jobs-scraper/services/scraper-hiringcafe/models" - "github.com/jobs-scraper/services/scraper-hiringcafe/scraper" - "github.com/joho/godotenv" -) - -func main() { - if err := godotenv.Load(".local.env"); err != nil { - log.Println("No .local.env file found, trying .env") - if err := godotenv.Load(); err != nil { - log.Println("No .env file found, using system environment variables") - } - } - - // Setup database connection - dbConfig := infrastructure.LoadConfigFromEnv() - db, err := infrastructure.NewConnection(dbConfig) - if err != nil { - log.Fatalf("Error connecting to db: %v", err) - } - defer db.Close() - - if err := db.Ping(); err != nil { - log.Fatalf("Error pinging db: %v", err) - } - log.Println("Successfully connected to db") - - // Initialize repositories - jobRepo := repo.NewJobRepository(db) - jobDescRepo := repo.NewJobDescriptionRepository(db) - - // Example filters - these will be configurable later - filters := models.SearchFilters{ - Keywords: "software engineer", - Location: "", - Remote: true, - } - - // Create scraper and run - s := scraper.New(jobRepo, jobDescRepo) - - // Handle graceful shutdown - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - - go func() { - <-sigChan - log.Println("Received shutdown signal, cleaning up...") - cancel() - }() - - if err := s.Scrape(ctx, filters); err != nil { - log.Printf("Error scraping: %v", err) - } - - log.Println("Scraper finished") -} diff --git a/services/scraper-hiringcafe/models/models.go b/services/scraper-hiringcafe/models/models.go deleted file mode 100644 index ed5eb1c..0000000 --- a/services/scraper-hiringcafe/models/models.go +++ /dev/null @@ -1,22 +0,0 @@ -package models - -// SearchFilters holds the filter configuration for hiring.cafe -type SearchFilters struct { - Keywords string - Location string - Remote bool - JobType string // full-time, part-time, contract, internship - DatePosted string // 24h, 7d, 30d -} - -// ScrapedJob holds the raw scraped job data before conversion to domain.Job -type ScrapedJob struct { - Title string - Company string - CompanyLink string - Location string - JobLink string - PostedTime string - Description string - Criteria map[string]string -} diff --git a/services/scraper-hiringcafe/package-lock.json b/services/scraper-hiringcafe/package-lock.json new file mode 100644 index 0000000..b8db1e5 --- /dev/null +++ b/services/scraper-hiringcafe/package-lock.json @@ -0,0 +1,1104 @@ +{ + "name": "glassdoor-scraper", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "glassdoor-scraper", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@types/amqplib": "^0.10.7", + "amqplib": "^0.10.9", + "dotenv": "^17.2.3", + "postgres": "^3.4.7", + "puppeteer-core": "^24.25.0" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.12", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.12.tgz", + "integrity": "sha512-mP9iLFZwH+FapKJLeA7/fLqOlSUwYpMwjR1P5J23qd4e7qGJwecJccJqHYrjw33jmIZYV4dtiTHPD/J+1e7cEw==", + "dependencies": { + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.3", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", + "integrity": "sha512-DcRjDCujK/kCk/cUe8Xz8ZSpm8mS3mNNpta+jGCA6USEDfktlNvm1+IuZ9eTcDbNk41BHwpHHeW+N1lKCz4zOw==", + "dev": true + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true + }, + "node_modules/@types/amqplib": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.10.8.tgz", + "integrity": "sha512-vtDp8Pk1wsE/AuQ8/Rgtm6KUZYqcnTgNvEHwzCkX8rL7AGsC6zqAfKAAJhUZXFhM/Pp++tbnUHiam/8vVpPztA==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "24.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz", + "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/amqplib": { + "version": "0.10.9", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.9.tgz", + "integrity": "sha512-jwSftI4QjS3mizvnSnOrPGYiUnm1vI2OP1iXeOUz5pb74Ua0nbf6nPyyTzuiCLEE3fMpaJORXh2K/TQ08H5xGA==", + "dependencies": { + "buffer-more-ints": "~1.0.0", + "url-parse": "~1.5.10" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/b4a": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.7.3.tgz", + "integrity": "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/bare-events": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.1.tgz", + "integrity": "sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.0.tgz", + "integrity": "sha512-GljgCjeupKZJNetTqxKaQArLK10vpmK28or0+RwWjEl5Rk+/xG3wkpmkv+WrcBm3q1BwHKlnhXzR8O37kcvkXQ==", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz", + "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz", + "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.2.tgz", + "integrity": "sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw==", + "optional": true, + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, + "node_modules/chromium-bidi": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-10.5.1.tgz", + "integrity": "sha512-rlj6OyhKhVTnk4aENcUme3Jl9h+cq4oXu4AzBcvr8RMmT6BR4a3zSNT9dbIfXr9/BS6ibzRyDhowuw4n2GgzsQ==", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1521046", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1521046.tgz", + "integrity": "sha512-vhE6eymDQSKWUXwwA37NtTTVEzjtGVfDr3pRbsWEQ5onH/Snp2c+2xZHWJJawG/0hCCJLRGt4xVtEVUVILol4w==" + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "node_modules/postgres": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.7.tgz", + "integrity": "sha512-Jtc2612XINuBjIl/QTWsV5UvE8UHuNblcO3vVADSrKsrc6RqGX6lOW1cEo3CM2v0XG4Nat8nI+YM7/f26VxXLw==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer-core": { + "version": "24.27.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.27.0.tgz", + "integrity": "sha512-yubwj2XXmTM3wRIpbhO5nCjbByPgpFHlgrsD4IK+gMPqO7/a5FfnoSXDKjmqi8A2M1Ewusz0rTI/r+IN0GU0MA==", + "dependencies": { + "@puppeteer/browsers": "2.10.12", + "chromium-bidi": "10.5.1", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1521046", + "typed-query-selector": "^2.12.0", + "webdriver-bidi-protocol": "0.3.8", + "ws": "^8.18.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/streamx": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==" + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==" + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true + }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.8.tgz", + "integrity": "sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ==" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/services/scraper-hiringcafe/package.json b/services/scraper-hiringcafe/package.json new file mode 100644 index 0000000..f24d6f4 --- /dev/null +++ b/services/scraper-hiringcafe/package.json @@ -0,0 +1,27 @@ +{ + "name": "glassdoor-scraper", + "version": "1.0.0", + "description": "", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "node dist/index.js", + "dev": "ts-node index.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@types/amqplib": "^0.10.7", + "amqplib": "^0.10.9", + "dotenv": "^17.2.3", + "postgres": "^3.4.7", + "puppeteer-core": "^24.25.0" + }, + "devDependencies": { + "@types/node": "^24.8.1", + "ts-node": "^10.9.2", + "typescript": "^5.9.3" + } +} diff --git a/services/scraper-hiringcafe/queries/save-job-description.ts b/services/scraper-hiringcafe/queries/save-job-description.ts new file mode 100644 index 0000000..9a10e2b --- /dev/null +++ b/services/scraper-hiringcafe/queries/save-job-description.ts @@ -0,0 +1,33 @@ +import sql from "../db/db"; +import { Job } from "../types/job"; +import { JobDescription } from "../types/job-description"; + +export const saveJobDescription = async ( + job: JobDescription +): Promise => { + if (!job.jobId) { + console.log("Job has no id, cannot save description."); + return; + } + + try { + await sql` + INSERT INTO public.job_descriptions ( + job_id, + description, + job_criteria + ) VALUES ( + ${job.jobId}, + ${job.description}, + ${JSON.stringify(job.criteria)} + ) + ON CONFLICT (job_id) DO UPDATE SET + description = EXCLUDED.description, + job_criteria = EXCLUDED.job_criteria, + updated_at = CURRENT_TIMESTAMP + `; + console.log(`Updated description for job ${job.jobId}.`); + } catch (error) { + console.error("Error updating job description:", error); + } +}; diff --git a/services/scraper-hiringcafe/queries/save-jobs.ts b/services/scraper-hiringcafe/queries/save-jobs.ts new file mode 100644 index 0000000..4240782 --- /dev/null +++ b/services/scraper-hiringcafe/queries/save-jobs.ts @@ -0,0 +1,42 @@ +import sql from "../db/db"; +import { Job } from "../types/job"; + +export const saveJob = async (job: Job): Promise => { + try { + const result = await sql` + INSERT INTO public.jobs ( + title, + company, + location, + company_link, + job_link, + provider, + status, + job_timestamp + ) VALUES ( + ${job.title}, + ${job.company}, + ${job.location}, + ${job.companyLink}, + ${job.jobLink}, + ${job.provider}, + ${job.status}, + ${job.jobPostTime} + ) + ON CONFLICT (job_link) DO UPDATE SET + title = EXCLUDED.title, + company = EXCLUDED.company, + company_link = EXCLUDED.company_link, + location = EXCLUDED.location, + job_timestamp = EXCLUDED.job_timestamp + RETURNING id + `; + + const id = result[0]?.id ?? null; + console.log(`Saved job with id ${id} to the database.`); + return id; + } catch (error) { + console.error("Error saving job to the database:", error); + return null; + } +}; diff --git a/services/scraper-hiringcafe/scraper-hiringcafe b/services/scraper-hiringcafe/scraper-hiringcafe deleted file mode 100755 index 6ec530f..0000000 Binary files a/services/scraper-hiringcafe/scraper-hiringcafe and /dev/null differ diff --git a/services/scraper-hiringcafe/scraper/scraper.go b/services/scraper-hiringcafe/scraper/scraper.go deleted file mode 100644 index c28d391..0000000 --- a/services/scraper-hiringcafe/scraper/scraper.go +++ /dev/null @@ -1,570 +0,0 @@ -package scraper - -import ( - "context" - "fmt" - "log" - "strings" - "time" - - "github.com/jobs-scraper/internal/pkg/domain" - "github.com/jobs-scraper/libs/repo" - "github.com/jobs-scraper/services/scraper-hiringcafe/api" - "github.com/jobs-scraper/services/scraper-hiringcafe/browser" - "github.com/jobs-scraper/services/scraper-hiringcafe/models" -) - -// HiringCafeScraper handles scraping jobs from hiring.cafe -type HiringCafeScraper struct { - jobRepo *repo.JobRepository - jobDescRepo *repo.JobDescriptionRepository - apiClient *api.Client - browser *browser.Browser -} - -// New creates a new scraper instance -func New(jobRepo *repo.JobRepository, jobDescRepo *repo.JobDescriptionRepository) *HiringCafeScraper { - return &HiringCafeScraper{ - jobRepo: jobRepo, - jobDescRepo: jobDescRepo, - apiClient: api.NewClient(), - } -} - -// Scrape performs the main scraping operation using the API -func (s *HiringCafeScraper) Scrape(ctx context.Context, filters models.SearchFilters) error { - log.Println("Starting hiring.cafe scraper using API...") - - // Convert models.SearchFilters to api.SearchFilters - apiFilters := api.SearchFilters{ - Query: filters.Keywords, - Country: "United States", - Remote: filters.Remote, - Hybrid: true, - Onsite: true, - FullTime: true, - PartTime: false, - Contract: false, - Internship: false, - EntryLevel: true, - MidLevel: true, - SeniorLevel: false, - DateRangeDays: 30, - MaxPages: 10, - } - - if filters.Location != "" { - apiFilters.Country = filters.Location - } - - log.Printf("Searching for: %s in %s", apiFilters.Query, apiFilters.Country) - - // Fetch jobs from API - apiJobs, err := s.apiClient.Search(apiFilters) - if err != nil { - return fmt.Errorf("failed to fetch jobs from API: %w", err) - } - - log.Printf("Found %d jobs from API", len(apiJobs)) - - if len(apiJobs) == 0 { - log.Println("No jobs found") - return nil - } - - // Convert API jobs to scraped jobs - var scrapedJobs []models.ScrapedJob - for _, job := range apiJobs { - scrapedJob := models.ScrapedJob{ - Title: job.Title, - Company: job.Company, - CompanyLink: job.CompanyURL, - Location: job.Location, - JobLink: job.ApplyURL, - PostedTime: job.PostedAt.Format("2006-01-02"), - Description: job.DescriptionClean, - Criteria: make(map[string]string), - } - - if job.Salary != "" { - scrapedJob.Criteria["Salary"] = job.Salary - } - if job.Remote { - scrapedJob.Criteria["Remote"] = "Yes" - } - - scrapedJobs = append(scrapedJobs, scrapedJob) - } - - // Save jobs to database - if err := s.saveJobs(scrapedJobs); err != nil { - return fmt.Errorf("failed to save jobs: %w", err) - } - - log.Printf("Successfully saved %d jobs to database", len(scrapedJobs)) - return nil -} - -// ScrapeWithBrowser performs scraping using browser automation (fallback if API fails) -func (s *HiringCafeScraper) ScrapeWithBrowser(ctx context.Context, filters models.SearchFilters) error { - // Create browser with anti-detection - config := browser.DefaultConfig() - config.Headless = false - - b, err := browser.New(ctx, config) - if err != nil { - return fmt.Errorf("failed to create browser: %w", err) - } - defer b.Close() - s.browser = b - - log.Println("Starting browser and navigating to hiring.cafe...") - - // Navigate to hiring.cafe - if err := b.Navigate("https://hiring.cafe/"); err != nil { - if err == browser.ErrCaptchaDetected { - log.Println("CAPTCHA/Bot detection encountered - terminating scraper") - return err - } - return fmt.Errorf("failed to navigate to hiring.cafe: %w", err) - } - - log.Println("Page loaded successfully, no CAPTCHA detected") - - // Apply filters - if err := s.applyFilters(filters); err != nil { - return fmt.Errorf("failed to apply filters: %w", err) - } - - log.Println("Filters applied, extracting job listings...") - - // Extract job listings - jobs, err := s.extractJobListings() - if err != nil { - return fmt.Errorf("failed to extract job listings: %w", err) - } - - log.Printf("Found %d jobs", len(jobs)) - - // Save jobs to database - if err := s.saveJobs(jobs); err != nil { - return fmt.Errorf("failed to save jobs: %w", err) - } - - log.Printf("Successfully saved %d jobs to database", len(jobs)) - return nil -} - -// applyFilters applies search filters on hiring.cafe -func (s *HiringCafeScraper) applyFilters(filters models.SearchFilters) error { - if err := s.browser.Sleep(2 * time.Second); err != nil { - return err - } - - // Apply keyword search if provided - if filters.Keywords != "" { - log.Printf("Searching for: %s", filters.Keywords) - - // Try multiple possible search input selectors - searchSelectors := []string{ - `input[type="search"]`, - `input[placeholder*="search" i]`, - `input[placeholder*="job" i]`, - `input[type="text"]`, - `input[name="q"]`, - `input[name="search"]`, - } - - var searchApplied bool - for _, selector := range searchSelectors { - if err := s.browser.Type(selector, filters.Keywords); err == nil { - searchApplied = true - log.Printf("Applied search using selector: %s", selector) - break - } - } - - if searchApplied { - browser.RandomDelay(500, 1000) - // Press Enter to search - var result interface{} - s.browser.Evaluate(` - const input = document.querySelector('input[type="search"], input[placeholder*="search" i], input[type="text"]'); - if (input) { - input.dispatchEvent(new KeyboardEvent('keydown', {key: 'Enter', code: 'Enter', keyCode: 13, which: 13, bubbles: true})); - input.form?.submit(); - } - `, &result) - s.browser.Sleep(3 * time.Second) - } else { - log.Println("Warning: could not find search input") - } - } - - // Apply remote filter if enabled - if filters.Remote { - log.Println("Applying remote filter...") - remoteSelectors := []string{ - `[data-filter="remote"]`, - `button:contains("Remote")`, - `label:contains("Remote")`, - `input[value="remote"]`, - `[class*="remote"]`, - `a[href*="remote"]`, - } - - for _, selector := range remoteSelectors { - if err := s.browser.WaitAndClick(selector); err == nil { - log.Printf("Applied remote filter using selector: %s", selector) - s.browser.Sleep(2 * time.Second) - break - } - } - } - - // Wait for results to load - return s.browser.Sleep(3 * time.Second) -} - -// extractJobListings extracts job listings from the current page -func (s *HiringCafeScraper) extractJobListings() ([]models.ScrapedJob, error) { - if err := s.browser.Sleep(2 * time.Second); err != nil { - return nil, err - } - - // Simulate scrolling to load more jobs - s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight / 2)`, nil) - s.browser.Sleep(1 * time.Second) - s.browser.Evaluate(`window.scrollTo(0, document.body.scrollHeight)`, nil) - s.browser.Sleep(2 * time.Second) - - // Extract job data using JavaScript - var jobData []map[string]string - err := s.browser.Evaluate(` - (function() { - const jobs = []; - - // Try multiple selectors for job cards - const selectors = [ - '[class*="job-card"]', - '[class*="job-listing"]', - '[class*="JobCard"]', - '[class*="job-item"]', - 'article[class*="job"]', - 'div[data-job-id]', - 'a[href*="/jobs/"]', - 'a[href*="/job/"]', - '[class*="listing-card"]', - '[class*="position-card"]', - 'li[class*="job"]', - 'tr[class*="job"]' - ]; - - let elements = []; - for (const selector of selectors) { - const found = document.querySelectorAll(selector); - if (found.length > 0) { - elements = Array.from(found); - console.log('Found jobs with selector:', selector, found.length); - break; - } - } - - // If no specific job cards found, try to find any links that look like job postings - if (elements.length === 0) { - const allLinks = document.querySelectorAll('a[href*="job"], a[href*="position"], a[href*="career"]'); - elements = Array.from(allLinks).filter(el => { - const text = el.textContent.toLowerCase(); - return text.length > 10 && text.length < 200; - }); - } - - elements.forEach((el, index) => { - try { - // Try to extract job information - const titleSelectors = ['h1', 'h2', 'h3', 'h4', '[class*="title"]', '[class*="Title"]', '[class*="name"]']; - const companySelectors = ['[class*="company"]', '[class*="Company"]', '[class*="employer"]', '[class*="org"]']; - const locationSelectors = ['[class*="location"]', '[class*="Location"]', '[class*="place"]', '[class*="city"]']; - const timeSelectors = ['[class*="time"]', '[class*="date"]', '[class*="posted"]', 'time', '[class*="ago"]']; - - let title = ''; - let company = ''; - let location = ''; - let postedTime = ''; - - for (const sel of titleSelectors) { - const titleEl = el.querySelector(sel); - if (titleEl && titleEl.textContent.trim()) { - title = titleEl.textContent.trim(); - break; - } - } - - for (const sel of companySelectors) { - const companyEl = el.querySelector(sel); - if (companyEl && companyEl.textContent.trim()) { - company = companyEl.textContent.trim(); - break; - } - } - - for (const sel of locationSelectors) { - const locationEl = el.querySelector(sel); - if (locationEl && locationEl.textContent.trim()) { - location = locationEl.textContent.trim(); - break; - } - } - - for (const sel of timeSelectors) { - const timeEl = el.querySelector(sel); - if (timeEl && timeEl.textContent.trim()) { - postedTime = timeEl.textContent.trim(); - break; - } - } - - // Get job link - let jobLink = ''; - const linkEl = el.tagName === 'A' ? el : el.querySelector('a[href*="job"], a[href*="position"], a'); - if (linkEl && linkEl.href) { - jobLink = linkEl.href; - } - - // Get company link - let companyLink = ''; - const companyLinkEl = el.querySelector('a[href*="company"], a[href*="employer"]'); - if (companyLinkEl && companyLinkEl.href) { - companyLink = companyLinkEl.href; - } - - // If no title found, use link text - if (!title && linkEl) { - title = linkEl.textContent.trim(); - } - - const job = { - title: title.substring(0, 500), - company: company.substring(0, 200), - location: location.substring(0, 200), - jobLink: jobLink, - postedTime: postedTime.substring(0, 100), - companyLink: companyLink - }; - - // Only add if we have at least a title and link - if (job.title && job.jobLink) { - jobs.push(job); - } - } catch (error) { - console.error('Error extracting job ' + index + ':', error); - } - }); - - // Remove duplicates based on jobLink - const seen = new Set(); - return jobs.filter(job => { - if (seen.has(job.jobLink)) return false; - seen.add(job.jobLink); - return true; - }); - })() - `, &jobData) - - if err != nil { - return nil, fmt.Errorf("failed to extract job data: %w", err) - } - - // Convert to ScrapedJob structs - var jobs []models.ScrapedJob - for _, data := range jobData { - job := models.ScrapedJob{ - Title: data["title"], - Company: data["company"], - CompanyLink: data["companyLink"], - Location: data["location"], - JobLink: data["jobLink"], - PostedTime: data["postedTime"], - Criteria: make(map[string]string), - } - jobs = append(jobs, job) - } - - return jobs, nil -} - -// getJobDetails navigates to a job page and extracts detailed information -func (s *HiringCafeScraper) getJobDetails(job models.ScrapedJob) (models.ScrapedJob, error) { - if job.JobLink == "" { - return job, nil - } - - // Navigate to job detail page - if err := s.browser.Navigate(job.JobLink); err != nil { - return job, fmt.Errorf("failed to navigate to job page: %w", err) - } - - s.browser.Sleep(2 * time.Second) - s.browser.SimulateHumanBehavior() - - // Extract job description - var description string - s.browser.Evaluate(` - (function() { - const descSelectors = [ - '[class*="description"]', - '[class*="Description"]', - '[class*="job-content"]', - '[class*="job-details"]', - '[class*="details"]', - '[id*="description"]', - 'article', - 'main', - '.content' - ]; - - for (const selector of descSelectors) { - const el = document.querySelector(selector); - if (el && el.textContent.trim().length > 100) { - return el.textContent.trim().substring(0, 10000); - } - } - - // Fallback: get body text - return document.body.innerText.substring(0, 10000); - })() - `, &description) - - // Extract criteria - var criteriaData map[string]string - s.browser.Evaluate(` - (function() { - const criteria = {}; - - // Look for common job criteria patterns - const patterns = [ - {key: 'Experience', selectors: ['[class*="experience"]', '[class*="Experience"]']}, - {key: 'Education', selectors: ['[class*="education"]', '[class*="Education"]']}, - {key: 'Skills', selectors: ['[class*="skills"]', '[class*="Skills"]']}, - {key: 'Salary', selectors: ['[class*="salary"]', '[class*="Salary"]', '[class*="compensation"]']}, - {key: 'JobType', selectors: ['[class*="job-type"]', '[class*="employment"]', '[class*="type"]']}, - ]; - - patterns.forEach(({key, selectors}) => { - for (const selector of selectors) { - const el = document.querySelector(selector); - if (el && el.textContent.trim()) { - criteria[key] = el.textContent.trim().substring(0, 500); - break; - } - } - }); - - return criteria; - })() - `, &criteriaData) - - job.Description = description - if criteriaData != nil { - job.Criteria = criteriaData - } - - return job, nil -} - -// saveJobs saves the scraped jobs to the database -func (s *HiringCafeScraper) saveJobs(scrapedJobs []models.ScrapedJob) error { - if len(scrapedJobs) == 0 { - return nil - } - - var domainJobs []domain.Job - var jobDescriptions []domain.JobDescription - - for _, sj := range scrapedJobs { - postTime := parsePostedTime(sj.PostedTime) - - job := domain.Job{ - Title: sj.Title, - Company: sj.Company, - CompanyLink: sj.CompanyLink, - Location: sj.Location, - JobLink: sj.JobLink, - Provider: domain.HiringCafe, - JobPostTime: postTime, - Status: domain.JobStatusCreated, - } - domainJobs = append(domainJobs, job) - } - - // Save jobs - if err := s.jobRepo.SaveJobs(domainJobs); err != nil { - return fmt.Errorf("failed to save jobs: %w", err) - } - - // Collect job descriptions - for _, sj := range scrapedJobs { - if sj.Description == "" { - continue - } - - jobDesc := domain.JobDescription{ - Description: sj.Description, - Criteria: sj.Criteria, - } - jobDescriptions = append(jobDescriptions, jobDesc) - } - - log.Printf("Job descriptions collected: %d (saving requires job IDs)", len(jobDescriptions)) - - return nil -} - -// parsePostedTime converts relative time strings to actual dates -func parsePostedTime(timeStr string) *time.Time { - now := time.Now() - timeStr = strings.ToLower(strings.TrimSpace(timeStr)) - - if timeStr == "" { - return &now - } - - // Parse common patterns - if strings.Contains(timeStr, "hour") || strings.HasSuffix(timeStr, "h") { - var hours int - fmt.Sscanf(timeStr, "%d", &hours) - if hours > 0 { - t := now.Add(-time.Duration(hours) * time.Hour) - return &t - } - } - - if strings.Contains(timeStr, "day") || strings.HasSuffix(timeStr, "d") { - var days int - fmt.Sscanf(timeStr, "%d", &days) - if days > 0 { - t := now.AddDate(0, 0, -days) - return &t - } - } - - if strings.Contains(timeStr, "week") || strings.HasSuffix(timeStr, "w") { - var weeks int - fmt.Sscanf(timeStr, "%d", &weeks) - if weeks > 0 { - t := now.AddDate(0, 0, -weeks*7) - return &t - } - } - - if strings.Contains(timeStr, "month") || strings.HasSuffix(timeStr, "m") { - var months int - fmt.Sscanf(timeStr, "%d", &months) - if months > 0 { - t := now.AddDate(0, -months, 0) - return &t - } - } - - return &now -} diff --git a/services/scraper-hiringcafe/services/browserService.ts b/services/scraper-hiringcafe/services/browserService.ts new file mode 100644 index 0000000..20da88f --- /dev/null +++ b/services/scraper-hiringcafe/services/browserService.ts @@ -0,0 +1,99 @@ +import { ElementHandle, Page } from "puppeteer-core"; +import { randomDelay } from "../utils/browser"; +import { Job } from "../types/job"; +import { createDateFromRelativeTime } from "../utils/parse-time"; + +export class BrowserService { + private page: Page; + + constructor(page: Page) { + this.page = page; + } + + async getJobLinks(): Promise { + const jobLinks = await this.page.evaluate(() => { + const links = document.querySelectorAll( + "a[href*='/viewjob/']" + ) as NodeListOf; + return Array.from(links).map((link) => link.href); + }); + + return jobLinks; + } + + async getJobData(jobLink: string): Promise { + let jobContainer: ElementHandle | null = null; + + await this.page.goto(jobLink, { + waitUntil: "domcontentloaded", + timeout: 60000, + }); + + // await this.page.waitForNetworkIdle({ idleTime: 3000, timeout: 0 }); + + try { + jobContainer = await this.page.waitForSelector( + "::-p-xpath(/html/body/div[1]/div[3]/div/div/div/div[1])", + { + timeout: 10000, + } + ); + } catch (error) { + console.log("No job listings found on the page"); + return {} as Job; + } + + if (jobContainer === null) { + console.log("Job container not found"); + return {} as Job; + } + + const jobTitle = await jobContainer.waitForSelector("h2"); + const jobTitleText = await jobTitle?.evaluate((el) => + el.textContent?.trim() + ); + const company = await jobContainer.waitForSelector( + "span.text-xl.font-semibold.text-gray-700.flex-none" + ); + const companyText = await company?.evaluate((el) => el.textContent?.trim()); + const location = await jobContainer.waitForSelector( + "::-p-xpath(/html/body/div[1]/div[3]/div/div/div/div[1]/div/div[2]/div[3]/span)" + ); + const locationText = await location?.evaluate((el) => + el.textContent?.trim() + ); + const jobPostTime = await jobContainer.waitForSelector( + "::-p-xpath(/html/body/div[1]/div[3]/div/div/div/div[1]/div/div[2]/div[1]/div/span)" + ); + const jobPostTimeText = await jobPostTime?.evaluate((el) => + el.textContent?.trim() + ); + + const time = jobPostTimeText?.split(" ")[1] ?? ""; + + const parsedDateTime = createDateFromRelativeTime(time); + + const description = await this.page.waitForSelector("article"); + + const descriptionText = await description?.evaluate((el) => + el.textContent?.trim() + ); + + const job: Job = { + company: companyText || "", + title: jobTitleText || "", + location: locationText || "", + description: descriptionText || "", + criteria: {}, + jobLink, + companyLink: "", + provider: 7, + jobPostTime: parsedDateTime.toISOString(), + status: 0, + }; + + console.log({ job }); + + return job; + } +} diff --git a/services/scraper-hiringcafe/tsconfig.json b/services/scraper-hiringcafe/tsconfig.json new file mode 100644 index 0000000..651cacf --- /dev/null +++ b/services/scraper-hiringcafe/tsconfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020", "DOM"], + "outDir": "./dist", + "rootDir": "./", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "removeComments": true, + "noImplicitAny": true, + "noImplicitReturns": true, + "noImplicitThis": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "exactOptionalPropertyTypes": false, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": false, + "noUncheckedIndexedAccess": false + }, + "include": [ + "**/*.ts", + "**/*.tsx" + ], + "exclude": [ + "node_modules", + "dist" + ] +} diff --git a/services/scraper-hiringcafe/types/job-description.ts b/services/scraper-hiringcafe/types/job-description.ts new file mode 100644 index 0000000..e70580d --- /dev/null +++ b/services/scraper-hiringcafe/types/job-description.ts @@ -0,0 +1,5 @@ +export interface JobDescription { + jobId: number; + description: string; + criteria: Record; +} diff --git a/services/scraper-hiringcafe/types/job.ts b/services/scraper-hiringcafe/types/job.ts new file mode 100644 index 0000000..ce48f00 --- /dev/null +++ b/services/scraper-hiringcafe/types/job.ts @@ -0,0 +1,13 @@ +export interface Job { + id?: number; + title: string; + company: string; + location: string; + description: string; + criteria: Record; + jobLink: string; + companyLink: string; + provider: 7; + jobPostTime: string; + status: number; +} diff --git a/services/scraper-hiringcafe/utils/browser.ts b/services/scraper-hiringcafe/utils/browser.ts new file mode 100644 index 0000000..df532d7 --- /dev/null +++ b/services/scraper-hiringcafe/utils/browser.ts @@ -0,0 +1,139 @@ +import fs from "fs"; +import path from "path"; +import os from "os"; +import { spawn } from "child_process"; +import net from "net"; + +export interface ChromeVersionResponse { + webSocketDebuggerUrl: string; +} + +const userDataDir = path.join(os.tmpdir(), "chrome-debug-profile"); + +export const getLocalISOString = (date = new Date()) => { + const offset = date.getTimezoneOffset(); + const offsetAbs = Math.abs(offset); + const hours = Math.floor(offsetAbs / 60) + .toString() + .padStart(2, "0"); + const minutes = (offsetAbs % 60).toString().padStart(2, "0"); + const sign = offset > 0 ? "-" : "+"; + + return new Date(date.getTime() - offset * 60000) + .toISOString() + .replace("Z", `${sign}${hours}:${minutes}`); +}; + +function getChromePath(): string { + const platform = os.platform(); + + const paths: Record = { + darwin: [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + ], + win32: [ + process.env.LOCALAPPDATA + "\\Google\\Chrome\\Application\\chrome.exe", + process.env.PROGRAMFILES + "\\Google\\Chrome\\Application\\chrome.exe", + process.env["PROGRAMFILES(X86)"] + + "\\Google\\Chrome\\Application\\chrome.exe", + ], + linux: [ + "/usr/bin/google-chrome", + "/usr/bin/google-chrome-stable", + "/usr/bin/chromium", + "/usr/bin/chromium-browser", + ], + aix: [], + freebsd: [], + openbsd: [], + sunos: [], + android: [], + haiku: [], + cygwin: [], + netbsd: [], + }; + + const platformPaths = paths[platform] || []; + + for (const path of platformPaths) { + try { + if (fs.existsSync(path)) { + return path; + } + } catch (err) { + continue; + } + } + + throw new Error("Could not find Chrome/Chromium installation"); +} + +export function launchChromeWithDebugging() { + try { + const chromePath = getChromePath(); + const args = [ + "--remote-debugging-port=9222", + `--user-data-dir=${userDataDir}`, + "--remote-allow-origins=*", + "--incognito", + // "--headless", + // `--proxy-server=46.3.202.240:2120`, + // `--proxy-server=http://jN8AqiMh:Pd9FEs6N@212.193.184.228:64616`, + ]; + + console.log(`Launching Chrome at: ${chromePath}`); + + const chromeProcess = spawn(chromePath, args, { + detached: true, + stdio: "ignore", + }); + + chromeProcess.unref(); + + chromeProcess.on("error", (err) => { + console.error("Failed to start Chrome:", err); + }); + + chromeProcess.on("exit", (code) => { + if (code !== 0) { + console.error(`Chrome process exited with code ${code}`); + } + }); + + console.log("Chrome launched successfully with debugging port 9222"); + return chromeProcess; + } catch (error) { + console.error("Failed to launch Chrome:", error); + process.exit(1); + } +} + +export async function waitForChromeReady(port = 9222, timeout = 30000) { + const startTime = Date.now(); + + return new Promise((resolve, reject) => { + function check() { + const client = net.createConnection({ port }, () => { + client.end(); + resolve(""); + }); + + client.on("error", () => { + if (Date.now() - startTime > timeout) { + reject(new Error("Timeout waiting for Chrome to start")); + } else { + setTimeout(check, 500); + } + }); + } + + check(); + }); +} + +export function randomDelay(min: number, max: number): Promise { + const delay = Math.floor(Math.random() * (max - min) + min); + return new Promise((resolve) => setTimeout(resolve, delay)); +} diff --git a/services/scraper-hiringcafe/utils/parse-time.ts b/services/scraper-hiringcafe/utils/parse-time.ts new file mode 100644 index 0000000..7a2afe5 --- /dev/null +++ b/services/scraper-hiringcafe/utils/parse-time.ts @@ -0,0 +1,28 @@ +export function createDateFromRelativeTime(text: string) { + const now = new Date(); + const cleanText = text + .toLowerCase() + .replace("posted", "") + .replace("ago", "") + .trim(); + + // Parse common patterns + const patterns = [ + { regex: /(\d+)\s*d/, multiplier: 24 * 60 * 60 * 1000 }, + { regex: /(\d+)\s*h/, multiplier: 60 * 60 * 1000 }, + { regex: /(\d+)\s*hr/, multiplier: 60 * 60 * 1000 }, + { regex: /(\d+)\s*min/, multiplier: 60 * 1000 }, + { regex: /(\d+)\s*m(?!in)/, multiplier: 60 * 1000 }, + { regex: /(\d+)\s*s/, multiplier: 1000 }, + ]; + + for (const pattern of patterns) { + const match = cleanText.match(pattern.regex); + if (match) { + const value = parseInt(match[1]); + return new Date(now.getTime() - value * pattern.multiplier); + } + } + + throw new Error(`Could not parse time string: ${text}`); +} diff --git a/services/scraper-hiringcafe/utils/rabbitmq.ts b/services/scraper-hiringcafe/utils/rabbitmq.ts new file mode 100644 index 0000000..d021070 --- /dev/null +++ b/services/scraper-hiringcafe/utils/rabbitmq.ts @@ -0,0 +1,165 @@ +import * as amqp from "amqplib"; + +export interface RabbitMQConfig { + url: string; + queueName: string; + exchangeName: string; + exchangeType: "direct" | "topic" | "fanout" | "headers"; + durable: boolean; +} + +export interface SearchQuery { + keywords: string; + location: string; + fwt?: string; + numPages: number; +} + +export class RabbitMQClient { + private connection: amqp.ChannelModel | null = null; + private channel: amqp.Channel | null = null; + private config: RabbitMQConfig; + + constructor(config: RabbitMQConfig) { + this.config = config; + } + + async connect(): Promise { + try { + console.log(`Connecting to RabbitMQ at: ${this.config.url}`); + this.connection = await amqp.connect(this.config.url); + console.log("Connected to RabbitMQ successfully"); + + if (!this.connection) { + throw new Error("Failed to establish RabbitMQ connection"); + } + + this.channel = await this.connection.createChannel(); + + if (!this.channel) { + throw new Error("Failed to create RabbitMQ channel"); + } + + // Ensure exchange and queue exist + await this.channel.assertExchange( + this.config.exchangeName, + this.config.exchangeType, + { durable: this.config.durable } + ); + + await this.channel.assertQueue(this.config.queueName, { + durable: this.config.durable, + arguments: { + "x-dead-letter-exchange": "scraper_dlx", + "x-dead-letter-routing-key": this.config.queueName, + "x-message-ttl": 24 * 60 * 60 * 1000, // 24 hours in milliseconds + "x-max-retries": 3, + }, + }); + + await this.channel.bindQueue( + this.config.queueName, + this.config.exchangeName, + this.config.queueName + ); + + // Set prefetch to process one message at a time + await this.channel.prefetch(1); + + // Handle connection events + this.connection.on("close", () => { + console.log("RabbitMQ connection closed"); + this.connection = null; + this.channel = null; + }); + + this.connection.on("error", (error: Error) => { + console.error("RabbitMQ connection error:", error); + this.connection = null; + this.channel = null; + }); + } catch (error) { + console.error("Failed to connect to RabbitMQ:", error); + throw error; + } + } + + async subscribe( + messageHandler: (message: SearchQuery) => Promise + ): Promise { + if (!this.channel) { + throw new Error( + "RabbitMQ channel not initialized. Call connect() first." + ); + } + + console.log( + `Waiting for messages from ${this.config.queueName}. To exit press CTRL+C` + ); + + await this.channel.consume( + this.config.queueName, + async (msg: amqp.ConsumeMessage | null) => { + if (msg && this.channel) { + try { + const messageContent = msg.content.toString(); + console.log(`Received message: ${messageContent}`); + + const searchQuery: SearchQuery = JSON.parse(messageContent); + console.log(`Processing Glassdoor scrape request:`, searchQuery); + + // Process the message using the provided handler + await messageHandler(searchQuery); + + // Acknowledge the message on success + this.channel.ack(msg); + console.log("Message processed successfully"); + } catch (error) { + console.error("Error processing message:", error); + // Reject the message and don't requeue it + if (this.channel) { + this.channel.nack(msg, false, false); + } + } + } + } + ); + } + + async close(): Promise { + try { + if (this.channel) { + await this.channel.close(); + this.channel = null; + } + if (this.connection) { + await this.connection.close(); + this.connection = null; + } + console.log("RabbitMQ connection closed gracefully"); + } catch (error) { + console.error("Error closing RabbitMQ connection:", error); + } + } + + isConnected(): boolean { + return this.connection !== null && this.channel !== null; + } +} + +export function createRabbitMQConfig(): RabbitMQConfig { + return { + url: process.env.RABBITMQ_URL || "amqp://guest:guest@localhost:5672/", + queueName: process.env.GLASSDOOR_QUEUE_NAME || "scraper.glassdoor", + exchangeName: process.env.SCRAPER_EXCHANGE_NAME || "scraper_exchange", + exchangeType: "topic", + durable: true, + }; +} + +export async function createRabbitMQClient(): Promise { + const config = createRabbitMQConfig(); + const client = new RabbitMQClient(config); + await client.connect(); + return client; +}